debug step 4 perf. issues

.gitignore

@@ -0,0 +1,10 @@
+# Pycache
+/__pycache__/*
+/util/__pycache__/*
+/scheduleGenerator/__pycache__/*
+
+# Test
+/*_test.py
+
+# Test output
+/output/*

README.md

@@ -0,0 +1,46 @@
+# Schedule Generator
+
+Hello there! This is the standalone repository for the schedule generator from my [school-management-api](https://github.com/SowinskiBraeden/school-management-api) project.
+
+The algorithm reads student requests and generates a timetable for all requested classes and fills them with students, giving the students their own schedules.
+
+The algorithm has been moved out of `tinker.py` into its own python script. Version 1 and 2 are removed and the script can be found in the `/scheduleGenerator` folder. The algorithm has come a long way; using real 2018 course selection data from my school, enabling me to better test the script. V3 has a entirely different approach from V1 and V2, that you can read about at the top of the function in [`generator.py`](/scheduleGenerator/generator.py). The function is broken up into 6 steps, each step is labeled within the function with a comment, giving a brief explination of what that step contributes to the algorithm.
+
+Here is the list of commands.
+1. Collect latest data from files and generate timetable
+```
+  $ ./tinker.py
+```
+
+2. Collect Data and store it in files for later
+```
+  $ ./tinker.py generate_data
+```
+
+3. Generate timetable from existing files (faster), requires last command run once
+```
+  $ ./tinker.py no_refresh
+```
+
+4. More details on errors
+```
+  $ ./tinker.py errors
+```
+
+<br>
+
+This is the output after running it with the following command.
+```
+  $ ./tinker.py no_refresh
+```
+
+![preview](/preview/generator-preview.png)
+
+<br>
+
+A more in depth error log can be read with the additional argument
+```
+  $ ./tinker.py errors
+```
+
+![preview](/preview/error-preview.png)

error.py

@@ -0,0 +1,20 @@
+#! python
+import json
+
+with open('./output/students.json') as f: students = json.load(f)
+
+couldnt_resolve = 0
+missing_classes = 0
+acceptable_missing_classes = 0
+
+for student in students:
+  blocks = [student["schedule"][block] for block in student["schedule"]]
+  conflicts = sum(1 for b in blocks if len(b)>1)
+  if conflicts == 0 and student["classes"] == student["expectedClasses"]: continue
+  if conflicts > 0: couldnt_resolve += 1
+  if (student["expectedClasses"] - 2) <= student["classes"] < student["expectedClasses"]: acceptable_missing_classes += 1
+  if student["classes"] < (student["expectedClasses"] - 2): missing_classes += 1
+
+print(f"Couldn't resolve            :  {couldnt_resolve}/{len(students)} - {round((couldnt_resolve/len(students))*100, 2)}%")
+print(f"Missing classes             :  {missing_classes}/{len(students)}  - {round((missing_classes/len(students))*100, 2)}%")
+print(f"Acceptable Missing classes  :  {acceptable_missing_classes}/{len(students)} - {round((acceptable_missing_classes/len(students))*100, 2)}%")

scheduleGenerator/generator.py

@@ -0,0 +1,550 @@
+#! python
+import json
+import random
+from inspect import currentframe
+from string import hexdigits
+
+# Import from custom utilities
+from util.mockStudents import getSampleStudents
+from util.generateCourses import getSampleCourses
+
+'''
+  Block 1-5 is first semester while
+  block 6-10 is second semester
+
+
+  schedule example:
+  schedule: {
+    "block1": "className",
+    "block2": "className",
+    "block3": "className",
+    ...
+  }
+
+  running example:
+  running: {
+    "block1": {
+      classCode: {
+        "className": name,
+        "students": [student Name]
+      },
+      classCode: {
+        "className": name,
+        "students": [student Name]
+      }
+    },
+    ...
+  }
+'''
+
+exists = lambda n : True if n not in ('', None) else False
+getLineNumber = lambda : currentframe().f_back.f_lineno
+
+# Takes in information to create or add a new conflict
+# Returns if the particular student has a previous error
+def newConflict(pupilNum: str, email: str, conflictType: str, code: str, description: str, logs: dict) -> bool:
+  exists = True if pupilNum in logs else False
+  log = {
+    "Pupil #": pupilNum,
+    "Email": email,
+    "Type": conflictType,
+    "Code": code,
+    "Conflict": description
+  }
+  if exists: logs[pupilNum].append(log)
+  else: logs[pupilNum] = [log]
+  return exists if conflictType == "Critical" else False
+
+minReq, median, classCap = 18, 24, 30
+running = {
+  "block1": {},
+  "block2": {},
+  "block3": {},
+  "block4": {},
+  "block5": {},
+  "block6": {},
+  "block7": {},
+  "block8": {},
+  "block9": {},
+  "block10": {}
+}
+
+# These are the codes for Flex (spare) blocks
+# Semester 1 and 2
+flex = ("XAT--12A-S", "XAT--12B-S")
+
+# V3 differs a lot by V1/2 as it does not focus on fitting the classes
+# into the time table first.
+# It starts by trying to get all classes full and give all students a full class list.
+# Then it starts to attempt to fit all classes into a timetable, making corretions along
+# the way. Corrections being moving a students class
+def generateScheduleV3(
+  students: list, # Refer to ../util/mockStudents.py to see the students list structure
+  courses: dict, # Reger to ../util/generateCourses.py to see the courses dictionary structure
+  blockClassLimit: int=40, # Block class limit is the number of classrooms available per block. Default 40 classes per block
+  studentsDir: str="../output/students.json",
+  conflictsDir: str="../output/conflicts.json"
+  ) -> dict[str, dict]: # Returns the completed 'running' dictionary from above
+  
+
+  def equal(l: list) -> list: # Used to equalize list of numbers
+    q,r = divmod(sum(l),len(l))
+    return [q+1]*r + [q]*(len(l)-r)
+
+
+  # Step 1 - Calculate which classes can run
+  activeCourses = {}
+  for student in students:
+    # Tally class request
+    for request in (request for request in student["requests"] if not request["alt"] and request["CrsNo"] not in flex):
+      code = request["CrsNo"]
+      courses[code]["Requests"] += 1
+      # Add course to active list if enough requests
+      if courses[code]["Requests"] > minReq and courses[code]["CrsNo"] not in activeCourses:
+        activeCourses[code] = courses[code]
+
+
+  # Step 2 - Generate empty classes
+  allClassRunCounts = []
+  courseRunInfo = {} # Generated now, used in step 4
+  emptyClasses = {} # List of all classes with how many students should be entered during step 3
+  # calculate number of times to run class
+  for i in range(len(activeCourses)):
+    index = list(activeCourses)[i]
+    if index not in emptyClasses: emptyClasses[index] = {}
+    classRunCount = activeCourses[index]["Requests"] // median
+    remaining = activeCourses[index]["Requests"] % median
+
+    # Put number of classRunCount classes in emptyClasses
+    for j in range(classRunCount):
+      emptyClasses[index][f"{index}-{hexdigits[j]}"] = {
+        "CrsNo": index,
+        "Description": activeCourses[index]["Description"],
+        "expectedLen": median # Number of students expected in this class / may be altered later
+      }
+
+    # If remaining fit in open slots in existing classes
+    if remaining <= classRunCount * (classCap - median):
+      # Equally disperse remaining into existing classes
+      for j in range(classRunCount):
+        if remaining == 0: break
+        emptyClasses[index][f"{index}-{hexdigits[j]}"]["expectedLen"] += 1
+        remaining -= 1
+
+    # If we can create a class using remaining, create class
+    elif remaining >= minReq:
+      # Create a class using remaining
+      emptyClasses[index][f"{index}-{hexdigits[classRunCount]}"] = {
+        "CrsNo": index,
+        "Description": activeCourses[index]["Description"],
+        "expectedLen": remaining
+      }
+
+      classRunCount += 1
+      # If a class previously existed
+      if classRunCount >= 2:
+        # Equalize (level) class expectedLen's
+        expectedLengths = [emptyClasses[index][f"{index}-{hexdigits[j]}"]["expectedLen"] for j in range(classRunCount)]
+        newExpectedLens = equal(expectedLengths)
+        for j in range(len(newExpectedLens)):
+          emptyClasses[index][f"{index}-{hexdigits[j]}"]["expectedLen"] = newExpectedLens[j]
+
+    # Else if we can't fit remaining into available slots in existing classes,
+    # and it's unable to create its own class,
+    # and the required amount (minReq - remaining) to make a class is less than
+    # the number that existing classes can provide (classRunCount * (median - minReq))
+    elif minReq - remaining < classRunCount * (median - minReq):
+      # Take 1 from each class till min requirment met
+      for j in range(classRunCount):
+        emptyClasses[index][f"{index}-{hexdigits[j]}"]["expectedLen"] -= 1
+        remaining += 1
+        if remaining == minReq: break
+
+      # Create a class using remaining + required amount from existing classes
+      emptyClasses[index][f"{index}-{hexdigits[classRunCount]}"] = {
+        "CrsNo": index,
+        "Description": activeCourses[index]["Description"],
+        "expectedLen": remaining
+      }
+      
+      classRunCount += 1
+
+      # Equalize (level) class expectedLen's
+      expectedLengths = [emptyClasses[index][f"{index}-{hexdigits[j]}"]["expectedLen"] for j in range(classRunCount)]
+      newExpectedLens = equal(expectedLengths)
+      for j in range(len(newExpectedLens)):
+        emptyClasses[index][f"{index}-{hexdigits[j]}"]["expectedLen"] = newExpectedLens[j]
+
+    else:
+      # In the case that the remaining requests are unable to be resolved
+      # Fill as many requests into existing classes. Any left that can't fit,
+      # Will need to be ignored so later we can fold them into their alternative
+      # requests
+      for j in range(classRunCount):
+        if remaining == 0: break
+        if emptyClasses[index][f"{index}-{hexdigits[j]}"]["expectedLen"] < classCap: 
+          emptyClasses[index][f"{index}-{hexdigits[j]}"]["expectedLen"] += 1
+          remaining -= 1
+
+    courseRunInfo[index] = {
+      "Total": classRunCount,
+      "CrsNo": index
+    }
+    allClassRunCounts.append(classRunCount)
+
+  
+  # Step 3 - Fill 'emptyClasses' with Students
+  selectedCourses = {}
+  tempStudents = list(students)
+   
+  while len(tempStudents) > 0:
+    # Choose random student to prevent any success
+    # bias to students at the top of the list
+    student = tempStudents[random.randint(0, len(tempStudents)-1)]
+
+    alternates = [request for request in student["requests"] if request["alt"]]
+    for request in (request for request in student["requests"] if not request["alt"] and request["CrsNo"] not in flex):
+      course = request["CrsNo"]
+      getAvailableCourse = True
+      isAlt = False
+      while getAvailableCourse:
+        if course in emptyClasses:
+          # if course exists, get first available class
+          for cname in emptyClasses[course]:
+            if cname in selectedCourses:
+              if isAlt and emptyClasses[course][cname]["expectedLen"] < classCap:
+                emptyClasses[course][cname]["expectedLen"] += 1
+              if len(selectedCourses[cname]["students"]) < emptyClasses[course][cname]["expectedLen"]:
+                # Class exists with room for student
+                selectedCourses[cname]["students"].append({
+                  "Pupil #": student["Pupil #"],
+                  "index": student["studentIndex"]
+                })
+                getAvailableCourse = False
+                break
+              elif len(selectedCourses[cname]["students"]) == emptyClasses[course][cname]["expectedLen"]:
+                # If class is full, and there's no more classes available for that course
+                if cname[len(cname)-1] == f"{len(emptyClasses[course])-1}":
+                  if len(alternates) > 0:
+                    # Use alternate
+                    course = alternates[0]["CrsNo"]
+                    alternates.remove(alternates[0])
+                    isAlt = True
+                    break
+                  else:
+                    # Force break loop, ignore and let an admin
+                    # handle options to solve for missing class
+                    getAvailableCourse = False
+                    break
+            elif cname not in selectedCourses:
+              selectedCourses[cname] = {
+                "students": [{
+                  "Pupil #": student["Pupil #"],
+                  "index": student["studentIndex"]
+                }],
+                "CrsNo": course,
+                "Description": courses[course]["Description"]
+              }
+              getAvailableCourse = False
+              break
+
+        elif course not in emptyClasses:
+          if len(alternates) > 0:
+            # Use alternate
+            course = alternates[0]["CrsNo"]
+            alternates.remove(alternates[0])
+            isAlt = True
+          else:
+            # Force break loop, ignore and let an admin
+            # handle options to solve for missing class
+            getAvailableCourse = False
+
+    students[student["studentIndex"]]["remainingAlts"] = alternates
+    tempStudents.remove(student)
+
+
+  # Step 4 - Attempt to fit classes into timetable
+  def stepIndex(offset: int, stepType: int) -> int:
+    # stepType 0 is for stepping between first and second semester
+    if stepType == 0: return 5 if offset == 0 or offset == -4 else -4
+    
+    # stepType 1 is for stepping between second and first semester
+    elif stepType == 1: return -5 if offset == 0 or offset == 6 else 6
+
+    # Return Error if code is altered to cause error
+    else: raise SystemExit(f"Invalid 'stepType' in func 'stepIndex' line {getLineNumber()}")
+
+  # Create copy for step 6
+  courseRunInfoCopy = dict(courseRunInfo)
+
+  while len(allClassRunCounts) > 0:
+    # Get highest resource class (most times run)
+    index = allClassRunCounts.index(max(allClassRunCounts))
+    course = list(courseRunInfo)[index]
+
+    # Tally first and second semester
+    sem1, sem2 = 0, 0
+    sem1List, sem2List = {}, {}
+    for i in range(1, 11):
+      if i <= 5:
+        sem1 += len(running[f"block{i}"])
+        sem1List[f"block{i}"] = running[f"block{i}"]
+      elif i > 5:
+        sem2 += len(running[f"block{i}"])
+        sem2List[f"block{i}"] = running[f"block{i}"]
+
+    # If there is more than one class Running
+    if allClassRunCounts[index] > 1:
+      blockIndex = 0 if sem1 <= sem2 else 5
+      stepType = 0 if sem1 <= sem2 else 1
+      offset = 0
+
+      # Spread classes throughout both semesters
+      for i in range(courseRunInfo[course]["Total"]):
+        cname = f"{course}-{hexdigits[i]}"
+        classInserted = False
+        while not classInserted:
+
+          blockIndex += offset
+          if len(running[list(running)[blockIndex]]) < blockClassLimit:
+            running[list(running)[blockIndex]][cname] = {
+              "CrsNo": course,
+              "Description": emptyClasses[course][cname]["Description"],
+              "students": selectedCourses[cname]["students"]
+            }
+            allClassRunCounts[index] -= 1
+            classInserted = True
+
+          offset = stepIndex(offset, stepType)
+
+          if blockIndex >= 9:
+            blockIndex = 0 if sem1 <= sem2 else 5
+            offset = 0
+
+    # If the class only runs once, place in semester with least classes
+    elif allClassRunCounts[index] == 1:
+      # Equally disperse into semesters classes
+      semBlocks = []
+      offset = 1
+
+      # If sem1 is less than or equal to sem2, add to sem1
+      if sem1 <= sem2: [semBlocks.append(len(block)) for block in sem1List]
+
+      # If sem2 is less than sem1, add to sem2
+      elif sem1 > sem2: [semBlocks.append(len(block)) for block in sem2List]
+
+      # Get block with least classes
+      leastBlock = semBlocks.index(min(semBlocks))
+      cname = f"{course}-0"
+
+      running[f"block{leastBlock+offset}"][cname] = {
+        "CrsNo": course,
+        "Description": emptyClasses[course][cname]["Description"],
+        "students": selectedCourses[cname]["students"],
+      }
+
+      allClassRunCounts[index] -= 1
+
+    # Remove course when fully inserted
+    if allClassRunCounts[index] == 0:
+      allClassRunCounts.remove(allClassRunCounts[index])
+      courseRunInfo.pop(list(courseRunInfo)[index])
+
+
+  # Step 5 - Fill student schedule
+  for block in running:
+    for cname in running[block]:
+      for student in running[block][cname]["students"]:
+        students[student["index"]]["schedule"][block].append(cname)
+        students[student["index"]]["classes"] += 1
+
+
+  # Step 6 - Evaluate, move students to fix conflicts
+  conflictLogs = {}
+  criticalCount, acceptableCount = 0, 0
+  c_mc_count, c_cr_count, a_mc_count = 0, 0, 0
+  studentsCritical, studentsAcceptable = 0, 0
+
+  for student in students:
+    blocks = [student["schedule"][block] for block in student["schedule"]]
+    hasConflicts = True if sum(1 for b in blocks if len(b)>1) > 0 else False
+
+    # If there is no conflicts
+    # and classes inserted to is equal to expectedClasses
+    # or classes the student is inserted to is missing
+    # no more than two classes:
+    # continue to next student
+    if not hasConflicts and student["classes"] == student["expectedClasses"]: continue
+    elif not hasConflicts and (student["expectedClasses"]-2) <= student["classes"] < student["expectedClasses"]:
+      a_mc_count += 1
+      acceptableCount += 1
+      if not newConflict(student["Pupil #"], "", "Acceptable", "A-MC", "Missing 1-2 Classses", conflictLogs): studentsAcceptable += 1
+      continue
+    
+    studentData = {
+      "Pupil #": student["Pupil #"],
+      "index": student["studentIndex"]
+    }
+
+    if hasConflicts:
+      # Clear student schedule to restructure
+      for block in student["schedule"]:
+        [running[block][cname]["students"].remove(studentData) for cname in student["schedule"][block]]
+        student["schedule"][block] = []
+
+      # Find class in student schedule that's run the least
+      classes, runCounts = [], []
+      for block in blocks:
+        for cname in block:
+          classes.append(cname[:-2])
+          runCounts.append(courseRunInfoCopy[cname[:-2]]["Total"])
+
+      # Rebuild student schedule
+      availableBlocks = [f'block{i}' for i in range(1, 11)]
+      while len(classes) > 0:
+        index = runCounts.index(min(runCounts)) # Get class least run
+        found = False
+
+        # find slot for class
+        for block in availableBlocks:
+          if found: break
+          for cname in running[block]:
+            if cname[:-2] == classes[index] and len(running[block][cname]["students"]) < classCap:
+              running[block][cname]["students"].append(studentData)
+              student["schedule"][block].append(cname)
+              availableBlocks.remove(block)
+              found = True
+              break
+
+        if not found:
+          # Determine all places class exists
+          existsIn, existingClassNames = [], []
+          for block in running:
+            for cname in running[block]:
+              if cname[:-2] == classes[index] and len(running[block][cname]["students"]) < classCap:
+                existsIn.append(block)
+                existingClassNames.append(cname)
+
+          # Attempt to fix
+          solution = False
+          if len(existsIn) > 0:
+            for i, existing in enumerate(existsIn):
+              if solution: break
+              classOut = student["schedule"][existing][0]
+              for block in running:
+                if solution: break
+                if block == existing or block not in availableBlocks: continue
+                for cname in running[block]:
+                  if cname[:-2] == classOut[:-2] and len(running[block][cname]["students"]) < classCap:
+                    # Move to existing class elsewhere
+                    student["schedule"][block].append(cname)
+                    running[block][cname]["students"].append(studentData)
+
+                    # Overwrite old class
+                    running[existing][student["schedule"][existing][0]]["students"].remove(studentData)
+                    student["schedule"][existing][0] = existingClassNames[i]
+                    running[existing][existingClassNames[i]]["students"].append(studentData)
+                    
+                    solution = True
+                    break
+
+          if not solution:
+            # Try alternate
+            alternates = [alt["CrsNo"] for alt in students[student["studentIndex"]]["remainingAlts"] if alt["CrsNo"] not in flex and alt["CrsNo"] in courseRunInfoCopy]
+            if len(alternates) == 0: # If no alternates, create critical error
+              c_cr_count += 1
+              criticalCount += 1
+              if not newConflict(student["Pupil #"], "", "Critical", "C-CR", "Couldn't Resolve", conflictLogs): studentsCritical += 1
+
+            else:
+              # Get alternate least run
+              altRunCounts = [courseRunInfoCopy[alt]["Total"] for alt in alternates]
+              altIndex = altRunCounts.index(min(altRunCounts))
+
+              # add alternate
+              classes.append(alternates[altIndex])
+              runCounts.append(altRunCounts[altIndex])
+
+              # Remove alternate from remaining alternates
+              for remaining in students[student["studentIndex"]]["remainingAlts"]:
+                if remaining["CrsNo"] == alternates[altIndex]:
+                  students[student["studentIndex"]]["remainingAlts"].remove(remaining)
+
+        # Remove class after inserted or failed to insert
+        classes.remove(classes[index])
+        runCounts.remove(runCounts[index])
+
+    metSelfRequirements = True if student["classes"] == student["expectedClasses"] else False
+    if not metSelfRequirements:
+      
+      if (student["expectedClasses"] - 2) <= student["classes"] < student["expectedClasses"]:
+        a_mc_count += 1
+        acceptableCount += 1
+        if not newConflict(student["Pupil #"], "", "Acceptable", "A-MC", "Missing 1-2 Classses", conflictLogs): studentsAcceptable += 1
+
+      elif student["classes"] < (student["expectedClasses"] - 2):
+        # Difference between classes inserted to and
+        # expected classes is too great
+        if student["Pupil #"] in conflictLogs:
+          c_mc_count += 1
+          criticalCount += 1
+          if not newConflict(student["Pupil #"], "", "Critical", "C-MC", "Missing too many Classses", conflictLogs): studentsCritical += 1
+
+      else:
+        print(f"Fatal error ({getLineNumber()}): Impossible error") # Just in case
+
+  finalConflictLogs = {
+    "Conflicts": conflictLogs,
+    "Critical": {
+      "Total": criticalCount,
+      "Students": studentsCritical,
+      "Errors": [{
+        "Total": c_mc_count,
+        "Description": "Missing too many Classes",
+        "Code": "C-MC"
+      }, {
+        "Total": c_cr_count,
+        "Description": "Couldn't Resolve",
+        "Code": "C-CR"
+      }]
+    },
+    "Acceptable": {
+      "Total": acceptableCount,
+      "Students": studentsAcceptable,
+      "Errors": [{
+        "Total": a_mc_count,
+        "Description": "Missing 1-2 Classes",
+        "Code": "A-MC"
+      }]
+    }
+  }
+
+  # Log Conflict to records
+  with open(conflictsDir, "w") as outfile:
+    json.dump(finalConflictLogs, outfile, indent=2)
+
+  # Insert flex (spare) course codes to empty blocks
+  for student in students:
+    for block in student["schedule"]:
+      if len(student["schedule"][block]) == 0:
+        student["schedule"][block].append(flex[0]) if int(block[5:]) <= 5 else student["schedule"][block].append(flex[1])
+
+  # Update Student records
+  with open(studentsDir, "w") as outfile:
+      json.dump(students, outfile, indent=2)
+  
+  return running
+
+if __name__ == '__main__':
+  print("Processing...")
+
+  sampleStudents = getSampleStudents(True)
+  samplemockCourses = getSampleCourses(True) 
+  timetable = {}
+  timetable["Version"] = 3
+  timetable["timetable"] = generateScheduleV3(sampleStudents, samplemockCourses)
+
+  with open("../output/timetable.json", "w") as outfile:
+    json.dump(timetable, outfile, indent=2)
+
+  print("Done")

tinker.py

@@ -0,0 +1,167 @@
+#! python
+from os.path import exists as file_exists
+from prettytable import PrettyTable
+from typing import Tuple
+from time import time, sleep
+import itertools
+import threading
+import json
+import sys
+
+# Import required utilities
+from util.mockStudents import getSampleStudents
+from util.generateCourses import getSampleCourses
+
+# Import Algorithm
+from scheduleGenerator.generator import generateScheduleV3
+
+done = False
+
+# consts
+blockClassLimit = 68
+
+print(f'\n>> Debug: Current blockClassLimit = {blockClassLimit}')
+
+def processing(msg: str):
+  for c in itertools.cycle(['|', '/', '-', '\\']):
+    if done: break
+    sys.stdout.write(f'\r{msg} {c}')
+    sys.stdout.flush()
+    sleep(0.1)
+
+def errorOutput(students): #-> Tuple[PrettyTable, dict, dict]:
+  # Error Table calulation / output  
+  f = open('./output/conflicts.json')
+  conflicts = json.load(f)
+  f.close()
+  totalCritical = conflicts["Critical"]["Students"]
+  totalAcceptable = conflicts["Acceptable"]["Students"]
+
+  t = PrettyTable(['Type', 'Error %', 'Success %', 'Student Error Ratio'])
+  
+  errorsC = round(totalCritical / len(students) * 100, 2)
+  successC = round(100 - errorsC, 2)
+  errorsA = round(totalAcceptable / len(students) * 100, 2)
+  successA = round(100 - errorsA, 2)
+  
+  t.add_row(['Critical', f"{errorsC} %", f"{successC} %", f"{totalCritical}/{len(students)} Students"])
+  t.add_row(['Acceptable', f"{errorsA} %", f"{successA} %", f"{totalAcceptable}/{len(students)} Students"])
+  
+  return t, conflicts["Critical"], conflicts["Acceptable"]
+
+if __name__ == '__main__':
+  
+  if len(sys.argv) == 1:
+    print()
+
+    st = time() # Start time
+
+    t = threading.Thread(target=processing, args=('Collection student requests',))
+    t.start() # Start animation
+    sampleStudents = getSampleStudents("./sample_data/course_selection_data.csv", True)
+    done = True # End Animation
+    print('\nStudent list generated.\n')
+
+    t = threading.Thread(target=processing, args=('Collection Course Information',))
+    done = False # reset animation
+    t.start() # Start animation
+    sampleCourses = getSampleCourses("./sample_data/course_selection_data.csv", True)
+    done = True # End Animation
+    print('\nCourse Information Collected.\n')
+    
+    sleep(0.1)
+    t = threading.Thread(target=processing, args=('Processing',))
+    done = False # reset animation
+    t.start() # Start animation
+    timetable = {}
+    timetable["Version"] = 3
+    timetable["timetable"] = generateScheduleV3(sampleStudents, sampleCourses, blockClassLimit, "./output/students.json", "./output/conflicts.json")
+
+    done = True # End Animation
+    et = time() # End time
+    elapsed_time = round((et - st), 3) # Execution time
+    print(f'\n\nDone - Finished in {elapsed_time} seconds\n')
+
+    for e in sys.argv:
+      if e.lower() == 'showerror':
+        errors, _, _ = errorOutput(sampleStudents)
+        print(errors)
+        break
+
+    with open("./output/timetable.json", "w") as outfile:
+      json.dump(timetable, outfile, indent=2)
+
+  elif sys.argv[1].lower() == "no_refresh":
+    print()
+
+    st = time() # Start time
+
+    if not file_exists('./output/students.json') or not file_exists('./output/courses.json'):
+      print('\n No previous data exists, please use the following command\n./tinker.py generate_data')
+      exit()
+
+    with open('./output/students.json') as f: sampleStudents = json.load(f)
+    with open('./output/courses.json') as f: sampleCourses = json.load(f)
+
+    for student in sampleStudents:
+      student["remainingAlts"] = []
+      for i in range(1, 11): student["schedule"][f"block{i}"] = []
+      student["classes"] = 0
+    
+    t = threading.Thread(target=processing, args=('Processing',))
+    t.start() # Start animation
+    timetable = {}
+    timetable["Version"] = 3
+    timetable["timetable"] = generateScheduleV3(sampleStudents, sampleCourses, blockClassLimit, "./output/students.json", "./output/conflicts.json")
+
+    done = True # End Animation
+    et = time() # End time
+    elapsed_time = round((et - st), 3) # Execution time
+    print(f'\n\nDone - Finished in {elapsed_time} seconds\n')
+
+    for e in sys.argv:
+      if e.lower() == 'showerror':
+        errors, _, _ = errorOutput(sampleStudents)
+        print(errors)
+        break
+
+    with open("./output/timetable.json", "w") as outfile:
+      json.dump(timetable, outfile, indent=2)
+
+  elif sys.argv[1].lower() == "generate_data":
+    t = threading.Thread(target=processing, args=('Collection student requests',))
+    t.start() # Start animation
+    _ = getSampleStudents("./sample_data/course_selection_data.csv", True)
+    done = True # End Animation
+    print('\nStudent list generated.\n')
+
+    t = threading.Thread(target=processing, args=('Collection Course Information',))
+    done = False # reset animation
+    t.start() # Start animation
+    _ = getSampleCourses("./sample_data/course_selection_data.csv", True)
+    done = True # End Animation
+    print('\nCourse Information Collected.\n')
+    print('You can now use the following command\n./tinker.py no_refresh')
+
+  elif sys.argv[1].upper() == "ERRORS":
+    f = open('./output/students.json')
+    studentData = json.load(f)
+    f.close()
+    errors, critical, acceptable = errorOutput(studentData)
+    print()
+    print(errors)
+
+    print(f"\n{critical['Total']} critical errors")
+    for i in range(len(critical["Errors"])):
+      print(f"x{critical['Errors'][i]['Total']} {critical['Errors'][i]['Code']} Errors: Critical - {critical['Errors'][i]['Description']}")
+
+    print(f"\n{acceptable['Total']} acceptable errors")
+    for i in range(len(acceptable["Errors"])):
+      print(f"x{acceptable['Errors'][i]['Total']} {acceptable['Errors'][i]['Code']} Errors: Critical - {acceptable['Errors'][i]['Description']}")
+
+    exit()
+
+  else:
+    print("Invalid argument")
+    exit()
+

util/estimateGrade.py

@@ -0,0 +1,26 @@
+#! python
+import json
+
+# Get students
+try:
+  with open("../output/students.json") as studentFile: students = json.load(studentFile)
+except FileNotFoundError:
+  with open("./output/students.json") as studentFile: students = json.load(studentFile)
+
+flex = ("XAT--12A-S", "XAT--12B-S")
+
+most_frequent = lambda l : max(set(l), key = l.count)
+
+def getEstimatedGrade(pupil: dict) -> int:
+  grades = [] # List of all possible grades
+  for request in pupil["requests"]:
+    if request["CrsNo"] in flex: continue
+    for extractedGrade in [int(s) for s in request["CrsNo"].split("-") if s.isdigit()]:
+      grades.append(extractedGrade)
+
+  return None if len(grades) == 0 else most_frequent(grades) # Final estimate of grade
+
+if __name__ == '__main__':
+  # Define students grade level
+  for student in students:
+    student["gradelevel"] = getEstimatedGrade(student)

util/generateCourses.py

@@ -0,0 +1,36 @@
+#! python
+import json
+import csv
+
+realCourses = {}
+
+# Get all courses from real sample data
+def getSampleCourses(data_dir, log=False) -> dict:
+  with open(data_dir, newline='') as csvfile:
+    reader = csv.DictReader(csvfile)
+    for row in reader:
+      exists = False
+      for course in realCourses:
+        exists = True if realCourses[course]["CrsNo"] == row["CrsNo"] else False
+        if exists: break
+      if not exists:
+        realCourses[row["CrsNo"]] = {
+          "CrsNo": row["CrsNo"],
+          "Requests": 0,
+          "Description": row["Description"],
+          "Credits": 4,
+          "students": []
+        }
+
+  if log:
+    with open("./output/courses.json", "w") as outfile:
+      json.dump(realCourses, outfile, indent=2)
+        
+  return realCourses
+
+
+if __name__ == '__main__':
+  courseSet: dict = getSampleCourses("../sample_data/course_selection_data.csv")
+
+  with open("../output/courses.json", "w") as outfile:
+    json.dump(courseSet, outfile, indent=2)

util/mockStudents.py

@@ -0,0 +1,68 @@
+#! python
+import json
+import csv
+
+flex= ("XAT--12A-S", "XAT--12B-S")
+
+mockStudents: list[dict] = []
+
+# sort real sample data into usable dictionary
+def getSampleStudents(data_dir: str, log: bool = False) -> list[dict]:
+  with open(data_dir, newline='') as csvfile:
+    reader = csv.DictReader(csvfile)
+    for row in reader:
+      exists = False
+      for student in mockStudents:
+        exists = True if student["Pupil #"] == row["Pupil #"] else False
+        if exists: break
+      alternate = True if row["Alternate?"] == 'TRUE' else False
+      if exists:
+        if len(mockStudents[student["studentIndex"]]["requests"]) >= 10 and not alternate and row["CrsNo"] not in flex: alternate = True
+        mockStudents[student["studentIndex"]]["requests"].append({
+          "CrsNo": row["CrsNo"],
+          "Description": row["Description"],
+          "alt": alternate
+        })
+        if row["CrsNo"] not in flex and not alternate and mockStudents[student["studentIndex"]]["expectedClasses"] < 10:
+          mockStudents[student["studentIndex"]]["expectedClasses"] += 1
+      else:
+        newStudent = {
+          "Pupil #": row["Pupil #"],
+          "requests": [{
+            "CrsNo": row["CrsNo"],
+            "Description": row["Description"],
+            "alt": alternate
+          }],
+          "schedule": {
+            "block1": [],
+            "block2": [],
+            "block3": [],
+            "block4": [],
+            "block5": [],
+            "block6": [],
+            "block7": [],
+            "block8": [],
+            "block9": [],
+            "block10": []
+          },
+          "expectedClasses": 1,
+          "classes": 0,
+          "remainingAlts": [],
+          "studentIndex": len(mockStudents)
+        }
+        mockStudents.append(newStudent)
+
+  if log:
+    with open("./output/students.json", "w") as outfile:
+      json.dump(mockStudents, outfile, indent=2)
+
+  return mockStudents
+
+
+if __name__ == '__main__':
+  studentRequests: list[dict] = getSampleStudents("../sample_data/course_selection_data.csv")
+
+  with open("../output/students.json", "w") as outfile:
+    json.dump(studentRequests, outfile, indent=2)
+
+  print("done")