/5 - Classification/autograder.py
Python | 348 lines | 335 code | 2 blank | 11 comment | 1 complexity | 2983db362c7c176cd21d36ef68322c46 MD5 | raw file
- # autograder.py
- # -------------
- # Licensing Information: Please do not distribute or publish solutions to this
- # project. You are free to use and extend these projects for educational
- # purposes. The Pacman AI projects were developed at UC Berkeley, primarily by
- # John DeNero (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
- # Student side autograding was added by Brad Miller, Nick Hay, and Pieter
- # Abbeel in Spring 2013.
- # For more info, see http://inst.eecs.berkeley.edu/~cs188/pacman/pacman.html
- # imports from python standard library
- import grading
- import imp
- import optparse
- import os
- import re
- import sys
- import projectParams
- import random
- random.seed(0)
- # register arguments and set default values
- def readCommand(argv):
- parser = optparse.OptionParser(description = 'Run public tests on student code')
- parser.set_defaults(generateSolutions=False, edxOutput=False, muteOutput=False, printTestCase=False, noGraphics=False, graphics=False)
- parser.add_option('--test-directory',
- dest = 'testRoot',
- default = 'test_cases',
- help = 'Root test directory which contains subdirectories corresponding to each question')
- parser.add_option('--student-code',
- dest = 'studentCode',
- default = projectParams.STUDENT_CODE_DEFAULT,
- help = 'comma separated list of student code files')
- parser.add_option('--code-directory',
- dest = 'codeRoot',
- default = "",
- help = 'Root directory containing the student and testClass code')
- parser.add_option('--test-case-code',
- dest = 'testCaseCode',
- default = projectParams.PROJECT_TEST_CLASSES,
- help = 'class containing testClass classes for this project')
- parser.add_option('--generate-solutions',
- dest = 'generateSolutions',
- action = 'store_true',
- help = 'Write solutions generated to .solution file')
- parser.add_option('--edx-output',
- dest = 'edxOutput',
- action = 'store_true',
- help = 'Generate edX output files')
- parser.add_option('--mute',
- dest = 'muteOutput',
- action = 'store_true',
- help = 'Mute output from executing tests')
- parser.add_option('--print-tests', '-p',
- dest = 'printTestCase',
- action = 'store_true',
- help = 'Print each test case before running them.')
- parser.add_option('--test', '-t',
- dest = 'runTest',
- default = None,
- help = 'Run one particular test. Relative to test root.')
- parser.add_option('--question', '-q',
- dest = 'gradeQuestion',
- default = None,
- help = 'Grade one particular question.')
- parser.add_option('--no-graphics',
- dest = 'noGraphics',
- action = 'store_true',
- help = 'No graphics display for pacman games.')
- parser.add_option('--graphics',
- dest = 'graphics',
- action = 'store_true',
- help = 'Display graphics for pacman games.')
- (options, args) = parser.parse_args(argv)
- return options
- # confirm we should author solution files
- def confirmGenerate():
- print 'WARNING: this action will overwrite any solution files.'
- print 'Are you sure you want to proceed? (yes/no)'
- while True:
- ans = sys.stdin.readline().strip()
- if ans == 'yes':
- break
- elif ans == 'no':
- sys.exit(0)
- else:
- print 'please answer either "yes" or "no"'
- # TODO: Fix this so that it tracebacks work correctly
- # Looking at source of the traceback module, presuming it works
- # the same as the intepreters, it uses co_filename. This is,
- # however, a readonly attribute.
- def setModuleName(module, filename):
- functionType = type(confirmGenerate)
- classType = type(optparse.Option)
- for i in dir(module):
- o = getattr(module, i)
- if hasattr(o, '__file__'): continue
-
- if type(o) == functionType:
- setattr(o, '__file__', filename)
- elif type(o) == classType:
- setattr(o, '__file__', filename)
- # TODO: assign member __file__'s?
- #print i, type(o)
-
- #from cStringIO import StringIO
- def loadModuleString(moduleSource):
- # Below broken, imp doesn't believe its being passed a file:
- # ValueError: load_module arg#2 should be a file or None
- #
- #f = StringIO(moduleCodeDict[k])
- #tmp = imp.load_module(k, f, k, (".py", "r", imp.PY_SOURCE))
- tmp = imp.new_module(k)
- exec moduleCodeDict[k] in tmp.__dict__
- setModuleName(tmp, k)
- return tmp
- import py_compile
- def loadModuleFile(moduleName, filePath):
- with open(filePath, 'r') as f:
- return imp.load_module(moduleName, f, "%s.py" % moduleName, (".py", "r", imp.PY_SOURCE))
-
- def readFile(path, root=""):
- "Read file from disk at specified path and return as string"
- with open(os.path.join(root, path), 'r') as handle:
- return handle.read()
- #######################################################################
- # Error Hint Map
- #######################################################################
- # TODO: use these
- ERROR_HINT_MAP = {
- 'q1': {
- "<type 'exceptions.IndexError'>": """
- We noticed that your project threw an IndexError on q1.
- While many things may cause this, it may have been from
- assuming a certain number of successors from a state space
- or assuming a certain number of actions available from a given
- state. Try making your code more general (no hardcoded indices)
- and submit again!
- """
- },
- 'q3': {
- "<type 'exceptions.AttributeError'>": """
- We noticed that your project threw an AttributeError on q3.
- While many things may cause this, it may have been from assuming
- a certain size or structure to the state space. For example, if you have
- a line of code assuming that the state is (x, y) and we run your code
- on a state space with (x, y, z), this error could be thrown. Try
- making your code more general and submit again!
- """
- }
- }
- import pprint
- def splitStrings(d):
- d2 = dict(d)
- for k in d:
- if k[0:2] == "__":
- del d2[k]
- continue
- if d2[k].find("\n") >= 0:
- d2[k] = d2[k].split("\n")
- return d2
-
- def printTest(testDict, solutionDict):
- pp = pprint.PrettyPrinter(indent=4)
- print "Test case:"
- for line in testDict["__raw_lines__"]:
- print " |", line
- print "Solution:"
- for line in solutionDict["__raw_lines__"]:
- print " |", line
-
- def runTest(testName, moduleDict, printTestCase=False, display=None):
- import testParser
- import testClasses
- for module in moduleDict:
- setattr(sys.modules[__name__], module, moduleDict[module])
-
- testDict = testParser.TestParser(testName + ".test").parse()
- solutionDict = testParser.TestParser(testName + ".solution").parse()
- test_out_file = os.path.join('%s.test_output' % testName)
- testDict['test_out_file'] = test_out_file
- testClass = getattr(projectTestClasses, testDict['class'])
- questionClass = getattr(testClasses, 'Question')
- question = questionClass({'max_points': 0}, display)
- testCase = testClass(question, testDict)
- if printTestCase:
- printTest(testDict, solutionDict)
-
- # This is a fragile hack to create a stub grades object
- grades = grading.Grades(projectParams.PROJECT_NAME, [(None,0)])
- testCase.execute(grades, moduleDict, solutionDict)
- # returns all the tests you need to run in order to run question
- def getDepends(testParser, testRoot, question):
- allDeps = [question]
- questionDict = testParser.TestParser(os.path.join(testRoot, question, 'CONFIG')).parse()
- if 'depends' in questionDict:
- depends = questionDict['depends'].split()
- for d in depends:
- # run dependencies first
- allDeps = getDepends(testParser, testRoot, d) + allDeps
- return allDeps
- # get list of questions to grade
- def getTestSubdirs(testParser, testRoot, questionToGrade):
- problemDict = testParser.TestParser(os.path.join(testRoot, 'CONFIG')).parse()
- if questionToGrade != None:
- questions = getDepends(testParser, testRoot, questionToGrade)
- if len(questions) > 1:
- print 'Note: due to dependencies, the following tests will be run: %s' % ' '.join(questions)
- return questions
- if 'order' in problemDict:
- return problemDict['order'].split()
- return sorted(os.listdir(testRoot))
- # evaluate student code
- def evaluate(generateSolutions, testRoot, moduleDict, exceptionMap=ERROR_HINT_MAP, edxOutput=False, muteOutput=False,
- printTestCase=False, questionToGrade=None, display=None):
- # imports of testbench code. note that the testClasses import must follow
- # the import of student code due to dependencies
- import testParser
- import testClasses
- for module in moduleDict:
- setattr(sys.modules[__name__], module, moduleDict[module])
-
- questions = []
- questionDicts = {}
- test_subdirs = getTestSubdirs(testParser, testRoot, questionToGrade)
- for q in test_subdirs:
- subdir_path = os.path.join(testRoot, q)
- if not os.path.isdir(subdir_path) or q[0] == '.':
- continue
-
- # create a question object
- questionDict = testParser.TestParser(os.path.join(subdir_path, 'CONFIG')).parse()
- questionClass = getattr(testClasses, questionDict['class'])
- question = questionClass(questionDict, display)
- questionDicts[q] = questionDict
- # load test cases into question
- tests = filter(lambda t: re.match('[^#~.].*\.test\Z', t), os.listdir(subdir_path))
- tests = map(lambda t: re.match('(.*)\.test\Z', t).group(1), tests)
- for t in sorted(tests):
- test_file = os.path.join(subdir_path, '%s.test' % t)
- solution_file = os.path.join(subdir_path, '%s.solution' % t)
- test_out_file = os.path.join(subdir_path, '%s.test_output' % t)
- testDict = testParser.TestParser(test_file).parse()
- if testDict.get("disabled", "false").lower() == "true":
- continue
- testDict['test_out_file'] = test_out_file
- testClass = getattr(projectTestClasses, testDict['class'])
- testCase = testClass(question, testDict)
- def makefun(testCase, solution_file):
- if generateSolutions:
- # write solution file to disk
- return lambda grades: testCase.writeSolution(moduleDict, solution_file)
- else:
- # read in solution dictionary and pass as an argument
- testDict = testParser.TestParser(test_file).parse()
- solutionDict = testParser.TestParser(solution_file).parse()
- if printTestCase:
- return lambda grades: printTest(testDict, solutionDict) or testCase.execute(grades, moduleDict, solutionDict)
- else:
- return lambda grades: testCase.execute(grades, moduleDict, solutionDict)
- question.addTestCase(testCase, makefun(testCase, solution_file))
- # Note extra function is necessary for scoping reasons
- def makefun(question):
- return lambda grades: question.execute(grades)
- setattr(sys.modules[__name__], q, makefun(question))
- questions.append((q, question.getMaxPoints()))
- grades = grading.Grades(projectParams.PROJECT_NAME, questions, edxOutput=edxOutput, muteOutput=muteOutput)
- if questionToGrade == None:
- for q in questionDicts:
- for prereq in questionDicts[q].get('depends', '').split():
- grades.addPrereq(q, prereq)
- grades.grade(sys.modules[__name__], bonusPic = projectParams.BONUS_PIC)
- return grades.points
- def getDisplay(graphicsByDefault, options=None):
- graphics = graphicsByDefault
- if options != None:
- if options.graphics:
- graphics = True
- if options.noGraphics:
- graphics = False
- if graphics:
- import graphicsDisplay
- return graphicsDisplay.PacmanGraphics(1, frameTime=.05)
- else:
- import textDisplay
- return textDisplay.NullGraphics()
- if __name__ == '__main__':
- options = readCommand(sys.argv)
- if options.generateSolutions:
- confirmGenerate()
- codePaths = options.studentCode.split(',')
- # moduleCodeDict = {}
- # for cp in codePaths:
- # moduleName = re.match('.*?([^/]*)\.py', cp).group(1)
- # moduleCodeDict[moduleName] = readFile(cp, root=options.codeRoot)
- # moduleCodeDict['projectTestClasses'] = readFile(options.testCaseCode, root=options.codeRoot)
- # moduleDict = loadModuleDict(moduleCodeDict)
-
- moduleDict = {}
- for cp in codePaths:
- moduleName = re.match('.*?([^/]*)\.py', cp).group(1)
- moduleDict[moduleName] = loadModuleFile(moduleName, os.path.join(options.codeRoot, cp))
- moduleName = re.match('.*?([^/]*)\.py', options.testCaseCode).group(1)
- moduleDict['projectTestClasses'] = loadModuleFile(moduleName, os.path.join(options.codeRoot, options.testCaseCode))
-
-
- if options.runTest != None:
- runTest(options.runTest, moduleDict, printTestCase=options.printTestCase, display=getDisplay(True, options))
- else:
- evaluate(options.generateSolutions, options.testRoot, moduleDict,
- edxOutput=options.edxOutput, muteOutput=options.muteOutput, printTestCase=options.printTestCase,
- questionToGrade=options.gradeQuestion, display=getDisplay(options.gradeQuestion!=None, options))