# logic_planTestClasses.py
# ------------------------
# Licensing Information:  You are free to use or extend these projects for
# educational purposes provided that (1) you do not distribute or publish
# solutions, (2) you retain this notice, and (3) you provide clear
# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
# 
# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
# The core projects and autograders were primarily created by John DeNero
# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
# Student side autograding was added by Brad Miller, Nick Hay, and
# Pieter Abbeel (pabbeel@cs.berkeley.edu).


import testClasses

import textDisplay
import graphicsDisplay
import layout
import pacman
import logicAgents 
from logicPlan import PlanningProblem
import logicPlan

import itertools

# Simple test case which evals an arbitrary piece of python code.
# The test is correct if the output of the code given the student's
# solution matches that of the instructor's.
        

class EvalTest(testClasses.TestCase):

    def __init__(self, question, testDict):
        super(EvalTest, self).__init__(question, testDict)
        self.preamble = compile(testDict.get('preamble', ""), "%s.preamble" % self.getPath(), 'exec')
        self.test = compile(testDict['test'], "%s.test" % self.getPath(), 'eval')
        self.success = testDict['success']
        self.failure = testDict['failure']

    def evalCode(self, moduleDict):
        bindings = dict(moduleDict)
        exec(self.preamble, bindings)
        return str(eval(self.test, bindings))

    def execute(self, grades, moduleDict, solutionDict):
        result = self.evalCode(moduleDict)
        if result == solutionDict['result']:
            grades.addMessage('PASS: %s' % self.path)
            grades.addMessage('\t%s' % self.success)
            return True
        else:
            grades.addMessage('FAIL: %s' % self.path)
            grades.addMessage('\t%s' % self.failure)
            grades.addMessage('\tstudent result: "%s"' % result)
            grades.addMessage('\tcorrect result: "%s"' % solutionDict['result'])

        return False

    def writeSolution(self, moduleDict, filePath):
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)
        handle.write('# The result of evaluating the test must equal the below when cast to a string.\n')

        handle.write('result: "%s"\n' % self.evalCode(moduleDict))
        handle.close()
        return True

    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT
        
class LogicTest(testClasses.TestCase):

    def __init__(self, question, testDict):
        super(LogicTest, self).__init__(question, testDict)
        self.preamble = compile(testDict.get('preamble', ""), "%s.preamble" % self.getPath(), 'exec')
        self.test = compile(testDict['test'], "%s.test" % self.getPath(), 'eval')
        self.success = testDict['success']
        self.failure = testDict['failure']

    def evalCode(self, moduleDict):
        bindings = dict(moduleDict)
        exec(self.preamble, bindings)
        return eval(self.test, bindings)

    def execute(self, grades, moduleDict, solutionDict):
        result = self.evalCode(moduleDict)
        result = map(lambda x: str(x), result)
        result = ' '.join(result)
        
        if result == solutionDict['result']:
            grades.addMessage('PASS: %s' % self.path)
            grades.addMessage('\t%s' % self.success)
            return True
        for i in range(100):
            solI = 'result' + str(i)
            if solI not in solutionDict:
                break
            if result == solutionDict[solI]:
                grades.addMessage('PASS: %s' % self.path)
                grades.addMessage('\t%s' % self.success)
                return True
        grades.addMessage('FAIL: %s' % self.path)
        grades.addMessage('\t%s' % self.failure)
        grades.addMessage('\tstudent result: "%s"' % result)
        grades.addMessage('\tcorrect result: "%s"' % solutionDict['result'])

        return False

    def writeSolution(self, moduleDict, filePath):
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)
        handle.write('# The result of evaluating the test must equal the below when cast to a string.\n')
        solution = self.evalCode(moduleDict)
        solution = map(lambda x: str(x), solution)
        handle.write('result: "%s"\n' % ' '.join(solution))
        handle.close()
        return True        

    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT

class PacphysicsTest(testClasses.TestCase):

    def __init__(self, question, testDict):
        super(PacphysicsTest, self).__init__(question, testDict)
        self.layoutText = testDict['layout']
        self.layoutName = testDict['layoutName']
        self.t = int(testDict['t'])
        self.soln_labels = ["pacphysicsAxioms"]
        self.axiom_type = testDict['axiomType']
        if self.axiom_type == 'sensor':
            self.sensorAxioms = logicPlan.sensorAxioms
            self.successorAxioms = logicPlan.allLegalSuccessorAxioms
        elif self.axiom_type == 'slam':
            self.sensorAxioms = logicPlan.SLAMSensorAxioms
            self.successorAxioms = logicPlan.SLAMSuccessorAxioms
        else:
            raise Exception('Bad test case!')

    def solution(self, logicPlan):
        lay = layout.Layout([l.strip() for l in self.layoutText.split('\n')])
        walls_list = lay.walls.data
        all_coords = lay.get_all_coords_list()
        non_outer_wall_coords = lay.get_non_outer_wall_coords_list()
        pacphysics_axioms = logicPlan.pacphysicsAxioms(self.t, all_coords, non_outer_wall_coords, walls_list, self.sensorAxioms, self.successorAxioms)
        return pacphysics_axioms

    def execute(self, grades, moduleDict, solutionDict):
        grades.addMessage('Testing pacphysicsAxioms')
        logicPlan = moduleDict['logicPlan']
        gold_solution = solutionDict[self.soln_labels[0]]

        solution = self.solution(logicPlan)

        gold_soln_clauses_list_being_conjoined = str(gold_solution)[1:-1].split(" & ")
        soln_clauses_list_being_conjoined = str(solution)[1:-1].split(" & ")

        # Check student used conjoin correctly; this is a weak check
        # after <=>, we get Action) | (Wall expresisons due to SLAM successor
        for soln_clause in soln_clauses_list_being_conjoined:
            if "<=>" in soln_clause:
                if self.axiom_type == 'sensor':
                    continue
                else:
                    break
            contains_open_parens = ("(" in soln_clause[1:-1]) or ("(" in soln_clause[1:-1])
            if contains_open_parens:
                grades.addMessage('FAIL: {}'.format(self.path))
                grades.addMessage('\tStudent solution does not combine sentences properly.')
                grades.addMessage('\tMake sure you append the items to join with "and",'
                    ' and conjoin at the end.')
                return False

        # Check number of clauses is correct.
        gold_soln_num_clauses_conjoined = len(gold_soln_clauses_list_being_conjoined)
        soln_num_clauses_conjoined = len(soln_clauses_list_being_conjoined)

        if gold_soln_num_clauses_conjoined != soln_num_clauses_conjoined:
            grades.addMessage('FAIL: {}'.format(self.path))
            grades.addMessage('\tStudent solution differed from autograder solution')
            grades.addMessage('\tNumber of clauses being conjoined in student solution: {}'.format(
                soln_num_clauses_conjoined))
            grades.addMessage('\tNumber of clauses being conjoined in correct solution: {}'.format(
                gold_soln_num_clauses_conjoined))
            return False

        for gold_clause in gold_soln_clauses_list_being_conjoined:
            if gold_clause not in soln_clauses_list_being_conjoined:
                grades.addMessage('FAIL: {}'.format(self.path))
                grades.addMessage('\tStudent solution does not contain clause {}'.format(gold_clause))
                return False

        if set(soln_clauses_list_being_conjoined) != set(gold_soln_clauses_list_being_conjoined):
            grades.addMessage('FAIL: {}'.format(self.path))
            grades.addMessage('\tStudent solution differed from autograder solution on clause set comparison')
            grades.addMessage('\tStudent solution: {}'.format(solution))
            grades.addMessage('\tCorrect solution: {}'.format(gold_solution))
            return False
        
        if sorted(str(solution)) != sorted(str(gold_solution)):
            grades.addMessage('FAIL: {}'.format(self.path))
            grades.addMessage('\tStudent solution differed from autograder solution on character list comparison')
            grades.addMessage('\tStudent solution: {}'.format(solution))
            grades.addMessage('\tCorrect solution: {}'.format(gold_solution))
            return False

        grades.addMessage('PASS: %s' % self.path)
        return True

    def writeSolution(self, moduleDict, filePath):
        logicPlan = moduleDict['logicPlan']
        # open file and write comments
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)

        print("Solving problem", self.layoutName)
        print(self.layoutText)

        solution = self.solution(logicPlan)

        print("Problem solved")

        handle.write('{}: "{}"\n'.format(self.soln_labels[0], str(solution)))
        handle.close()

    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT

class LocationSatisfiabilityTest(testClasses.TestCase):

    def __init__(self, question, testDict):
        super(LocationSatisfiabilityTest, self).__init__(question, testDict)
        self.layoutText = testDict['layout']
        self.layoutName = testDict['layoutName']
        self.x0_y0 = eval(testDict['x0_y0'])
        self.action0 = testDict['action0']
        self.x1_y1 = eval(testDict['x1_y1'])
        self.action1 = testDict['action1']
        self.soln_labels = ["model_at_x1_y1_1", "model_not_at_x1_y1_1"]

    def solution(self, logicPlan):
        lay = layout.Layout([l.strip() for l in self.layoutText.split('\n')])
        pac = logicAgents.CheckSatisfiabilityAgent('checkLocationSatisfiability', 'LocMapProblem', logicPlan)
        ghosts = []
        disp = textDisplay.NullGraphics()
        games = next(pacman.runGames(lay, pac, ghosts, disp, 1, False, catchExceptions=True, timeout=180))
        loc_sat_models = logicPlan.checkLocationSatisfiability(self.x1_y1, self.x0_y0, self.action0, self.action1, pac.problem)
        return loc_sat_models

    def execute(self, grades, moduleDict, solutionDict):
        grades.addMessage('Testing checkLocationSatisfiability')
        logicPlan = moduleDict['logicPlan']

        solution = self.solution(logicPlan)

        for i, solution_i in enumerate(solution):
            gold_solution_i = solutionDict[self.soln_labels[i]]
            solution_i = logicPlan.modelToString(solution_i)

            if gold_solution_i == "False" and solution_i != "False":
                grades.addMessage('FAIL: {}'.format(self.path))
                grades.addMessage('\tStudent solution differed from autograder solution for {}'.format(self.soln_labels[i]))
                grades.addMessage('\tStudent model found satisfiable solution but no satisfiable solution exists.')
                return False
            elif gold_solution_i != "False" and solution_i == "False":
                grades.addMessage('FAIL: {}'.format(self.path))
                grades.addMessage('\tStudent solution differed from autograder solution for {}'.format(self.soln_labels[i]))
                grades.addMessage('\tStudent model found no satisfiable solution when a satisfiable solution exists.')
                return False
            elif gold_solution_i == "False" and solution_i == "False":
                continue
            else:
                pass

            gold_solution_i_str_pairs_list = gold_solution_i[2:-2].split("), (")
            gold_solution_i_tuples_list = [tuple(pair.split(", ")) for pair in gold_solution_i_str_pairs_list]
            gold_solution_i_dict = dict(gold_solution_i_tuples_list)
            solution_i_str_pairs_list = solution_i[2:-2].split("), (")
            solution_i_tuples_list = [tuple(pair.split(", ")) for pair in solution_i_str_pairs_list]
            solution_i_dict = dict(solution_i_tuples_list)

            # Check if student has all of the correct variables.
            gold_solution_i_num_vars = len(gold_solution_i_tuples_list)
            solution_i_num_vars = len(solution_i_tuples_list)
            if gold_solution_i_num_vars != solution_i_num_vars:
                grades.addMessage('FAIL: {}'.format(self.path))
                grades.addMessage('\tStudent solution differed from autograder solution')
                grades.addMessage('\tNumber of variables in student solution: {}'.format(
                    solution_i_num_vars))
                grades.addMessage('\tNumber of variables in correct solution: {}'.format(
                    gold_solution_i_num_vars))
                return False

            for gold_solution_var in gold_solution_i_dict:
                if gold_solution_var not in solution_i_dict:
                    grades.addMessage('FAIL: {}'.format(self.path))
                    grades.addMessage('\tStudent solution does not contain the same variables as correct solution')
                    grades.addMessage('\tCorrect solution variable missing in student solution: {}'.format(
                        gold_solution_var))
                    return False

            # Some miscellaneous inequality; return which variables are different between solution and student.
            for key in gold_solution_i_dict:
                if gold_solution_i_dict[key] != solution_i_dict[key]:
                    grades.addMessage('FAIL: {}'.format(self.path))
                    grades.addMessage('\tStudent model does not assign the correct value for variable {}'.format(key))
                    grades.addMessage('\tStudent value for {}: {}'.format(key, solution_i_dict[key]))
                    grades.addMessage('\tCorrect value for {}: {}'.format(key, gold_solution_i_dict[key]))
                    if "WALL" in key:
                        grades.addMessage('\tDouble check that you are loading the map properly.')
                    return False

            if str(solution_i) != str(gold_solution_i):
                grades.addMessage('FAIL: {}'.format(self.path))
                grades.addMessage('\tStudent solution differed from autograder solution for {}'.format(self.soln_labels[i]))
                grades.addMessage('\tStudent solution: {}'.format(solution_i))
                grades.addMessage('\tCorrect solution: {}'.format(gold_solution_i))
                return False

        grades.addMessage('PASS: %s' % self.path)
        return True

    def writeSolution(self, moduleDict, filePath):
        logicPlan = moduleDict['logicPlan']
        # open file and write comments
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)

        print("Solving problem", self.layoutName)
        print(self.layoutText)

        solution = self.solution(logicPlan)

        print("Problem solved")

        for i, solution_i in enumerate(solution):
            handle.write('{}: "{}"\n'.format(self.soln_labels[i], logicPlan.modelToString(solution_i)))
        handle.close()

    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT


class PositionProblemTest(testClasses.TestCase):

    def __init__(self, question, testDict):
        super(PositionProblemTest, self).__init__(question, testDict)
        self.layoutText = testDict['layout']
        self.layoutName = testDict['layoutName']

    def solution(self, logicPlan):
        lay = layout.Layout([l.strip() for l in self.layoutText.split('\n')])
        pac = logicAgents.LogicAgent('plp', 'PositionPlanningProblem', logicPlan)
        ghosts = []
        disp = textDisplay.NullGraphics()
        games = next(pacman.runGames(lay, pac, ghosts, disp, 1, False, catchExceptions=True, timeout=300))
        gameState = games[0].state
        return (gameState.isWin(), gameState.getScore(), pac.actions)

    def execute(self, grades, moduleDict, solutionDict):
        logicPlan = moduleDict['logicPlan']
        gold_path = solutionDict['solution_path']
        gold_score = int(solutionDict['solution_score'])

        solution = self.solution(logicPlan)

        if not solution[0] or solution[1] < gold_score:
            grades.addMessage('FAIL: %s' % self.path)
            grades.addMessage('\tpacman layout:\t\t%s' % self.layoutName)
            if solution[0]:
                result_str = "wins"
            else:
                result_str = "loses"
            grades.addMessage('\tstudent solution result: Pacman %s' % result_str)
            grades.addMessage('\tstudent solution score: %d' % solution[1])
            grades.addMessage('\tstudent solution path: %s' % ' '.join(solution[2]))
            if solution[1] < gold_score:
                grades.addMessage('Optimal solution not found.')
            grades.addMessage('')
            grades.addMessage('\tcorrect solution score: %d' % gold_score)
            grades.addMessage('\tcorrect solution path: %s' % gold_path)
            return False

        grades.addMessage('PASS: %s' % self.path)
        grades.addMessage('\tpacman layout:\t\t%s' % self.layoutName)
        grades.addMessage('\tsolution score:\t\t%d' % gold_score)
        grades.addMessage('\tsolution path:\t\t%s' % gold_path)
        return True

    def writeSolution(self, moduleDict, filePath):
        logicPlan = moduleDict['logicPlan']
        # open file and write comments
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)

        print("Solving problem", self.layoutName)
        print(self.layoutText)

        solution = self.solution(logicPlan)

        print("Problem solved")

        handle.write('solution_win: "%s"\n' % str(solution[0]))
        handle.write('solution_score: "%d"\n' % solution[1])
        handle.write('solution_path: "%s"\n' % ' '.join(solution[2]))
        handle.close()

    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT


class FoodProblemTest(testClasses.TestCase):

    def __init__(self, question, testDict):
        super(FoodProblemTest, self).__init__(question, testDict)
        self.layoutText = testDict['layout']
        self.layoutName = testDict['layoutName']

    def solution(self, logicPlan):
        lay = layout.Layout([l.strip() for l in self.layoutText.split('\n')])
        pac = logicAgents.LogicAgent('flp', 'FoodPlanningProblem', logicPlan)
        ghosts = []
        disp = textDisplay.NullGraphics()
        games = next(pacman.runGames(lay, pac, ghosts, disp, 1, False, catchExceptions=True, timeout=300))
        gameState = games[0].state
        return (gameState.isWin(), gameState.getScore(), pac.actions)

    def execute(self, grades, moduleDict, solutionDict):
        logicPlan = moduleDict['logicPlan']
        gold_path = solutionDict['solution_path']
        gold_score = int(solutionDict['solution_score'])

        solution = self.solution(logicPlan)

        if not solution[0] or solution[1] < gold_score:
            grades.addMessage('FAIL: %s' % self.path)
            grades.addMessage('\tpacman layout:\t\t%s' % self.layoutName)
            if solution[0]:
                result_str = "wins"
            else:
                result_str = "loses"
            grades.addMessage('\tstudent solution result: Pacman %s' % result_str)
            grades.addMessage('\tstudent solution score: %d' % solution[1])
            grades.addMessage('\tstudent solution path: %s' % ' '.join(solution[2]))
            if solution[1] < gold_score:
                grades.addMessage('Optimal solution not found.')
            grades.addMessage('')
            grades.addMessage('\tcorrect solution score: %d' % gold_score)
            grades.addMessage('\tcorrect solution path: %s' % gold_path)
            return False

        grades.addMessage('PASS: %s' % self.path)
        grades.addMessage('\tpacman layout:\t\t%s' % self.layoutName)
        grades.addMessage('\tsolution score:\t\t%d' % gold_score)
        grades.addMessage('\tsolution path:\t\t%s' % gold_path)
        return True

    def writeSolution(self, moduleDict, filePath):
        logicPlan = moduleDict['logicPlan']
        # open file and write comments
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)

        print("Solving problem", self.layoutName)
        print(self.layoutText)

        solution = self.solution(logicPlan)

        print("Problem solved")

        handle.write('solution_win: "%s"\n' % str(solution[0]))
        handle.write('solution_score: "%d"\n' % solution[1])
        handle.write('solution_path: "%s"\n' % ' '.join(solution[2]))
        handle.close()

    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT


class LocalizationProblemTest(testClasses.TestCase):

    def __init__(self, question, testDict):
        super(LocalizationProblemTest, self).__init__(question, testDict)
        self.layoutText = testDict['layout']
        self.layoutName = testDict['layoutName']
        self.scriptedActions = eval(testDict['actions'])

    def solution(self, logicPlan):
        lay = layout.Layout([l.strip() for l in self.layoutText.split('\n')])
        ghosts = []
        # TODO: Figure out if we can use no-graphics cleaner
        disp = self.question.display
        if isinstance(disp, graphicsDisplay.PacmanGraphics): # autograder.py has incorrect options
            disp = graphicsDisplay.PacmanGraphics(frameTime=0.5)
        pac = logicAgents.LocalizationLogicAgent(
            'loc', 'LocalizationProblem', logicPlan, display=disp, scripted_actions=self.scriptedActions)
        yield from pacman.runGames(lay, pac, ghosts, disp, 1, False, catchExceptions=True, timeout=300)

    def execute(self, grades, moduleDict, solutionDict):
        logicPlan = moduleDict['logicPlan']
        gold_solution = eval(solutionDict['possible_locations_per_timestep'])

        num_timesteps = 0
        for t, solution in enumerate(self.solution(logicPlan)):
            if solution is None:
                num_timesteps = t
                break
            if set(solution) != set(gold_solution[t]):
                grades.addMessage('FAIL: {}'.format(self.path))
                grades.addMessage('\tStudent solution differed from autograder solution at timestep t = {}'.format(t))
                grades.addMessage('\tStudent solution at time t = {}: {}'.format(t, solution))
                grades.addMessage('\tCorrect solution at time t = {}: {}'.format(t, gold_solution[t]))
                return False
        
        if num_timesteps != len(gold_solution):
            grades.addMessage('FAIL: {}'.format(self.path))
            grades.addMessage('\tStudent solution differed from autograder solution')
            grades.addMessage('\tStudent solution timestep number: {}'.format(num_timesteps))
            grades.addMessage('\tCorrect solution timestep number: {}'.format(len(eval(solutionDict['possible_locations_per_timestep']))))
            return False

        grades.addMessage('PASS: %s' % self.path)
        return True

    def writeSolution(self, moduleDict, filePath):
        logicPlan = moduleDict['logicPlan']
        # open file and write comments
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)

        print("Solving problem", self.layoutName)
        print(self.layoutText)

        solution = self.solution(logicPlan)

        print("Problem solved")

        handle.write('possible_locations_per_timestep: "{}"\n'.format(str(solution)))
        handle.close()

    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT

class MappingProblemTest(testClasses.TestCase):

    def __init__(self, question, testDict):
        super(MappingProblemTest, self).__init__(question, testDict)
        self.layoutText = testDict['layout']
        self.layoutName = testDict['layoutName']
        self.scriptedActions = eval(testDict['actions'])
        self.solution_label = 'known_map_per_timestep'

    def solution(self, logicPlan):
        lay = layout.Layout([l.strip() for l in self.layoutText.split('\n')])
        ghosts = []
        # TODO: Figure out if we can use no-graphics cleaner
        disp = self.question.display
        if isinstance(disp, graphicsDisplay.PacmanGraphics): # autograder.py has incorrect options
            disp = graphicsDisplay.PacmanGraphics(frameTime=0.5, render_walls_beforehand=False)
        pac = logicAgents.MappingLogicAgent(
            'mp', 'MappingProblem', logicPlan, display=disp, scripted_actions=self.scriptedActions)
        yield from pacman.runGames(lay, pac, ghosts, disp, 1, False, catchExceptions=True, timeout=300)

    def check_len(self, grades, soln, gold_soln, str_info=""):
        if len(soln) != len(gold_soln):
            grades.addMessage('FAIL: {}'.format(self.path))
            grades.addMessage('\tstudent solution length {}: {}'.format(str_info, len(soln)))
            grades.addMessage('\tcorrect solution length {}: {}'.format(str_info, len(gold_soln)))
            return False
        return True

    def execute(self, grades, moduleDict, solutionDict):
        logicPlan = moduleDict['logicPlan']
        gold_solution = eval(solutionDict[self.solution_label])

        num_timesteps = 0

        for t, solution_t in enumerate(self.solution(logicPlan)):
            if solution_t == None:
                num_timesteps = t
                break
            if not self.check_len(grades, solution_t, gold_solution[t], "at time t = {}".format(t)):
                return False

            if solution_t != gold_solution[t]:
                grades.addMessage('FAIL: {}'.format(self.path))
                grades.addMessage('\tStudent solution differed from autograder solution at timestep t = {}'.format(t))
                grades.addMessage('\tStudent solution at time t = {}: {}'.format(t, solution_t))
                grades.addMessage('\tCorrect solution at time t = {}: {}'.format(t, gold_solution[t]))
                return False
        
        if num_timesteps != len(gold_solution):
            grades.addMessage('FAIL: {}'.format(self.path))
            grades.addMessage('\tStudent solution differed from autograder solution')
            grades.addMessage('\tStudent solution timestep number: {}'.format(num_timesteps))
            grades.addMessage('\tCorrect solution timestep number: {}'.format(len(eval(solutionDict[self.solution_label]))))
            return False

        grades.addMessage('PASS: %s' % self.path)
        return True

    def writeSolution(self, moduleDict, filePath):
        logicPlan = moduleDict['logicPlan']
        # open file and write comments
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)

        print("Solving problem", self.layoutName)
        print(self.layoutText)

        solution = self.solution(logicPlan)

        print("Problem solved")

        handle.write('{}: "{}"\n'.format(self.solution_label, str(solution)))
        handle.close()

    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT

class SLAMProblemTest(testClasses.TestCase):

    def __init__(self, question, testDict):
        super(SLAMProblemTest, self).__init__(question, testDict)
        self.layoutText = testDict['layout']
        self.layoutName = testDict['layoutName']
        self.scriptedActions = eval(testDict['actions'])
        self.solution_labels = ['known_map_per_timestep', 'possible_locations_per_timestep']

    def solution(self, logicPlan):
        lay = layout.Layout([l.strip() for l in self.layoutText.split('\n')])
        ghosts = []
        # TODO: Figure out if we can use no-graphics cleaner
        disp = self.question.display
        if isinstance(disp, graphicsDisplay.PacmanGraphics): # autograder.py has incorrect options
            disp = graphicsDisplay.PacmanGraphics(frameTime=0.5, render_walls_beforehand=False)
        pac = logicAgents.SLAMLogicAgent(
            'slam', 'SLAMProblem', logicPlan, display=disp, scripted_actions=self.scriptedActions)
        yield from pacman.runGames(lay, pac, ghosts, disp, 1, False, catchExceptions=True, timeout=1800)

    def check_len(self, grades, soln, gold_soln, str_info=""):
        if len(soln) != len(gold_soln):
            grades.addMessage('FAIL: {}'.format(self.path))
            grades.addMessage('\tstudent solution length {}: {}'.format(str_info, len(soln)))
            grades.addMessage('\tcorrect solution length {}: {}'.format(str_info, len(gold_soln)))
            return False
        return True

    def execute(self, grades, moduleDict, solutionDict):
        logicPlan = moduleDict['logicPlan']
        num_timesteps = 0
        for t, solutions_at_t in enumerate(self.solution(logicPlan)):
            if solutions_at_t is None:
                num_timesteps = t
                break
            for soln_label, solution in zip(self.solution_labels, solutions_at_t):
                gold_solution = eval(solutionDict[soln_label])

                if solution != gold_solution[t]:
                    grades.addMessage('FAIL: {}'.format(self.path))
                    grades.addMessage('\tStudent solution differed from autograder solution at timestep t = {}'.format(t))
                    grades.addMessage('\tStudent solution for {} at time t = {}: {}'.format(soln_label, t, solution))
                    grades.addMessage('\tCorrect solution for {} at time t = {}: {}'.format(soln_label, t, gold_solution[t]))
                    return False
        
        if num_timesteps != len(eval(solutionDict[self.solution_labels[0]])):
            grades.addMessage('FAIL: {}'.format(self.path))
            grades.addMessage('\tStudent solution differed from autograder solution')
            grades.addMessage('\tStudent solution timestep number: {}'.format(num_timesteps))
            grades.addMessage('\tCorrect solution timestep number: {}'.format(len(eval(solutionDict[self.solution_labels[0]]))))
            return False

        grades.addMessage('PASS: %s' % self.path)
        return True

    def writeSolution(self, moduleDict, filePath):
        logicPlan = moduleDict['logicPlan']
        # open file and write comments
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)

        print("Solving problem", self.layoutName)
        print(self.layoutText)

        solution = self.solution(logicPlan)

        print("Problem solved")

        for soln_label, solution_i in zip(self.solution_labels, solution):
            handle.write('{}: "{}"\n'.format(soln_label, str(solution_i)))
        handle.close()

    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT

class LogicStatementTest(testClasses.TestCase):
 
    def __init__(self, question, testDict):
        super(LogicStatementTest, self).__init__(question, testDict)
        self.preamble = compile(testDict.get('preamble', ""), "%s.preamble" % self.getPath(), 'exec')
        self.test = compile(testDict['test'], "%s.test" % self.getPath(), 'eval')
        self.pairs = testDict['pairs']
        self.success = testDict['success']
        self.failure = testDict['failure']
 
    def evalCode(self, moduleDict):
        bindings = dict(moduleDict)
        exec(self.preamble, bindings)
        return eval(self.test, bindings)
 
    def execute(self, grades, moduleDict, solutionDict):
        bindings = dict(moduleDict)
        exec(self.preamble, bindings)
        truths = eval(self.test, bindings)
        model_truth_pairs = eval(self.pairs, bindings)
        if str(truths) == solutionDict['result']:
            grades.addMessage('PASS: %s' % self.path)
            grades.addMessage('\t%s' % self.success)
            return True
        else:
            solution_truths = eval(solutionDict['result'])
            firstError = 1
            while truths[firstError-1] == solution_truths[firstError-1]:
                firstError += 1
            model = model_truth_pairs[firstError-1][0]
 
            grades.addMessage('FAIL: %s' % self.path)
            # grades.addMessage('\t%s' % self.failure)            
            grades.addMessage('Your solution\'s first error occurred on model %d.' % firstError)
            grades.addMessage('MODEL:  %s' % model)
            grades.addMessage('The correct answer is %s but you returned %s.' % (solution_truths[firstError-1], truths[firstError-1]))
 
 
        return False
 
    def writeSolution(self, moduleDict, filePath):
        handle = open(filePath, 'w')
        handle.write('# This is the solution file for %s.\n' % self.path)
        handle.write('# The result of evaluating the test must equal the below when cast to a string.\n')
 
        handle.write('result: "%s"\n' % self.evalCode(moduleDict))
        handle.close()
        return True
 
    # BEGIN SOLUTION NO PROMPT
    def createPublicVersion(self):
        pass
    # END SOLUTION NO PROMPT
