enter reinforcement

This commit is contained in:
2024-07-06 01:30:00 +08:00
parent f105ba0150
commit e3f8181056
116 changed files with 19698 additions and 0 deletions

View File

@ -0,0 +1,429 @@
weights_k_0: """
{((0, 0), 'exit'): 0,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_0_action_north: """
illegal
0.0000
illegal
"""
q_values_k_0_action_east: """
illegal
0.0000
illegal
"""
q_values_k_0_action_exit: """
0.0000
illegal
0.0000
"""
q_values_k_0_action_south: """
illegal
0.0000
illegal
"""
q_values_k_0_action_west: """
illegal
0.0000
illegal
"""
weights_k_1: """
{((0, 0), 'exit'): 1.0,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_1_action_north: """
illegal
0.0000
illegal
"""
q_values_k_1_action_east: """
illegal
0.0000
illegal
"""
q_values_k_1_action_exit: """
0.0000
illegal
1.0000
"""
q_values_k_1_action_south: """
illegal
0.0000
illegal
"""
q_values_k_1_action_west: """
illegal
0.0000
illegal
"""
weights_k_2: """
{((0, 0), 'exit'): 1.0,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_2_action_north: """
illegal
0.0000
illegal
"""
q_values_k_2_action_east: """
illegal
0.0000
illegal
"""
q_values_k_2_action_exit: """
0.0000
illegal
1.0000
"""
q_values_k_2_action_south: """
illegal
0.0000
illegal
"""
q_values_k_2_action_west: """
illegal
0.0000
illegal
"""
weights_k_3: """
{((0, 0), 'exit'): 1.9,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_3_action_north: """
illegal
0.0000
illegal
"""
q_values_k_3_action_east: """
illegal
0.0000
illegal
"""
q_values_k_3_action_exit: """
0.0000
illegal
1.9000
"""
q_values_k_3_action_south: """
illegal
0.0000
illegal
"""
q_values_k_3_action_west: """
illegal
0.0000
illegal
"""
weights_k_4: """
{((0, 0), 'exit'): 2.71,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_4_action_north: """
illegal
0.0000
illegal
"""
q_values_k_4_action_east: """
illegal
0.0000
illegal
"""
q_values_k_4_action_exit: """
0.0000
illegal
2.7100
"""
q_values_k_4_action_south: """
illegal
0.0000
illegal
"""
q_values_k_4_action_west: """
illegal
0.0000
illegal
"""
weights_k_5: """
{((0, 0), 'exit'): 2.71,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_5_action_north: """
illegal
0.0000
illegal
"""
q_values_k_5_action_east: """
illegal
0.0000
illegal
"""
q_values_k_5_action_exit: """
-1.0000
illegal
2.7100
"""
q_values_k_5_action_south: """
illegal
0.0000
illegal
"""
q_values_k_5_action_west: """
illegal
0.0000
illegal
"""
weights_k_6: """
{((0, 0), 'exit'): 3.439,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_6_action_north: """
illegal
0.0000
illegal
"""
q_values_k_6_action_east: """
illegal
0.0000
illegal
"""
q_values_k_6_action_exit: """
-1.0000
illegal
3.4390
"""
q_values_k_6_action_south: """
illegal
0.0000
illegal
"""
q_values_k_6_action_west: """
illegal
0.0000
illegal
"""
weights_k_7: """
{((0, 0), 'exit'): 3.439,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.17195000000000002,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_7_action_north: """
illegal
0.0000
illegal
"""
q_values_k_7_action_east: """
illegal
0.0000
illegal
"""
q_values_k_7_action_exit: """
-1.0000
illegal
3.4390
"""
q_values_k_7_action_south: """
illegal
0.1720
illegal
"""
q_values_k_7_action_west: """
illegal
0.0000
illegal
"""
weights_k_8: """
{((0, 0), 'exit'): 4.0951,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.17195000000000002,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_8_action_north: """
illegal
0.0000
illegal
"""
q_values_k_8_action_east: """
illegal
0.0000
illegal
"""
q_values_k_8_action_exit: """
-1.0000
illegal
4.0951
"""
q_values_k_8_action_south: """
illegal
0.1720
illegal
"""
q_values_k_8_action_west: """
illegal
0.0000
illegal
"""
weights_k_9: """
{((0, 0), 'exit'): 4.68559,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.17195000000000002,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_9_action_north: """
illegal
0.0000
illegal
"""
q_values_k_9_action_east: """
illegal
0.0000
illegal
"""
q_values_k_9_action_exit: """
-1.0000
illegal
4.6856
"""
q_values_k_9_action_south: """
illegal
0.1720
illegal
"""
q_values_k_9_action_west: """
illegal
0.0000
illegal
"""
weights_k_100: """
{((0, 0), 'exit'): 9.817519963685992,
((0, 1), 'east'): 0.40629236674335106,
((0, 1), 'north'): -0.4534185789984799,
((0, 1), 'south'): 2.126721095524319,
((0, 1), 'west'): 0.39193283364906867,
((0, 2), 'exit'): -9.476652366972639}
"""
q_values_k_100_action_north: """
illegal
-0.4534
illegal
"""
q_values_k_100_action_east: """
illegal
0.4063
illegal
"""
q_values_k_100_action_exit: """
-9.4767
illegal
9.8175
"""
q_values_k_100_action_south: """
illegal
2.1267
illegal
"""
q_values_k_100_action_west: """
illegal
0.3919
illegal
"""

View File

@ -0,0 +1,22 @@
class: "ApproximateQLearningTest"
# GridWorld specification
# _ is empty space
# numbers are terminal states with that value
# # is a wall
# S is a start state
#
grid: """
-10
S
10
"""
discount: "0.5"
noise: "0.0"
livingReward: "0.0"
epsilon: "0.5"
learningRate: "0.1"
numExperiences: "100"
valueIterations: "100"
iterations: "10000"

View File

@ -0,0 +1,71 @@
==================== Iteration 0 ====================
Q-Values at iteration 0 for action 'south' are NOT correct. Student solution:
q_values_k_0_action_south: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_south: """
illegal
0.0000
illegal
"""
Q-Values at iteration 0 for action 'west' are NOT correct. Student solution:
q_values_k_0_action_west: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_west: """
illegal
0.0000
illegal
"""
Q-Values at iteration 0 for action 'exit' are NOT correct. Student solution:
q_values_k_0_action_exit: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_exit: """
0.0000
illegal
0.0000
"""
Q-Values at iteration 0 for action 'east' are NOT correct. Student solution:
q_values_k_0_action_east: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_east: """
illegal
0.0000
illegal
"""
Q-Values at iteration 0 for action 'north' are NOT correct. Student solution:
q_values_k_0_action_north: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_north: """
illegal
0.0000
illegal
"""

View File

@ -0,0 +1,429 @@
weights_k_0: """
{((0, 0), 'exit'): 0,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_0_action_north: """
illegal
0.0000
illegal
"""
q_values_k_0_action_east: """
illegal
0.0000
illegal
"""
q_values_k_0_action_exit: """
0.0000
illegal
0.0000
"""
q_values_k_0_action_south: """
illegal
0.0000
illegal
"""
q_values_k_0_action_west: """
illegal
0.0000
illegal
"""
weights_k_1: """
{((0, 0), 'exit'): 1.0,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_1_action_north: """
illegal
0.0000
illegal
"""
q_values_k_1_action_east: """
illegal
0.0000
illegal
"""
q_values_k_1_action_exit: """
0.0000
illegal
1.0000
"""
q_values_k_1_action_south: """
illegal
0.0000
illegal
"""
q_values_k_1_action_west: """
illegal
0.0000
illegal
"""
weights_k_2: """
{((0, 0), 'exit'): 1.0,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_2_action_north: """
illegal
0.0000
illegal
"""
q_values_k_2_action_east: """
illegal
0.0000
illegal
"""
q_values_k_2_action_exit: """
0.0000
illegal
1.0000
"""
q_values_k_2_action_south: """
illegal
0.0000
illegal
"""
q_values_k_2_action_west: """
illegal
0.0000
illegal
"""
weights_k_3: """
{((0, 0), 'exit'): 1.9,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_3_action_north: """
illegal
0.0000
illegal
"""
q_values_k_3_action_east: """
illegal
0.0000
illegal
"""
q_values_k_3_action_exit: """
0.0000
illegal
1.9000
"""
q_values_k_3_action_south: """
illegal
0.0000
illegal
"""
q_values_k_3_action_west: """
illegal
0.0000
illegal
"""
weights_k_4: """
{((0, 0), 'exit'): 2.71,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): 0}
"""
q_values_k_4_action_north: """
illegal
0.0000
illegal
"""
q_values_k_4_action_east: """
illegal
0.0000
illegal
"""
q_values_k_4_action_exit: """
0.0000
illegal
2.7100
"""
q_values_k_4_action_south: """
illegal
0.0000
illegal
"""
q_values_k_4_action_west: """
illegal
0.0000
illegal
"""
weights_k_5: """
{((0, 0), 'exit'): 2.71,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_5_action_north: """
illegal
0.0000
illegal
"""
q_values_k_5_action_east: """
illegal
0.0000
illegal
"""
q_values_k_5_action_exit: """
-1.0000
illegal
2.7100
"""
q_values_k_5_action_south: """
illegal
0.0000
illegal
"""
q_values_k_5_action_west: """
illegal
0.0000
illegal
"""
weights_k_6: """
{((0, 0), 'exit'): 3.439,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.0,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_6_action_north: """
illegal
0.0000
illegal
"""
q_values_k_6_action_east: """
illegal
0.0000
illegal
"""
q_values_k_6_action_exit: """
-1.0000
illegal
3.4390
"""
q_values_k_6_action_south: """
illegal
0.0000
illegal
"""
q_values_k_6_action_west: """
illegal
0.0000
illegal
"""
weights_k_7: """
{((0, 0), 'exit'): 3.439,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.257925,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_7_action_north: """
illegal
0.0000
illegal
"""
q_values_k_7_action_east: """
illegal
0.0000
illegal
"""
q_values_k_7_action_exit: """
-1.0000
illegal
3.4390
"""
q_values_k_7_action_south: """
illegal
0.2579
illegal
"""
q_values_k_7_action_west: """
illegal
0.0000
illegal
"""
weights_k_8: """
{((0, 0), 'exit'): 4.0951,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.257925,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_8_action_north: """
illegal
0.0000
illegal
"""
q_values_k_8_action_east: """
illegal
0.0000
illegal
"""
q_values_k_8_action_exit: """
-1.0000
illegal
4.0951
"""
q_values_k_8_action_south: """
illegal
0.2579
illegal
"""
q_values_k_8_action_west: """
illegal
0.0000
illegal
"""
weights_k_9: """
{((0, 0), 'exit'): 4.68559,
((0, 1), 'east'): 0,
((0, 1), 'north'): 0,
((0, 1), 'south'): 0.257925,
((0, 1), 'west'): 0,
((0, 2), 'exit'): -1.0}
"""
q_values_k_9_action_north: """
illegal
0.0000
illegal
"""
q_values_k_9_action_east: """
illegal
0.0000
illegal
"""
q_values_k_9_action_exit: """
-1.0000
illegal
4.6856
"""
q_values_k_9_action_south: """
illegal
0.2579
illegal
"""
q_values_k_9_action_west: """
illegal
0.0000
illegal
"""
weights_k_100: """
{((0, 0), 'exit'): 9.817519963685992,
((0, 1), 'east'): 0.9498968104823575,
((0, 1), 'north'): -0.66699795412272,
((0, 1), 'south'): 3.256207905310105,
((0, 1), 'west'): 0.8236280735014627,
((0, 2), 'exit'): -9.476652366972639}
"""
q_values_k_100_action_north: """
illegal
-0.6670
illegal
"""
q_values_k_100_action_east: """
illegal
0.9499
illegal
"""
q_values_k_100_action_exit: """
-9.4767
illegal
9.8175
"""
q_values_k_100_action_south: """
illegal
3.2562
illegal
"""
q_values_k_100_action_west: """
illegal
0.8236
illegal
"""

View File

@ -0,0 +1,22 @@
class: "ApproximateQLearningTest"
# GridWorld specification
# _ is empty space
# numbers are terminal states with that value
# # is a wall
# S is a start state
#
grid: """
-10
S
10
"""
discount: "0.75"
noise: "0.25"
livingReward: "0.0"
epsilon: "0.5"
learningRate: "0.1"
numExperiences: "100"
valueIterations: "100"
iterations: "10000"

View File

@ -0,0 +1,71 @@
==================== Iteration 0 ====================
Q-Values at iteration 0 for action 'south' are NOT correct. Student solution:
q_values_k_0_action_south: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_south: """
illegal
0.0000
illegal
"""
Q-Values at iteration 0 for action 'west' are NOT correct. Student solution:
q_values_k_0_action_west: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_west: """
illegal
0.0000
illegal
"""
Q-Values at iteration 0 for action 'exit' are NOT correct. Student solution:
q_values_k_0_action_exit: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_exit: """
0.0000
illegal
0.0000
"""
Q-Values at iteration 0 for action 'east' are NOT correct. Student solution:
q_values_k_0_action_east: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_east: """
illegal
0.0000
illegal
"""
Q-Values at iteration 0 for action 'north' are NOT correct. Student solution:
q_values_k_0_action_north: """
illegal
illegal
illegal
"""
Correct solution:
q_values_k_0_action_north: """
illegal
0.0000
illegal
"""

View File

@ -0,0 +1,935 @@
weights_k_0: """
{((0, 1), 'exit'): 0,
((0, 2), 'exit'): 0,
((0, 3), 'exit'): 0,
((0, 4), 'exit'): 0,
((0, 5), 'exit'): 0,
((1, 0), 'exit'): 0,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): 0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_0_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_0_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_0_action_exit: """
__________ 0.0000 __________
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
__________ 0.0000 __________
"""
q_values_k_0_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_0_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_1: """
{((0, 1), 'exit'): 0,
((0, 2), 'exit'): 0,
((0, 3), 'exit'): 0,
((0, 4), 'exit'): 0,
((0, 5), 'exit'): -10.0,
((1, 0), 'exit'): 0,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): 0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_1_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_1_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_1_action_exit: """
__________ 0.0000 __________
-10.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
__________ 0.0000 __________
"""
q_values_k_1_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_1_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_2: """
{((0, 1), 'exit'): 0,
((0, 2), 'exit'): 0,
((0, 3), 'exit'): 0,
((0, 4), 'exit'): 0,
((0, 5), 'exit'): -10.0,
((1, 0), 'exit'): 0,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0.0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): 0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_2_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_2_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_2_action_exit: """
__________ 0.0000 __________
-10.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
__________ 0.0000 __________
"""
q_values_k_2_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_2_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_3: """
{((0, 1), 'exit'): -10.0,
((0, 2), 'exit'): 0,
((0, 3), 'exit'): 0,
((0, 4), 'exit'): 0,
((0, 5), 'exit'): -10.0,
((1, 0), 'exit'): 0,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0.0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): 0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_3_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_3_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_3_action_exit: """
__________ 0.0000 __________
-10.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
-10.0000 illegal 0.0000
__________ 0.0000 __________
"""
q_values_k_3_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_3_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_4: """
{((0, 1), 'exit'): -10.0,
((0, 2), 'exit'): 0,
((0, 3), 'exit'): 0,
((0, 4), 'exit'): -10.0,
((0, 5), 'exit'): -10.0,
((1, 0), 'exit'): 0,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0.0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): 0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_4_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_4_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_4_action_exit: """
__________ 0.0000 __________
-10.0000 illegal 0.0000
-10.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
-10.0000 illegal 0.0000
__________ 0.0000 __________
"""
q_values_k_4_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_4_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_5: """
{((0, 1), 'exit'): -10.0,
((0, 2), 'exit'): 0,
((0, 3), 'exit'): 0,
((0, 4), 'exit'): -10.0,
((0, 5), 'exit'): -10.0,
((1, 0), 'exit'): 0,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0.0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): -10.0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_5_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_5_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_5_action_exit: """
__________ 0.0000 __________
-10.0000 illegal 0.0000
-10.0000 illegal 0.0000
0.0000 illegal -10.0000
0.0000 illegal 0.0000
-10.0000 illegal 0.0000
__________ 0.0000 __________
"""
q_values_k_5_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_5_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_6: """
{((0, 1), 'exit'): -10.0,
((0, 2), 'exit'): 0,
((0, 3), 'exit'): -10.0,
((0, 4), 'exit'): -10.0,
((0, 5), 'exit'): -10.0,
((1, 0), 'exit'): 0,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0.0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): -10.0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_6_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_6_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_6_action_exit: """
__________ 0.0000 __________
-10.0000 illegal 0.0000
-10.0000 illegal 0.0000
-10.0000 illegal -10.0000
0.0000 illegal 0.0000
-10.0000 illegal 0.0000
__________ 0.0000 __________
"""
q_values_k_6_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_6_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_7: """
{((0, 1), 'exit'): -10.0,
((0, 2), 'exit'): 0,
((0, 3), 'exit'): -10.0,
((0, 4), 'exit'): -10.0,
((0, 5), 'exit'): -10.0,
((1, 0), 'exit'): 0,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0.0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0.0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): -10.0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_7_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_7_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_7_action_exit: """
__________ 0.0000 __________
-10.0000 illegal 0.0000
-10.0000 illegal 0.0000
-10.0000 illegal -10.0000
0.0000 illegal 0.0000
-10.0000 illegal 0.0000
__________ 0.0000 __________
"""
q_values_k_7_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_7_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_8: """
{((0, 1), 'exit'): -10.0,
((0, 2), 'exit'): -10.0,
((0, 3), 'exit'): -10.0,
((0, 4), 'exit'): -10.0,
((0, 5), 'exit'): -10.0,
((1, 0), 'exit'): 0,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0.0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0.0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): -10.0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_8_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_8_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_8_action_exit: """
__________ 0.0000 __________
-10.0000 illegal 0.0000
-10.0000 illegal 0.0000
-10.0000 illegal -10.0000
-10.0000 illegal 0.0000
-10.0000 illegal 0.0000
__________ 0.0000 __________
"""
q_values_k_8_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_8_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_9: """
{((0, 1), 'exit'): -10.0,
((0, 2), 'exit'): -10.0,
((0, 3), 'exit'): -10.0,
((0, 4), 'exit'): -10.0,
((0, 5), 'exit'): -10.0,
((1, 0), 'exit'): 0.1,
((1, 1), 'east'): 0,
((1, 1), 'north'): 0,
((1, 1), 'south'): 0,
((1, 1), 'west'): 0,
((1, 2), 'east'): 0,
((1, 2), 'north'): 0,
((1, 2), 'south'): 0,
((1, 2), 'west'): 0,
((1, 3), 'east'): 0,
((1, 3), 'north'): 0,
((1, 3), 'south'): 0.0,
((1, 3), 'west'): 0,
((1, 4), 'east'): 0,
((1, 4), 'north'): 0,
((1, 4), 'south'): 0,
((1, 4), 'west'): 0,
((1, 5), 'east'): 0,
((1, 5), 'north'): 0,
((1, 5), 'south'): 0.0,
((1, 5), 'west'): 0,
((1, 6), 'exit'): 0,
((2, 1), 'exit'): 0,
((2, 2), 'exit'): 0,
((2, 3), 'exit'): -10.0,
((2, 4), 'exit'): 0,
((2, 5), 'exit'): 0}
"""
q_values_k_9_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_9_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_9_action_exit: """
__________ 0.0000 __________
-10.0000 illegal 0.0000
-10.0000 illegal 0.0000
-10.0000 illegal -10.0000
-10.0000 illegal 0.0000
-10.0000 illegal 0.0000
__________ 0.1000 __________
"""
q_values_k_9_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
q_values_k_9_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
weights_k_500: """
{((0, 1), 'exit'): -92.82102012308148,
((0, 2), 'exit'): -97.21871610556306,
((0, 3), 'exit'): -89.05810108684878,
((0, 4), 'exit'): -97.74716004550608,
((0, 5), 'exit'): -96.56631617970748,
((1, 0), 'exit'): 0.9576088417247839,
((1, 1), 'east'): -31.68839649871871,
((1, 1), 'north'): -0.5871409700255297,
((1, 1), 'south'): -5.571799344704395,
((1, 1), 'west'): -20.777007017445538,
((1, 2), 'east'): -43.595242197319,
((1, 2), 'north'): -1.264202431807023,
((1, 2), 'south'): -8.401530599975509,
((1, 2), 'west'): -16.283916171605192,
((1, 3), 'east'): -3.6956691,
((1, 3), 'north'): -0.16712710492783758,
((1, 3), 'south'): -3.4722840178579073,
((1, 3), 'west'): -40.58867937480968,
((1, 4), 'east'): -26.553386621338632,
((1, 4), 'north'): -0.799493322153628,
((1, 4), 'south'): -6.727671187497919,
((1, 4), 'west'): -39.06095135014759,
((1, 5), 'east'): -17.067638934181446,
((1, 5), 'north'): -5.864753060887024,
((1, 5), 'south'): -6.83769420759525,
((1, 5), 'west'): -27.062643066307515,
((1, 6), 'exit'): 9.353891811077332,
((2, 1), 'exit'): -97.21871610556306,
((2, 2), 'exit'): -87.84233454094309,
((2, 3), 'exit'): -96.90968456173674,
((2, 4), 'exit'): -94.185026299696,
((2, 5), 'exit'): -96.90968456173674}
"""
q_values_k_500_action_north: """
__________ illegal __________
illegal -5.8648 illegal
illegal -0.7995 illegal
illegal -0.1671 illegal
illegal -1.2642 illegal
illegal -0.5871 illegal
__________ illegal __________
"""
q_values_k_500_action_east: """
__________ illegal __________
illegal -17.0676 illegal
illegal -26.5534 illegal
illegal -3.6957 illegal
illegal -43.5952 illegal
illegal -31.6884 illegal
__________ illegal __________
"""
q_values_k_500_action_exit: """
__________ 9.3539 __________
-96.5663 illegal -96.9097
-97.7472 illegal -94.1850
-89.0581 illegal -96.9097
-97.2187 illegal -87.8423
-92.8210 illegal -97.2187
__________ 0.9576 __________
"""
q_values_k_500_action_south: """
__________ illegal __________
illegal -6.8377 illegal
illegal -6.7277 illegal
illegal -3.4723 illegal
illegal -8.4015 illegal
illegal -5.5718 illegal
__________ illegal __________
"""
q_values_k_500_action_west: """
__________ illegal __________
illegal -27.0626 illegal
illegal -39.0610 illegal
illegal -40.5887 illegal
illegal -16.2839 illegal
illegal -20.7770 illegal
__________ illegal __________
"""

View File

@ -0,0 +1,27 @@
class: "ApproximateQLearningTest"
# GridWorld specification
# _ is empty space
# numbers are terminal states with that value
# # is a wall
# S is a start state
#
grid: """
# 10 #
-100 _ -100
-100 _ -100
-100 _ -100
-100 _ -100
-100 S -100
# 1 #
"""
gridName: "bridgeGrid"
discount: "0.85"
noise: "0.1"
livingReward: "0.0"
epsilon: "0.5"
learningRate: "0.1"
numExperiences: "500"
valueIterations: "100"
iterations: "10000"

View File

@ -0,0 +1,111 @@
==================== Iteration 0 ====================
Q-Values at iteration 0 for action 'south' are NOT correct. Student solution:
q_values_k_0_action_south: """
__________ illegal __________
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
__________ illegal __________
"""
Correct solution:
q_values_k_0_action_south: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
Q-Values at iteration 0 for action 'west' are NOT correct. Student solution:
q_values_k_0_action_west: """
__________ illegal __________
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
__________ illegal __________
"""
Correct solution:
q_values_k_0_action_west: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
Q-Values at iteration 0 for action 'exit' are NOT correct. Student solution:
q_values_k_0_action_exit: """
__________ illegal __________
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
__________ illegal __________
"""
Correct solution:
q_values_k_0_action_exit: """
__________ 0.0000 __________
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
0.0000 illegal 0.0000
__________ 0.0000 __________
"""
Q-Values at iteration 0 for action 'east' are NOT correct. Student solution:
q_values_k_0_action_east: """
__________ illegal __________
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
__________ illegal __________
"""
Correct solution:
q_values_k_0_action_east: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""
Q-Values at iteration 0 for action 'north' are NOT correct. Student solution:
q_values_k_0_action_north: """
__________ illegal __________
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
illegal illegal illegal
__________ illegal __________
"""
Correct solution:
q_values_k_0_action_north: """
__________ illegal __________
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
illegal 0.0000 illegal
__________ illegal __________
"""

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,24 @@
class: "ApproximateQLearningTest"
# GridWorld specification
# _ is empty space
# numbers are terminal states with that value
# # is a wall
# S is a start state
#
grid: """
-10 _ 10 _ _
-10 _ # _ _
-10 _ 1 _ _
-10 _ # # _
-10 S _ _ _
"""
discount: "0.9"
noise: "0.2"
livingReward: "0.0"
epsilon: "0.2"
learningRate: "0.1"
numExperiences: "3000"
valueIterations: "100"
iterations: "10000"

View File

@ -0,0 +1,91 @@
==================== Iteration 0 ====================
Q-Values at iteration 0 for action 'south' are NOT correct. Student solution:
q_values_k_0_action_south: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
Q-Values at iteration 0 for action 'west' are NOT correct. Student solution:
q_values_k_0_action_west: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
Q-Values at iteration 0 for action 'exit' are NOT correct. Student solution:
q_values_k_0_action_exit: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_exit: """
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ illegal illegal
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ __________ illegal
0.0000 illegal illegal illegal illegal
"""
Q-Values at iteration 0 for action 'east' are NOT correct. Student solution:
q_values_k_0_action_east: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
Q-Values at iteration 0 for action 'north' are NOT correct. Student solution:
q_values_k_0_action_north: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""

View File

@ -0,0 +1,880 @@
weights_k_0: """
{'action=east': 0,
'action=exit': 0,
'action=north': 0,
'action=south': 0,
'action=west': 0,
'x=0': 0,
'x=1': 0,
'x=2': 0,
'x=3': 0,
'x=4': 0,
'y=0': 0,
'y=1': 0,
'y=2': 0,
'y=3': 0,
'y=4': 0,
(0, 0): 0,
(0, 1): 0,
(0, 2): 0,
(0, 3): 0,
(0, 4): 0,
(1, 0): 0,
(1, 1): 0,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0,
(2, 4): 0,
(3, 0): 0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_0_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_0_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_0_action_exit: """
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ illegal illegal
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ __________ illegal
0.0000 illegal illegal illegal illegal
"""
q_values_k_0_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_0_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
weights_k_1: """
{'action=east': 0,
'action=exit': 0,
'action=north': 0,
'action=south': 0.0,
'action=west': 0,
'x=0': 0,
'x=1': 0.0,
'x=2': 0,
'x=3': 0,
'x=4': 0,
'y=0': 0,
'y=1': 0.0,
'y=2': 0,
'y=3': 0,
'y=4': 0,
(0, 0): 0,
(0, 1): 0,
(0, 2): 0,
(0, 3): 0,
(0, 4): 0,
(1, 0): 0.0,
(1, 1): 0,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0,
(2, 4): 0,
(3, 0): 0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_1_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_1_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_1_action_exit: """
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ illegal illegal
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ __________ illegal
0.0000 illegal illegal illegal illegal
"""
q_values_k_1_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_1_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
weights_k_2: """
{'action=east': 0,
'action=exit': 0,
'action=north': 0,
'action=south': 0.0,
'action=west': 0,
'x=0': 0,
'x=1': 0.0,
'x=2': 0,
'x=3': 0.0,
'x=4': 0,
'y=0': 0,
'y=1': 0.0,
'y=2': 0,
'y=3': 0.0,
'y=4': 0,
(0, 0): 0,
(0, 1): 0,
(0, 2): 0,
(0, 3): 0,
(0, 4): 0,
(1, 0): 0.0,
(1, 1): 0,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0,
(2, 4): 0,
(3, 0): 0.0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_2_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_2_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_2_action_exit: """
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ illegal illegal
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ __________ illegal
0.0000 illegal illegal illegal illegal
"""
q_values_k_2_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_2_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
weights_k_3: """
{'action=east': 0,
'action=exit': -1.0,
'action=north': 0,
'action=south': 0.0,
'action=west': 0,
'x=0': -1.0,
'x=1': 0.0,
'x=2': 0,
'x=3': 0.0,
'x=4': 0,
'y=0': -1.0,
'y=1': 0.0,
'y=2': 0,
'y=3': 0.0,
'y=4': 0,
(0, 0): -1.0,
(0, 1): 0,
(0, 2): 0,
(0, 3): 0,
(0, 4): 0,
(1, 0): 0.0,
(1, 1): 0,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0,
(2, 4): 0,
(3, 0): 0.0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_3_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_3_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_3_action_exit: """
-3.0000 illegal -1.0000 illegal illegal
-3.0000 illegal __________ illegal illegal
-3.0000 illegal -1.0000 illegal illegal
-3.0000 illegal __________ __________ illegal
-4.0000 illegal illegal illegal illegal
"""
q_values_k_3_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_3_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
weights_k_4: """
{'action=east': 0.0,
'action=exit': -1.0,
'action=north': 0,
'action=south': 0.0,
'action=west': 0,
'x=0': -1.0,
'x=1': 0.0,
'x=2': 0,
'x=3': 0.0,
'x=4': 0,
'y=0': -1.0,
'y=1': 0.0,
'y=2': 0,
'y=3': 0.0,
'y=4': 0,
(0, 0): -1.0,
(0, 1): 0,
(0, 2): 0,
(0, 3): 0,
(0, 4): 0,
(1, 0): 0.0,
(1, 1): 0,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0,
(2, 4): 0,
(3, 0): 0.0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_4_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_4_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_4_action_exit: """
-3.0000 illegal -1.0000 illegal illegal
-3.0000 illegal __________ illegal illegal
-3.0000 illegal -1.0000 illegal illegal
-3.0000 illegal __________ __________ illegal
-4.0000 illegal illegal illegal illegal
"""
q_values_k_4_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_4_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
weights_k_5: """
{'action=east': 0.0,
'action=exit': -1.0,
'action=north': 0,
'action=south': 0.0,
'action=west': 0.0,
'x=0': -1.0,
'x=1': 0.0,
'x=2': 0,
'x=3': 0.0,
'x=4': 0.0,
'y=0': -1.0,
'y=1': 0.0,
'y=2': 0,
'y=3': 0.0,
'y=4': 0.0,
(0, 0): -1.0,
(0, 1): 0,
(0, 2): 0,
(0, 3): 0,
(0, 4): 0,
(1, 0): 0.0,
(1, 1): 0,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0,
(2, 4): 0,
(3, 0): 0.0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0.0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_5_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_5_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_5_action_exit: """
-3.0000 illegal -1.0000 illegal illegal
-3.0000 illegal __________ illegal illegal
-3.0000 illegal -1.0000 illegal illegal
-3.0000 illegal __________ __________ illegal
-4.0000 illegal illegal illegal illegal
"""
q_values_k_5_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_5_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
weights_k_6: """
{'action=east': 0.0,
'action=exit': -1.7000000000000002,
'action=north': 0,
'action=south': 0.0,
'action=west': 0.0,
'x=0': -1.7000000000000002,
'x=1': 0.0,
'x=2': 0,
'x=3': 0.0,
'x=4': 0.0,
'y=0': -1.7000000000000002,
'y=1': 0.0,
'y=2': 0,
'y=3': 0.0,
'y=4': 0.0,
(0, 0): -1.0,
(0, 1): 0,
(0, 2): 0,
(0, 3): -0.7000000000000001,
(0, 4): 0,
(1, 0): 0.0,
(1, 1): 0,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0,
(2, 4): 0,
(3, 0): 0.0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0.0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_6_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_6_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_6_action_exit: """
-5.1000 illegal -1.7000 illegal illegal
-5.8000 illegal __________ illegal illegal
-5.1000 illegal -1.7000 illegal illegal
-5.1000 illegal __________ __________ illegal
-6.1000 illegal illegal illegal illegal
"""
q_values_k_6_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
q_values_k_6_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
weights_k_7: """
{'action=east': 0.0,
'action=exit': -1.4300000000000002,
'action=north': 0,
'action=south': 0.0,
'action=west': 0.0,
'x=0': -1.7000000000000002,
'x=1': 0.0,
'x=2': 0.27,
'x=3': 0.0,
'x=4': 0.0,
'y=0': -1.7000000000000002,
'y=1': 0.0,
'y=2': 0.27,
'y=3': 0.0,
'y=4': 0.0,
(0, 0): -1.0,
(0, 1): 0,
(0, 2): 0,
(0, 3): -0.7000000000000001,
(0, 4): 0,
(1, 0): 0.0,
(1, 1): 0,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0.27,
(2, 4): 0,
(3, 0): 0.0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0.0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_7_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.5400 0.0000 0.0000
"""
q_values_k_7_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.5400 0.0000 0.0000
"""
q_values_k_7_action_exit: """
-4.8300 illegal -0.8900 illegal illegal
-5.5300 illegal __________ illegal illegal
-4.8300 illegal -0.6200 illegal illegal
-4.8300 illegal __________ __________ illegal
-5.8300 illegal illegal illegal illegal
"""
q_values_k_7_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.5400 0.0000 0.0000
"""
q_values_k_7_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.5400 0.0000 0.0000
"""
weights_k_8: """
{'action=east': 0.0,
'action=exit': -1.947,
'action=north': 0,
'action=south': 0.0,
'action=west': 0.0,
'x=0': -2.217,
'x=1': 0.0,
'x=2': 0.27,
'x=3': 0.0,
'x=4': 0.0,
'y=0': -2.217,
'y=1': 0.0,
'y=2': 0.27,
'y=3': 0.0,
'y=4': 0.0,
(0, 0): -1.0,
(0, 1): -0.517,
(0, 2): 0,
(0, 3): -0.7000000000000001,
(0, 4): 0,
(1, 0): 0.0,
(1, 1): 0,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0.27,
(2, 4): 0,
(3, 0): 0.0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0.0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_8_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.5400 0.0000 0.0000
"""
q_values_k_8_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.5400 0.0000 0.0000
"""
q_values_k_8_action_exit: """
-6.3810 illegal -1.4070 illegal illegal
-7.0810 illegal __________ illegal illegal
-6.3810 illegal -1.1370 illegal illegal
-6.8980 illegal __________ __________ illegal
-7.3810 illegal illegal illegal illegal
"""
q_values_k_8_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.5400 0.0000 0.0000
"""
q_values_k_8_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.5400 0.0000 0.0000
"""
weights_k_9: """
{'action=east': 0.0,
'action=exit': -1.947,
'action=north': -0.62082,
'action=south': 0.0,
'action=west': 0.0,
'x=0': -2.217,
'x=1': -0.62082,
'x=2': 0.27,
'x=3': 0.0,
'x=4': 0.0,
'y=0': -2.217,
'y=1': -0.62082,
'y=2': 0.27,
'y=3': 0.0,
'y=4': 0.0,
(0, 0): -1.0,
(0, 1): -0.517,
(0, 2): 0,
(0, 3): -0.7000000000000001,
(0, 4): 0,
(1, 0): 0.0,
(1, 1): -0.62082,
(1, 2): 0,
(1, 3): 0,
(1, 4): 0,
(2, 0): 0,
(2, 2): 0.27,
(2, 4): 0,
(3, 0): 0.0,
(3, 2): 0,
(3, 3): 0,
(3, 4): 0,
(4, 0): 0,
(4, 1): 0.0,
(4, 2): 0,
(4, 3): 0,
(4, 4): 0}
"""
q_values_k_9_action_north: """
illegal -1.8625 illegal -0.6208 -0.6208
illegal -1.8625 __________ -0.6208 -0.6208
illegal -1.8625 illegal -0.6208 -0.6208
illegal -2.4833 __________ __________ -0.6208
illegal -1.8625 -0.0808 -0.6208 -0.6208
"""
q_values_k_9_action_east: """
illegal -1.2416 illegal 0.0000 0.0000
illegal -1.2416 __________ 0.0000 0.0000
illegal -1.2416 illegal 0.0000 0.0000
illegal -1.8625 __________ __________ 0.0000
illegal -1.2416 0.5400 0.0000 0.0000
"""
q_values_k_9_action_exit: """
-6.3810 illegal -1.4070 illegal illegal
-7.0810 illegal __________ illegal illegal
-6.3810 illegal -1.1370 illegal illegal
-6.8980 illegal __________ __________ illegal
-7.3810 illegal illegal illegal illegal
"""
q_values_k_9_action_south: """
illegal -1.2416 illegal 0.0000 0.0000
illegal -1.2416 __________ 0.0000 0.0000
illegal -1.2416 illegal 0.0000 0.0000
illegal -1.8625 __________ __________ 0.0000
illegal -1.2416 0.5400 0.0000 0.0000
"""
q_values_k_9_action_west: """
illegal -1.2416 illegal 0.0000 0.0000
illegal -1.2416 __________ 0.0000 0.0000
illegal -1.2416 illegal 0.0000 0.0000
illegal -1.8625 __________ __________ 0.0000
illegal -1.2416 0.5400 0.0000 0.0000
"""
weights_k_3000: """
{'action=east': 6.719916513522846,
'action=exit': -2.2444981376861555,
'action=north': 4.568574519923728,
'action=south': 3.761510351874819,
'action=west': 1.2828606322891556,
'x=0': -3.604063955849794,
'x=1': 0.6731476152061693,
'x=2': 4.000208353074704,
'x=3': 5.988311380073477,
'x=4': 7.0307604874198235,
'y=0': -3.604063955849794,
'y=1': 0.6731476152061693,
'y=2': 4.000208353074704,
'y=3': 5.988311380073477,
'y=4': 7.0307604874198235,
(0, 0): -0.7073688447583666,
(0, 1): -0.7542862401704076,
(0, 2): -0.7043014501203066,
(0, 3): -0.7433344649617668,
(0, 4): -0.6947729558389527,
(1, 0): 2.364273811399719,
(1, 1): -0.2695405704605499,
(1, 2): -0.7105979212702271,
(1, 3): -1.4866826750327933,
(1, 4): 0.7756949705700219,
(2, 0): 2.64064253491107,
(2, 2): -3.7381118310263166,
(2, 4): 5.097677649189953,
(3, 0): 2.505262939441149,
(3, 2): 0.27218788923837256,
(3, 3): 2.2611084206093195,
(3, 4): 0.9497521307846304,
(4, 0): 1.7330586015291545,
(4, 1): 0.980194046153168,
(4, 2): 0.78786289128181,
(4, 3): 1.493343270762865,
(4, 4): 2.0363016776928333}
"""
q_values_k_3000_action_north: """
illegal 6.6906 illegal 17.4949 20.6664
illegal 4.4282 __________ 18.8063 20.1234
illegal 5.2043 illegal 16.8174 19.4180
illegal 5.6453 __________ __________ 19.6103
illegal 8.2791 15.2096 19.0505 20.3632
"""
q_values_k_3000_action_east: """
illegal 8.8419 illegal 19.6463 22.8177
illegal 6.5795 __________ 20.9576 22.2748
illegal 7.3556 illegal 18.9687 21.5693
illegal 7.7967 __________ __________ 21.7616
illegal 10.4305 17.3610 21.2018 22.5145
"""
q_values_k_3000_action_exit: """
-10.1474 illegal 10.8536 illegal illegal
-10.1960 illegal __________ illegal illegal
-10.1569 illegal 2.0178 illegal illegal
-10.2069 illegal __________ __________ illegal
-10.1600 illegal illegal illegal illegal
"""
q_values_k_3000_action_south: """
illegal 5.8835 illegal 16.6879 19.8593
illegal 3.6211 __________ 17.9992 19.3164
illegal 4.3972 illegal 16.0103 18.6109
illegal 4.8383 __________ __________ 18.8032
illegal 7.4721 14.4026 18.2434 19.5561
"""
q_values_k_3000_action_west: """
illegal 3.4049 illegal 14.2092 17.3807
illegal 1.1425 __________ 15.5206 16.8377
illegal 1.9186 illegal 13.5317 16.1322
illegal 2.3596 __________ __________ 16.3246
illegal 4.9934 11.9239 15.7647 17.0774
"""

View File

@ -0,0 +1,25 @@
class: "ApproximateQLearningTest"
# GridWorld specification
# _ is empty space
# numbers are terminal states with that value
# # is a wall
# S is a start state
#
grid: """
-10 _ 10 _ _
-10 _ # _ _
-10 _ 1 _ _
-10 _ # # _
-10 S _ _ _
"""
discount: "0.9"
noise: "0.2"
livingReward: "0.0"
epsilon: "0.2"
learningRate: "0.1"
numExperiences: "3000"
valueIterations: "100"
iterations: "10000"
extractor: "CoordinateExtractor"

View File

@ -0,0 +1,91 @@
==================== Iteration 0 ====================
Q-Values at iteration 0 for action 'south' are NOT correct. Student solution:
q_values_k_0_action_south: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_south: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
Q-Values at iteration 0 for action 'west' are NOT correct. Student solution:
q_values_k_0_action_west: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_west: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
Q-Values at iteration 0 for action 'exit' are NOT correct. Student solution:
q_values_k_0_action_exit: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_exit: """
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ illegal illegal
0.0000 illegal 0.0000 illegal illegal
0.0000 illegal __________ __________ illegal
0.0000 illegal illegal illegal illegal
"""
Q-Values at iteration 0 for action 'east' are NOT correct. Student solution:
q_values_k_0_action_east: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_east: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""
Q-Values at iteration 0 for action 'north' are NOT correct. Student solution:
q_values_k_0_action_north: """
illegal illegal illegal illegal illegal
illegal illegal __________ illegal illegal
illegal illegal illegal illegal illegal
illegal illegal __________ __________ illegal
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_0_action_north: """
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ 0.0000 0.0000
illegal 0.0000 illegal 0.0000 0.0000
illegal 0.0000 __________ __________ 0.0000
illegal 0.0000 0.0000 0.0000 0.0000
"""

View File

@ -0,0 +1,2 @@
max_points: "3"
class: "PassAllTestsQuestion"