gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2026-01-04 12:54:17 +01:00
Code Issues Projects Releases Wiki Activity

Itaicaspi/episode reset refactoring (#105)

Browse Source 
* reordering of the episode reset operation and allowing to store episodes only when they are terminated

* reordering of the episode reset operation and allowing to store episodes only when they are terminated

* revert tensorflow-gpu to 1.9.0 + bug fix in should_train()

* tests readme file and refactoring of policy optimization agent train function

* Update README.md

* Update README.md

* additional policy optimization train function simplifications

* Updated the traces after the reordering of the environment reset

* docker and jenkins files

* updated the traces to the ones from within the docker container

* updated traces and added control suite to the docker

* updated jenkins file with the intel proxy + updated doom basic a3c test params

* updated line breaks in jenkins file

* added a missing line break in jenkins file

* refining trace tests ignored presets + adding a configurable beta entropy value

* switch the order of trace and golden tests in jenkins + fix golden tests processes not killed issue

* updated benchmarks for dueling ddqn breakout and pong

* allowing dynamic updates to the loss weights + bug fix in episode.update_returns

* remove docker and jenkins file
This commit is contained in:
Itai Caspi
2018-09-04 15:07:54 +03:00
committed by GitHub
parent 7086492127
commit 72a1d9d426
92 changed files with 9803 additions and 9740 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
rl_coach/traces/Doom_Basic_DFP/trace.csv
View File

@@ -29,38 +29,38 @@ Episode #,Training Iter,In Heatup,ER #Transitions,ER #Episodes,Episode Length,To
28,0.0,1.0,1000.0,28.0,7.0,1000.0,0.5,,,0.0,,,,,,,,,,,,,,,
29,0.0,1.0,1012.0,29.0,12.0,1012.0,0.5,,,0.0,,,,,,,,,,,,,,,
30,0.0,1.0,1058.0,30.0,46.0,1058.0,0.5,,,0.0,,,,,,,,,,,,,,,
31,74.0,0.0,1133.0,31.0,75.0,1133.0,0.4962500000000004,-310.0,-310.0,0.0,,,,9862.112614812078,5108.627430680448,21896.46875,4428.638671875,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,19652.328,8994.96,52983.04,6429.6387
32,149.0,0.0,1208.0,32.0,75.0,1208.0,0.4925000000000008,-330.0,-330.0,0.0,,,,3830.6351790364574,643.7340248125081,5340.5830078125,2646.113525390625,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8379.321,5266.063,22653.16,1860.6244
33,194.0,0.0,1253.0,33.0,45.0,1253.0,0.4902500000000011,-122.0,-122.0,0.0,,,,3059.363802083333,588.2946622227233,4644.080078125,1812.8056640625,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8209.5625,4655.1914,19927.584,1891.3015
34,269.0,0.0,1328.0,34.0,75.0,1328.0,0.4865000000000015,-350.0,-350.0,0.0,,,,2846.889031575521,476.7265607773383,4139.458984375,1698.8955078125,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,6443.3677,3683.1653,17744.254,1654.5441
35,344.0,0.0,1403.0,35.0,75.0,1403.0,0.4827500000000019,-345.0,-345.0,0.0,,,,2482.868015950521,453.8919717206632,3878.35205078125,1454.6219482421875,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,7611.8936,3933.5952,19037.822,1435.7284
36,419.0,0.0,1478.0,36.0,75.0,1478.0,0.4790000000000023,-320.0,-320.0,0.0,,,,2371.336440429688,465.8108835988031,3901.125244140625,1313.65380859375,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9474.646,5553.234,24757.568,1271.4858
37,494.0,0.0,1553.0,37.0,75.0,1553.0,0.4752500000000027,-325.0,-325.0,0.0,,,,2169.166463216146,425.75691268537287,3692.046630859375,1355.68359375,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8821.158000000001,5241.8975,27872.492000000002,1564.2307
38,569.0,0.0,1628.0,38.0,75.0,1628.0,0.4715000000000032,-340.0,-340.0,0.0,,,,2110.4203141276043,459.47185940817917,3180.8662109375,1098.3739013671877,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8521.446,6072.1846,34998.13,1994.5767
39,644.0,0.0,1703.0,39.0,75.0,1703.0,0.4677500000000036,-340.0,-340.0,0.0,,,,2034.7563606770834,460.3998665400683,3170.345947265625,1119.7576904296877,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,12934.934,7226.505,32803.66,792.9248
40,678.0,0.0,1737.0,40.0,34.0,1737.0,0.4660500000000037,-48.0,-48.0,0.0,,,,1953.1156221277572,545.1958983368963,3336.509521484375,1179.089111328125,0.0001,1.3552527156068802e-20,0.0001,0.0001,12777.6875,7534.3335,28442.002,1934.1208
41,753.0,0.0,1812.0,41.0,75.0,1812.0,0.4623000000000042,-335.0,-335.0,0.0,,,,2128.1311783854167,520.2510966435275,3962.064208984375,1156.8756103515625,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9502.953000000001,5733.1,35032.797,1371.4753
42,828.0,0.0,1887.0,42.0,75.0,1887.0,0.4585500000000046,-330.0,-330.0,0.0,,,,2020.592638346354,508.0626314771112,3320.949462890625,1051.14404296875,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,10068.6,5923.7305,25615.827999999998,1038.5610000000001
43,838.0,0.0,1897.0,43.0,10.0,1897.0,0.4580500000000046,58.0,58.0,0.0,,,,1911.0807373046875,484.4154870198784,2693.1943359375,1030.531982421875,0.0001,0.0,0.0001,0.0001,10108.119,6024.936,23948.987999999998,4229.723
44,913.0,0.0,1972.0,44.0,75.0,1972.0,0.454300000000005,-320.0,-320.0,0.0,,,,1927.001064453125,514.9306701568819,3966.235595703125,1034.498046875,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9262.863000000001,6342.0645,31713.102000000003,1045.8743
45,922.0,0.0,1981.0,45.0,9.0,1981.0,0.4538500000000051,63.0,63.0,0.0,,,,1808.1862657335068,297.1387353146108,2182.508056640625,1355.8675537109375,0.0001,0.0,0.0001,0.0001,10279.511999999999,4780.2217,17691.68,4316.143
46,997.0,0.0,2056.0,46.0,75.0,2056.0,0.4501000000000055,-315.0,-315.0,0.0,,,,2032.1784993489584,475.90695176095335,3158.3447265625,1094.5611572265623,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,10441.543,6276.14,26625.377,1116.9589
47,1072.0,0.0,2131.0,47.0,75.0,2131.0,0.4463500000000059,-335.0,-335.0,0.0,,,,1901.55345703125,414.601381621737,2863.72021484375,1102.75927734375,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,10468.1455,5887.8525,28733.54,1633.4474
48,1147.0,0.0,2206.0,48.0,75.0,2206.0,0.4426000000000063,-325.0,-325.0,0.0,,,,1844.411834309896,443.70547118732435,3207.015380859375,854.2076416015625,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9129.1,4776.86,23007.46,1170.9935
49,1222.0,0.0,2281.0,49.0,75.0,2281.0,0.4388500000000067,-355.0,-355.0,0.0,,,,1776.5706233723959,471.82444062467584,3012.540283203125,1001.5508422851562,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9043.123,5778.9585,33845.637,1300.7959
50,1230.0,0.0,2289.0,50.0,8.0,2289.0,0.4384500000000068,67.0,67.0,0.0,,,,1590.4605712890625,280.995974030903,2036.3619384765625,1259.98388671875,0.0001,0.0,0.0001,0.0001,15035.923999999999,5842.4980000000005,20706.568,3507.5747
51,1259.0,0.0,2318.0,51.0,29.0,2318.0,0.4370000000000069,-28.0,-28.0,0.0,,,,1663.7892898033404,464.8343258797997,2668.75830078125,751.3123168945312,9.999999999999998e-05,2.7105054312137605e-20,0.0001,0.0001,12248.117,7167.755,30616.121,2583.2129999999997
52,1334.0,0.0,2393.0,52.0,75.0,2393.0,0.43325000000000735,-340.0,-340.0,0.0,,,,1850.2906884765625,415.024752038264,3271.372314453125,1060.467041015625,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,7113.795999999999,4360.2896,23007.64,1550.2559
53,1409.0,0.0,2468.0,53.0,75.0,2468.0,0.42950000000000776,-345.0,-345.0,0.0,,,,1730.308134765625,427.79760138333467,2762.183837890625,775.44775390625,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8396.429,4558.9259999999995,23571.781000000003,1970.5563
54,1431.0,0.0,2490.0,54.0,22.0,2490.0,0.4284000000000079,0.0,0.0,0.0,,,,1846.6991105513134,385.4605561709925,2543.722412109375,915.1123657226562,0.0001,1.3552527156068802e-20,0.0001,0.0001,8495.571,6709.9890000000005,33833.473,2038.4215
55,1473.0,0.0,2532.0,55.0,42.0,2532.0,0.4263000000000081,-90.0,-90.0,0.0,,,,1890.0395827520458,446.4111479171437,2914.133056640625,755.6181640625,0.00010000000000000002,1.3552527156068802e-20,0.0001,0.0001,7235.0454,4041.2095,21379.494,1302.8073
56,1476.0,0.0,2535.0,56.0,3.0,2535.0,0.4261500000000081,91.0,91.0,0.0,,,,1912.030314127604,342.86427285162813,2363.271728515625,1532.722412109375,0.0001,0.0,0.0001,0.0001,7210.1055,2696.3638,10781.295,4266.702
57,1479.0,0.0,2538.0,57.0,3.0,2538.0,0.42600000000000815,91.0,91.0,0.0,,,,2022.01904296875,359.73376536375997,2496.085693359375,1625.116455078125,0.0001,0.0,0.0001,0.0001,4923.2085,1971.7198,6627.281,2159.7270000000003
58,1510.0,0.0,2569.0,58.0,31.0,2569.0,0.4244500000000083,-31.0,-31.0,0.0,,,,1936.5541893743698,445.76280005387605,2937.03125,987.7350463867188,9.999999999999996e-05,4.0657581468206416e-20,0.0001,0.0001,8962.213,4653.1475,18630.568,2292.3245
59,1585.0,0.0,2644.0,59.0,75.0,2644.0,0.4207000000000087,-345.0,-345.0,0.0,,,,2062.300531412761,461.74250925020425,3443.134521484375,908.7151489257812,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,10726.526000000002,6288.1504,34759.223,1402.5997
60,1594.0,0.0,2653.0,60.0,9.0,2653.0,0.4202500000000088,57.0,57.0,0.0,,,,1937.900404188368,449.70101558453797,2752.651123046875,1147.4949951171875,0.0001,0.0,0.0001,0.0001,9633.898000000001,6012.1284,24146.093999999997,3509.1484
61,1669.0,0.0,2728.0,61.0,75.0,2728.0,0.4165000000000092,-335.0,-335.0,0.0,,,,2081.188806152344,489.54920371293326,3186.210205078125,793.710205078125,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9517.860999999999,6173.943,25958.996,1470.1646
62,1714.0,0.0,2773.0,62.0,45.0,2773.0,0.41425000000000944,-107.0,-107.0,0.0,,,,1979.9250569661456,444.95341455418776,3409.033203125,1067.50146484375,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,11889.282,7202.778,28449.816000000003,1965.8135
63,1755.0,0.0,2814.0,63.0,41.0,2814.0,0.4122000000000097,-86.0,-86.0,0.0,,,,1949.386736613948,438.01420887891226,2898.2509765625,1162.8082275390625,0.0001,0.0,0.0001,0.0001,8248.635,5983.8467,27771.922000000002,1338.8822
64,1766.0,0.0,2825.0,64.0,11.0,2825.0,0.4116500000000097,47.0,47.0,0.0,,,,2142.1350430575285,338.90633193526367,2704.8408203125,1582.5853271484375,0.0001,0.0,0.0001,0.0001,6870.5347,3548.6636,14772.337,2794.1462
65,1841.0,0.0,2900.0,65.0,75.0,2900.0,0.4079000000000101,-345.0,-345.0,0.0,,,,2000.6128336588545,430.99328281858436,2908.90380859375,960.3004150390624,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,10136.796,5358.4326,28253.752,1138.26
31,74.0,0.0,1133.0,31.0,75.0,1133.0,0.4962500000000004,-310.0,-310.0,0.0,,,,10044.869384765623,5242.989884619314,22254.58203125,4401.970703125,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,19706.668,8858.38,54568.945,7987.5923
32,149.0,0.0,1208.0,32.0,75.0,1208.0,0.4925000000000008,-330.0,-330.0,0.0,,,,3891.6225783889363,637.6236269793292,5566.59326171875,2664.003662109375,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8448.435,5481.049,26043.842,2302.9417
33,224.0,0.0,1283.0,33.0,75.0,1283.0,0.4887500000000013,-350.0,-350.0,0.0,,,,3077.779415646115,566.1168689908503,4660.55615234375,1820.854248046875,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,7292.5693,4368.0903,18343.902,1369.8833
34,269.0,0.0,1328.0,34.0,45.0,1328.0,0.4865000000000015,-107.0,-107.0,0.0,,,,2778.7146329012785,533.6904650386124,3798.990478515625,1563.0560302734375,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,7935.2515,4627.6504,17427.271,1315.7484
35,344.0,0.0,1403.0,35.0,75.0,1403.0,0.4827500000000019,-350.0,-350.0,0.0,,,,2548.251075538429,503.7895321500006,3885.131103515625,1336.6544189453125,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,6998.5156,3740.0554,15615.944,1059.72
36,419.0,0.0,1478.0,36.0,75.0,1478.0,0.4790000000000023,-330.0,-330.0,0.0,,,,2403.516982619826,544.6621766946007,3862.224609375,1325.1793212890625,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9307.738000000001,5141.1465,26465.752,1347.5527
37,494.0,0.0,1553.0,37.0,75.0,1553.0,0.4752500000000027,-340.0,-340.0,0.0,,,,2239.989168734164,523.1246187150106,3724.3359375,1135.0155029296875,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,11598.228000000001,7139.8769999999995,32025.642999999996,1107.6741
38,569.0,0.0,1628.0,38.0,75.0,1628.0,0.4715000000000032,-340.0,-340.0,0.0,,,,2182.150799725507,468.15083657342313,3649.64501953125,1002.2430419921876,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,12550.713,7233.5986,36740.902,1885.9489
39,572.0,0.0,1631.0,39.0,3.0,1631.0,0.4713500000000032,91.0,91.0,0.0,,,,2704.028686523437,118.6094970703125,2822.63818359375,2585.419189453125,0.0001,0.0,0.0001,0.0001,25972.59,3807.837,29780.428,22164.754
40,604.0,0.0,1663.0,40.0,32.0,1663.0,0.4697500000000033,-45.0,-45.0,0.0,,,,1930.68742124496,502.9846513531696,3201.093017578125,1072.453125,9.999999999999996e-05,4.0657581468206416e-20,0.0001,0.0001,10140.787,5985.6997,23714.36,2408.1848
41,638.0,0.0,1697.0,41.0,34.0,1697.0,0.4680500000000035,-43.0,-43.0,0.0,,,,2243.9218897964015,436.8811365455244,2995.576416015625,1454.529541015625,0.0001,0.0,0.0001,0.0001,9840.296999999999,6234.153,22699.38,1652.4387
42,713.0,0.0,1772.0,42.0,75.0,1772.0,0.4643000000000039,-335.0,-335.0,0.0,,,,2289.5660070470867,429.4154174081929,3671.47412109375,1392.2723388671875,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9503.265,5894.2974,23384.83,1498.6058
43,739.0,0.0,1798.0,43.0,26.0,1798.0,0.4630000000000041,-6.0,-6.0,0.0,,,,2134.9425048828125,455.004647509431,3364.474853515625,1390.66064453125,0.0001,0.0,0.0001,0.0001,9851.771,5102.7183,21334.49,2035.835
44,814.0,0.0,1873.0,44.0,75.0,1873.0,0.4592500000000045,-315.0,-315.0,0.0,,,,2201.6732433422194,514.7195921344477,3464.29150390625,1020.7645874023438,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8819.354,5085.938,21895.092,1296.8114
45,889.0,0.0,1948.0,45.0,75.0,1948.0,0.4555000000000049,-345.0,-345.0,0.0,,,,2207.215939083615,554.4240054132572,4052.049072265625,1164.0147705078125,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9008.161,5704.8369999999995,28084.967,1202.0845
46,964.0,0.0,2023.0,46.0,75.0,2023.0,0.4517500000000053,-335.0,-335.0,0.0,,,,2089.54076406118,431.9918054466581,2953.16748046875,1021.1974487304688,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8598.306,4905.4497,22985.017999999996,1388.7579999999998
47,983.0,0.0,2042.0,47.0,19.0,2042.0,0.4508000000000054,17.0,17.0,0.0,,,,2057.0487060546875,486.034561988616,3028.0859375,1063.8616943359377,0.0001,0.0,0.0001,0.0001,12460.983999999999,7062.5854,26178.94,2331.4316
48,1008.0,0.0,2067.0,48.0,25.0,2067.0,0.4495500000000056,-22.0,-22.0,0.0,,,,2126.609670003256,519.2346744619607,3212.98486328125,1283.1513671875,0.0001,0.0,0.0001,0.0001,12462.714,5630.728,23076.666,4272.0425
49,1083.0,0.0,2142.0,49.0,75.0,2142.0,0.445800000000006,-325.0,-325.0,0.0,,,,2137.4704177444046,463.33556068091485,3200.396484375,1114.6522216796877,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,11589.511999999999,6253.302,30283.914,1995.5632
50,1111.0,0.0,2170.0,50.0,28.0,2170.0,0.4444000000000061,-23.0,-23.0,0.0,,,,2167.5146213107637,418.6486196316667,2978.5595703125,1144.399169921875,0.0001,1.3552527156068802e-20,0.0001,0.0001,10905.161,6223.7334,30859.734,2938.511
51,1125.0,0.0,2184.0,51.0,14.0,2184.0,0.4437000000000062,47.0,47.0,0.0,,,,2260.243361253005,270.9846229965501,2814.1787109375,1916.0750732421875,0.00010000000000000002,1.3552527156068802e-20,0.0001,0.0001,10421.190999999999,4779.9146,18740.24,4033.5125
52,1147.0,0.0,2206.0,52.0,22.0,2206.0,0.4426000000000063,0.0,0.0,0.0,,,,2312.171648297991,431.8343393681421,3130.859130859375,1384.267822265625,0.0001,1.3552527156068802e-20,0.0001,0.0001,10536.329,3903.035,17744.582,2334.9014
53,1222.0,0.0,2281.0,53.0,75.0,2281.0,0.4388500000000067,-345.0,-345.0,0.0,,,,2214.8194217166383,490.6178898345874,3438.375,1277.5692138671875,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9800.902,4857.2,25690.402000000002,1710.8159
54,1297.0,0.0,2356.0,54.0,75.0,2356.0,0.4351000000000072,-345.0,-345.0,0.0,,,,2152.6272467535896,493.85394692218085,3566.579833984375,1254.210693359375,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8921.473,5468.608,22560.812,1285.9827
55,1304.0,0.0,2363.0,55.0,7.0,2363.0,0.4347500000000072,75.0,75.0,0.0,,,,2388.065470377604,576.765661200786,3248.211669921875,1657.5279541015625,0.0001,0.0,0.0001,0.0001,10461.333,3841.9446,13288.776000000002,2332.1982
56,1379.0,0.0,2438.0,56.0,75.0,2438.0,0.4310000000000076,-330.0,-330.0,0.0,,,,2083.6998043575804,451.0886134002163,3321.603759765625,1253.80126953125,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,7971.2637,4696.2324,25760.362999999998,1254.4551
57,1381.0,0.0,2440.0,57.0,2.0,2440.0,0.4309000000000076,95.0,95.0,0.0,,,,1848.1578369140625,0.0,1848.1578369140625,1848.1578369140625,0.0001,0.0,0.0001,0.0001,9745.857,0.0,9745.857,9745.857
58,1386.0,0.0,2445.0,58.0,5.0,2445.0,0.4306500000000077,76.0,76.0,0.0,,,,2469.050109863281,378.86445288616164,2849.742431640625,2050.549072265625,0.0001,0.0,0.0001,0.0001,4347.0146,1005.71375,5272.0645,2648.5105
59,1461.0,0.0,2520.0,59.0,75.0,2520.0,0.4269000000000081,-330.0,-330.0,0.0,,,,2129.3425457928633,451.2132821428463,3655.289794921875,1113.34326171875,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,8102.646,4872.226,22095.9,1591.7819
60,1536.0,0.0,2595.0,60.0,75.0,2595.0,0.4231500000000085,-345.0,-345.0,0.0,,,,2085.1278554555533,437.58567881234,3290.2333984375,1268.5421142578125,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9111.574,5091.9233,25067.517999999996,1348.4824
61,1611.0,0.0,2670.0,61.0,75.0,2670.0,0.4194000000000089,-350.0,-350.0,0.0,,,,2200.5184804555533,462.4888624828058,3118.872314453125,1072.3905029296877,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,11947.702,6489.1646,25158.611,1028.8323
62,1686.0,0.0,2745.0,62.0,75.0,2745.0,0.4156500000000093,-325.0,-325.0,0.0,,,,2026.1754109150654,450.22828808801705,3065.60986328125,987.6249389648438,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,10752.766000000001,6122.658,27771.957000000002,1372.0846
63,1761.0,0.0,2820.0,63.0,75.0,2820.0,0.4119000000000097,-335.0,-335.0,0.0,,,,1894.7486151617927,481.5040244992176,3235.630126953125,915.2020263671876,0.00010000000000000003,2.7105054312137605e-20,0.0001,0.0001,9413.462,4855.9287,24218.564,1202.1124
64,1772.0,0.0,2831.0,64.0,11.0,2831.0,0.41135000000000976,49.0,49.0,0.0,,,,1993.5283447265624,453.2569012371,3084.971435546875,1584.0709228515625,0.0001,0.0,0.0001,0.0001,8501.79,4672.495,16319.774,3136.5254
65,1784.0,0.0,2843.0,65.0,12.0,2843.0,0.4107500000000098,38.0,38.0,0.0,,,,1816.0982111150568,312.48056576674486,2356.739990234375,1309.124755859375,0.0001,0.0,0.0001,0.0001,11706.223999999998,6139.613,21345.24,3042.6313
1 Episode # Training Iter In Heatup ER #Transitions ER #Episodes Episode Length Total steps Epsilon Shaped Training Reward Training Reward Update Target Network Evaluation Reward Shaped Evaluation Reward Success Rate Loss/Mean Loss/Stdev Loss/Max Loss/Min Learning Rate/Mean Learning Rate/Stdev Learning Rate/Max Learning Rate/Min Grads (unclipped)/Mean Grads (unclipped)/Stdev Grads (unclipped)/Max Grads (unclipped)/Min
29 28 0.0 1.0 1000.0 28.0 7.0 1000.0 0.5 0.0
30 29 0.0 1.0 1012.0 29.0 12.0 1012.0 0.5 0.0
31 30 0.0 1.0 1058.0 30.0 46.0 1058.0 0.5 0.0
32 31 74.0 0.0 1133.0 31.0 75.0 1133.0 0.4962500000000004 -310.0 -310.0 0.0 9862.112614812078 10044.869384765623 5108.627430680448 5242.989884619314 21896.46875 22254.58203125 4428.638671875 4401.970703125 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 19652.328 19706.668 8994.96 8858.38 52983.04 54568.945 6429.6387 7987.5923
33 32 149.0 0.0 1208.0 32.0 75.0 1208.0 0.4925000000000008 -330.0 -330.0 0.0 3830.6351790364574 3891.6225783889363 643.7340248125081 637.6236269793292 5340.5830078125 5566.59326171875 2646.113525390625 2664.003662109375 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 8379.321 8448.435 5266.063 5481.049 22653.16 26043.842 1860.6244 2302.9417
34 33 194.0 224.0 0.0 1253.0 1283.0 33.0 45.0 75.0 1253.0 1283.0 0.4902500000000011 0.4887500000000013 -122.0 -350.0 -122.0 -350.0 0.0 3059.363802083333 3077.779415646115 588.2946622227233 566.1168689908503 4644.080078125 4660.55615234375 1812.8056640625 1820.854248046875 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 8209.5625 7292.5693 4655.1914 4368.0903 19927.584 18343.902 1891.3015 1369.8833
35 34 269.0 0.0 1328.0 34.0 75.0 45.0 1328.0 0.4865000000000015 -350.0 -107.0 -350.0 -107.0 0.0 2846.889031575521 2778.7146329012785 476.7265607773383 533.6904650386124 4139.458984375 3798.990478515625 1698.8955078125 1563.0560302734375 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 6443.3677 7935.2515 3683.1653 4627.6504 17744.254 17427.271 1654.5441 1315.7484
36 35 344.0 0.0 1403.0 35.0 75.0 1403.0 0.4827500000000019 -345.0 -350.0 -345.0 -350.0 0.0 2482.868015950521 2548.251075538429 453.8919717206632 503.7895321500006 3878.35205078125 3885.131103515625 1454.6219482421875 1336.6544189453125 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 7611.8936 6998.5156 3933.5952 3740.0554 19037.822 15615.944 1435.7284 1059.72
37 36 419.0 0.0 1478.0 36.0 75.0 1478.0 0.4790000000000023 -320.0 -330.0 -320.0 -330.0 0.0 2371.336440429688 2403.516982619826 465.8108835988031 544.6621766946007 3901.125244140625 3862.224609375 1313.65380859375 1325.1793212890625 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 9474.646 9307.738000000001 5553.234 5141.1465 24757.568 26465.752 1271.4858 1347.5527
38 37 494.0 0.0 1553.0 37.0 75.0 1553.0 0.4752500000000027 -325.0 -340.0 -325.0 -340.0 0.0 2169.166463216146 2239.989168734164 425.75691268537287 523.1246187150106 3692.046630859375 3724.3359375 1355.68359375 1135.0155029296875 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 8821.158000000001 11598.228000000001 5241.8975 7139.8769999999995 27872.492000000002 32025.642999999996 1564.2307 1107.6741
39 38 569.0 0.0 1628.0 38.0 75.0 1628.0 0.4715000000000032 -340.0 -340.0 0.0 2110.4203141276043 2182.150799725507 459.47185940817917 468.15083657342313 3180.8662109375 3649.64501953125 1098.3739013671877 1002.2430419921876 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 8521.446 12550.713 6072.1846 7233.5986 34998.13 36740.902 1994.5767 1885.9489
40 39 644.0 572.0 0.0 1703.0 1631.0 39.0 75.0 3.0 1703.0 1631.0 0.4677500000000036 0.4713500000000032 -340.0 91.0 -340.0 91.0 0.0 2034.7563606770834 2704.028686523437 460.3998665400683 118.6094970703125 3170.345947265625 2822.63818359375 1119.7576904296877 2585.419189453125 0.00010000000000000003 0.0001 2.7105054312137605e-20 0.0 0.0001 0.0001 12934.934 25972.59 7226.505 3807.837 32803.66 29780.428 792.9248 22164.754
41 40 678.0 604.0 0.0 1737.0 1663.0 40.0 34.0 32.0 1737.0 1663.0 0.4660500000000037 0.4697500000000033 -48.0 -45.0 -48.0 -45.0 0.0 1953.1156221277572 1930.68742124496 545.1958983368963 502.9846513531696 3336.509521484375 3201.093017578125 1179.089111328125 1072.453125 0.0001 9.999999999999996e-05 1.3552527156068802e-20 4.0657581468206416e-20 0.0001 0.0001 12777.6875 10140.787 7534.3335 5985.6997 28442.002 23714.36 1934.1208 2408.1848
42 41 753.0 638.0 0.0 1812.0 1697.0 41.0 75.0 34.0 1812.0 1697.0 0.4623000000000042 0.4680500000000035 -335.0 -43.0 -335.0 -43.0 0.0 2128.1311783854167 2243.9218897964015 520.2510966435275 436.8811365455244 3962.064208984375 2995.576416015625 1156.8756103515625 1454.529541015625 0.00010000000000000003 0.0001 2.7105054312137605e-20 0.0 0.0001 0.0001 9502.953000000001 9840.296999999999 5733.1 6234.153 35032.797 22699.38 1371.4753 1652.4387
43 42 828.0 713.0 0.0 1887.0 1772.0 42.0 75.0 1887.0 1772.0 0.4585500000000046 0.4643000000000039 -330.0 -335.0 -330.0 -335.0 0.0 2020.592638346354 2289.5660070470867 508.0626314771112 429.4154174081929 3320.949462890625 3671.47412109375 1051.14404296875 1392.2723388671875 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 10068.6 9503.265 5923.7305 5894.2974 25615.827999999998 23384.83 1038.5610000000001 1498.6058
44 43 838.0 739.0 0.0 1897.0 1798.0 43.0 10.0 26.0 1897.0 1798.0 0.4580500000000046 0.4630000000000041 58.0 -6.0 58.0 -6.0 0.0 1911.0807373046875 2134.9425048828125 484.4154870198784 455.004647509431 2693.1943359375 3364.474853515625 1030.531982421875 1390.66064453125 0.0001 0.0 0.0001 0.0001 10108.119 9851.771 6024.936 5102.7183 23948.987999999998 21334.49 4229.723 2035.835
45 44 913.0 814.0 0.0 1972.0 1873.0 44.0 75.0 1972.0 1873.0 0.454300000000005 0.4592500000000045 -320.0 -315.0 -320.0 -315.0 0.0 1927.001064453125 2201.6732433422194 514.9306701568819 514.7195921344477 3966.235595703125 3464.29150390625 1034.498046875 1020.7645874023438 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 9262.863000000001 8819.354 6342.0645 5085.938 31713.102000000003 21895.092 1045.8743 1296.8114
46 45 922.0 889.0 0.0 1981.0 1948.0 45.0 9.0 75.0 1981.0 1948.0 0.4538500000000051 0.4555000000000049 63.0 -345.0 63.0 -345.0 0.0 1808.1862657335068 2207.215939083615 297.1387353146108 554.4240054132572 2182.508056640625 4052.049072265625 1355.8675537109375 1164.0147705078125 0.0001 0.00010000000000000003 0.0 2.7105054312137605e-20 0.0001 0.0001 10279.511999999999 9008.161 4780.2217 5704.8369999999995 17691.68 28084.967 4316.143 1202.0845
47 46 997.0 964.0 0.0 2056.0 2023.0 46.0 75.0 2056.0 2023.0 0.4501000000000055 0.4517500000000053 -315.0 -335.0 -315.0 -335.0 0.0 2032.1784993489584 2089.54076406118 475.90695176095335 431.9918054466581 3158.3447265625 2953.16748046875 1094.5611572265623 1021.1974487304688 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 10441.543 8598.306 6276.14 4905.4497 26625.377 22985.017999999996 1116.9589 1388.7579999999998
48 47 1072.0 983.0 0.0 2131.0 2042.0 47.0 75.0 19.0 2131.0 2042.0 0.4463500000000059 0.4508000000000054 -335.0 17.0 -335.0 17.0 0.0 1901.55345703125 2057.0487060546875 414.601381621737 486.034561988616 2863.72021484375 3028.0859375 1102.75927734375 1063.8616943359377 0.00010000000000000003 0.0001 2.7105054312137605e-20 0.0 0.0001 0.0001 10468.1455 12460.983999999999 5887.8525 7062.5854 28733.54 26178.94 1633.4474 2331.4316
49 48 1147.0 1008.0 0.0 2206.0 2067.0 48.0 75.0 25.0 2206.0 2067.0 0.4426000000000063 0.4495500000000056 -325.0 -22.0 -325.0 -22.0 0.0 1844.411834309896 2126.609670003256 443.70547118732435 519.2346744619607 3207.015380859375 3212.98486328125 854.2076416015625 1283.1513671875 0.00010000000000000003 0.0001 2.7105054312137605e-20 0.0 0.0001 0.0001 9129.1 12462.714 4776.86 5630.728 23007.46 23076.666 1170.9935 4272.0425
50 49 1222.0 1083.0 0.0 2281.0 2142.0 49.0 75.0 2281.0 2142.0 0.4388500000000067 0.445800000000006 -355.0 -325.0 -355.0 -325.0 0.0 1776.5706233723959 2137.4704177444046 471.82444062467584 463.33556068091485 3012.540283203125 3200.396484375 1001.5508422851562 1114.6522216796877 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 9043.123 11589.511999999999 5778.9585 6253.302 33845.637 30283.914 1300.7959 1995.5632
51 50 1230.0 1111.0 0.0 2289.0 2170.0 50.0 8.0 28.0 2289.0 2170.0 0.4384500000000068 0.4444000000000061 67.0 -23.0 67.0 -23.0 0.0 1590.4605712890625 2167.5146213107637 280.995974030903 418.6486196316667 2036.3619384765625 2978.5595703125 1259.98388671875 1144.399169921875 0.0001 0.0 1.3552527156068802e-20 0.0001 0.0001 15035.923999999999 10905.161 5842.4980000000005 6223.7334 20706.568 30859.734 3507.5747 2938.511
52 51 1259.0 1125.0 0.0 2318.0 2184.0 51.0 29.0 14.0 2318.0 2184.0 0.4370000000000069 0.4437000000000062 -28.0 47.0 -28.0 47.0 0.0 1663.7892898033404 2260.243361253005 464.8343258797997 270.9846229965501 2668.75830078125 2814.1787109375 751.3123168945312 1916.0750732421875 9.999999999999998e-05 0.00010000000000000002 2.7105054312137605e-20 1.3552527156068802e-20 0.0001 0.0001 12248.117 10421.190999999999 7167.755 4779.9146 30616.121 18740.24 2583.2129999999997 4033.5125
53 52 1334.0 1147.0 0.0 2393.0 2206.0 52.0 75.0 22.0 2393.0 2206.0 0.43325000000000735 0.4426000000000063 -340.0 0.0 -340.0 0.0 0.0 1850.2906884765625 2312.171648297991 415.024752038264 431.8343393681421 3271.372314453125 3130.859130859375 1060.467041015625 1384.267822265625 0.00010000000000000003 0.0001 2.7105054312137605e-20 1.3552527156068802e-20 0.0001 0.0001 7113.795999999999 10536.329 4360.2896 3903.035 23007.64 17744.582 1550.2559 2334.9014
54 53 1409.0 1222.0 0.0 2468.0 2281.0 53.0 75.0 2468.0 2281.0 0.42950000000000776 0.4388500000000067 -345.0 -345.0 0.0 1730.308134765625 2214.8194217166383 427.79760138333467 490.6178898345874 2762.183837890625 3438.375 775.44775390625 1277.5692138671875 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 8396.429 9800.902 4558.9259999999995 4857.2 23571.781000000003 25690.402000000002 1970.5563 1710.8159
55 54 1431.0 1297.0 0.0 2490.0 2356.0 54.0 22.0 75.0 2490.0 2356.0 0.4284000000000079 0.4351000000000072 0.0 -345.0 0.0 -345.0 0.0 1846.6991105513134 2152.6272467535896 385.4605561709925 493.85394692218085 2543.722412109375 3566.579833984375 915.1123657226562 1254.210693359375 0.0001 0.00010000000000000003 1.3552527156068802e-20 2.7105054312137605e-20 0.0001 0.0001 8495.571 8921.473 6709.9890000000005 5468.608 33833.473 22560.812 2038.4215 1285.9827
56 55 1473.0 1304.0 0.0 2532.0 2363.0 55.0 42.0 7.0 2532.0 2363.0 0.4263000000000081 0.4347500000000072 -90.0 75.0 -90.0 75.0 0.0 1890.0395827520458 2388.065470377604 446.4111479171437 576.765661200786 2914.133056640625 3248.211669921875 755.6181640625 1657.5279541015625 0.00010000000000000002 0.0001 1.3552527156068802e-20 0.0 0.0001 0.0001 7235.0454 10461.333 4041.2095 3841.9446 21379.494 13288.776000000002 1302.8073 2332.1982
57 56 1476.0 1379.0 0.0 2535.0 2438.0 56.0 3.0 75.0 2535.0 2438.0 0.4261500000000081 0.4310000000000076 91.0 -330.0 91.0 -330.0 0.0 1912.030314127604 2083.6998043575804 342.86427285162813 451.0886134002163 2363.271728515625 3321.603759765625 1532.722412109375 1253.80126953125 0.0001 0.00010000000000000003 0.0 2.7105054312137605e-20 0.0001 0.0001 7210.1055 7971.2637 2696.3638 4696.2324 10781.295 25760.362999999998 4266.702 1254.4551
58 57 1479.0 1381.0 0.0 2538.0 2440.0 57.0 3.0 2.0 2538.0 2440.0 0.42600000000000815 0.4309000000000076 91.0 95.0 91.0 95.0 0.0 2022.01904296875 1848.1578369140625 359.73376536375997 0.0 2496.085693359375 1848.1578369140625 1625.116455078125 1848.1578369140625 0.0001 0.0 0.0001 0.0001 4923.2085 9745.857 1971.7198 0.0 6627.281 9745.857 2159.7270000000003 9745.857
59 58 1510.0 1386.0 0.0 2569.0 2445.0 58.0 31.0 5.0 2569.0 2445.0 0.4244500000000083 0.4306500000000077 -31.0 76.0 -31.0 76.0 0.0 1936.5541893743698 2469.050109863281 445.76280005387605 378.86445288616164 2937.03125 2849.742431640625 987.7350463867188 2050.549072265625 9.999999999999996e-05 0.0001 4.0657581468206416e-20 0.0 0.0001 0.0001 8962.213 4347.0146 4653.1475 1005.71375 18630.568 5272.0645 2292.3245 2648.5105
60 59 1585.0 1461.0 0.0 2644.0 2520.0 59.0 75.0 2644.0 2520.0 0.4207000000000087 0.4269000000000081 -345.0 -330.0 -345.0 -330.0 0.0 2062.300531412761 2129.3425457928633 461.74250925020425 451.2132821428463 3443.134521484375 3655.289794921875 908.7151489257812 1113.34326171875 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 10726.526000000002 8102.646 6288.1504 4872.226 34759.223 22095.9 1402.5997 1591.7819
61 60 1594.0 1536.0 0.0 2653.0 2595.0 60.0 9.0 75.0 2653.0 2595.0 0.4202500000000088 0.4231500000000085 57.0 -345.0 57.0 -345.0 0.0 1937.900404188368 2085.1278554555533 449.70101558453797 437.58567881234 2752.651123046875 3290.2333984375 1147.4949951171875 1268.5421142578125 0.0001 0.00010000000000000003 0.0 2.7105054312137605e-20 0.0001 0.0001 9633.898000000001 9111.574 6012.1284 5091.9233 24146.093999999997 25067.517999999996 3509.1484 1348.4824
62 61 1669.0 1611.0 0.0 2728.0 2670.0 61.0 75.0 2728.0 2670.0 0.4165000000000092 0.4194000000000089 -335.0 -350.0 -335.0 -350.0 0.0 2081.188806152344 2200.5184804555533 489.54920371293326 462.4888624828058 3186.210205078125 3118.872314453125 793.710205078125 1072.3905029296877 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 9517.860999999999 11947.702 6173.943 6489.1646 25958.996 25158.611 1470.1646 1028.8323
63 62 1714.0 1686.0 0.0 2773.0 2745.0 62.0 45.0 75.0 2773.0 2745.0 0.41425000000000944 0.4156500000000093 -107.0 -325.0 -107.0 -325.0 0.0 1979.9250569661456 2026.1754109150654 444.95341455418776 450.22828808801705 3409.033203125 3065.60986328125 1067.50146484375 987.6249389648438 0.00010000000000000003 2.7105054312137605e-20 0.0001 0.0001 11889.282 10752.766000000001 7202.778 6122.658 28449.816000000003 27771.957000000002 1965.8135 1372.0846
64 63 1755.0 1761.0 0.0 2814.0 2820.0 63.0 41.0 75.0 2814.0 2820.0 0.4122000000000097 0.4119000000000097 -86.0 -335.0 -86.0 -335.0 0.0 1949.386736613948 1894.7486151617927 438.01420887891226 481.5040244992176 2898.2509765625 3235.630126953125 1162.8082275390625 915.2020263671876 0.0001 0.00010000000000000003 0.0 2.7105054312137605e-20 0.0001 0.0001 8248.635 9413.462 5983.8467 4855.9287 27771.922000000002 24218.564 1338.8822 1202.1124
65 64 1766.0 1772.0 0.0 2825.0 2831.0 64.0 11.0 2825.0 2831.0 0.4116500000000097 0.41135000000000976 47.0 49.0 47.0 49.0 0.0 2142.1350430575285 1993.5283447265624 338.90633193526367 453.2569012371 2704.8408203125 3084.971435546875 1582.5853271484375 1584.0709228515625 0.0001 0.0 0.0001 0.0001 6870.5347 8501.79 3548.6636 4672.495 14772.337 16319.774 2794.1462 3136.5254
66 65 1841.0 1784.0 0.0 2900.0 2843.0 65.0 75.0 12.0 2900.0 2843.0 0.4079000000000101 0.4107500000000098 -345.0 38.0 -345.0 38.0 0.0 2000.6128336588545 1816.0982111150568 430.99328281858436 312.48056576674486 2908.90380859375 2356.739990234375 960.3004150390624 1309.124755859375 0.00010000000000000003 0.0001 2.7105054312137605e-20 0.0 0.0001 0.0001 10136.796 11706.223999999998 5358.4326 6139.613 28253.752 21345.24 1138.26 3042.6313