gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2025-12-17 19:20:19 +01:00
Code Issues Projects Releases Wiki Activity

add options for more verbose test errors

Browse Source
This commit is contained in:
Zach Dwiel
2018-01-16 17:12:50 -05:00
parent 8f026bb46f
commit 40e5c628c6
1 changed files with 44 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
run_test.py
View File

@@ -26,7 +26,7 @@ import sys
import time import time
from logger import screen from logger import screen
from utils import list_all_classes_in_module, threaded_cmd_line_run, killed_processes from utils import list_all_classes_in_module, threaded_cmd_line_run, killed_processes
from subprocess import Popen import subprocess
import signal import signal
import argparse import argparse
@@ -43,6 +43,12 @@ if __name__ == '__main__':
parser.add_argument('-in', '--ignore_neon', parser.add_argument('-in', '--ignore_neon',
help="(flag) Don't test neon presets.", help="(flag) Don't test neon presets.",
action='store_true') action='store_true')
parser.add_argument('-v', '--verbose',
help="(flag) display verbose logs in the event of an error",
action='store_true')
parser.add_argument('--stop_after_first_failure',
help="(flag) stop executing tests after the first error",
action='store_true')
args = parser.parse_args() args = parser.parse_args()
if args.preset is not None: if args.preset is not None:
@@ -70,13 +76,33 @@ if __name__ == '__main__':
frameworks.append('neon') frameworks.append('neon')
for framework in frameworks: for framework in frameworks:
if args.stop_after_first_failure and fail_count > 0:
break
test_count += 1 test_count += 1
# run the experiment in a separate thread # run the experiment in a separate thread
screen.log_title("Running test {} - {}".format(preset_name, framework)) screen.log_title("Running test {} - {}".format(preset_name, framework))
cmd = 'CUDA_VISIBLE_DEVICES='' python3 coach.py -p {} -f {} -e {} -n {} -cp "seed=0" &> test_log_{}_{}.txt '\ log_file_name = 'test_log_{preset_name}_{framework}.txt'.format(
.format(preset_name, framework, test_name, preset.test_num_workers, preset_name, framework) preset_name=preset_name,
p = Popen(cmd, shell=True, executable="/bin/bash", preexec_fn=os.setsid) framework=framework,
)
cmd = (
'CUDA_VISIBLE_DEVICES='' python3 coach.py '
'-p {preset_name} '
'-f {framework} '
'-e {test_name} '
'-n {num_workers} '
'-cp "seed=0" '
'&> {log_file_name} '
).format(
preset_name=preset_name,
framework=framework,
test_name=test_name,
num_workers=preset.test_num_workers,
log_file_name=log_file_name,
)
p = subprocess.Popen(cmd, shell=True, executable="/bin/bash", preexec_fn=os.setsid)
# get the csv with the results # get the csv with the results
csv_path = None csv_path = None
@@ -153,10 +179,22 @@ if __name__ == '__main__':
if test_passed: if test_passed:
screen.success("Passed successfully") screen.success("Passed successfully")
else: else:
screen.error("Failed due to a mismatch with the golden", crash=False) if csv_paths:
screen.error("Failed due to insufficient reward", crash=False)
screen.error("preset.test_max_step_threshold: {}".format(preset.test_max_step_threshold), crash=False)
screen.error("preset.test_min_return_threshold: {}".format(preset.test_min_return_threshold), crash=False)
screen.error("averaged_rewards: {}".format(averaged_rewards), crash=False)
screen.error("episode number: {}".format(csv['Episode #'].values[-1]), crash=False)
else:
screen.error("csv file never found", crash=False)
if args.verbose:
screen.error("command exitcode: {}".format(p.returncode), crash=False)
screen.error(open(log_file_name).read(), crash=False)
fail_count += 1 fail_count += 1
shutil.rmtree(test_path) shutil.rmtree(test_path)
screen.separator() screen.separator()
if fail_count == 0: if fail_count == 0:
screen.success(" Summary: " + str(test_count) + "/" + str(test_count) + " tests passed successfully") screen.success(" Summary: " + str(test_count) + "/" + str(test_count) + " tests passed successfully")