mirror of
https://github.com/gryf/coach.git
synced 2025-12-17 19:20:19 +01:00
Workaround the OSError due to bad address failure on the CI runs (#370)
workaround the OSError due to bad address failure on the CI runs
This commit is contained in:
Gal Novik
@@ -184,7 +184,7 @@ jobs:
|
||||
name: run functional tests
|
||||
command: |
|
||||
export FUNCTIONAL_PRESETS="'not Mujoco and not CARLA and not Doom and not Starcraft'"
|
||||
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test -tc "export FUNCTIONAL_PRESETS=${FUNCTIONAL_PRESETS} && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
|
||||
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test -tc "export FUNCTIONAL_PRESETS=${FUNCTIONAL_PRESETS} && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 8192
|
||||
no_output_timeout: 45m
|
||||
- run:
|
||||
name: cleanup
|
||||
|
||||
@@ -50,19 +50,24 @@ def test_preset_args(preset_args, flag, clres, start_time=time.time(),
|
||||
run_cmd.extend(test_flag)
|
||||
print(str(run_cmd))
|
||||
|
||||
proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
|
||||
|
||||
try:
|
||||
a_utils.validate_arg_result(flag=test_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
except AssertionError:
|
||||
# close process once get assert false
|
||||
proc.kill()
|
||||
# if test failed - print logs
|
||||
screen.error(open(clres.stdout.name).read(), crash=False)
|
||||
assert False
|
||||
proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
|
||||
|
||||
try:
|
||||
a_utils.validate_arg_result(flag=test_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
except AssertionError:
|
||||
# close process once get assert false
|
||||
proc.kill()
|
||||
# if test failed - print logs
|
||||
screen.error(open(clres.stdout.name).read(), crash=False)
|
||||
assert False
|
||||
|
||||
except OSError as e:
|
||||
# if test launch failed due to OSError - skip test
|
||||
pytest.skip(e)
|
||||
|
||||
proc.kill()
|
||||
|
||||
@@ -102,20 +107,24 @@ def test_preset_seed(preset_args_for_seed, clres, start_time=time.time(),
|
||||
run_cmd.extend(test_flag)
|
||||
print(str(run_cmd))
|
||||
|
||||
for _ in range(seed_num):
|
||||
proc.append(subprocess.Popen(run_cmd, stdout=clres.stdout,
|
||||
stderr=clres.stdout))
|
||||
|
||||
try:
|
||||
a_utils.validate_arg_result(flag=test_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
except AssertionError:
|
||||
close_processes()
|
||||
# if test failed - print logs
|
||||
screen.error(open(clres.stdout.name).read(), crash=False)
|
||||
assert False
|
||||
for _ in range(seed_num):
|
||||
proc.append(subprocess.Popen(run_cmd, stdout=clres.stdout,
|
||||
stderr=clres.stdout))
|
||||
try:
|
||||
a_utils.validate_arg_result(flag=test_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
except AssertionError:
|
||||
close_processes()
|
||||
# if test failed - print logs
|
||||
screen.error(open(clres.stdout.name).read(), crash=False)
|
||||
assert False
|
||||
|
||||
except OSError as e:
|
||||
# if test launch failed due to OSError - skip test
|
||||
pytest.skip(e)
|
||||
|
||||
close_processes()
|
||||
|
||||
@@ -145,24 +154,29 @@ def test_preset_n_and_ew(preset_args, clres, start_time=time.time(),
|
||||
|
||||
print(str(run_cmd))
|
||||
|
||||
proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
|
||||
|
||||
try:
|
||||
a_utils.validate_arg_result(flag=test_ew_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
|
||||
|
||||
a_utils.validate_arg_result(flag=test_n_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
except AssertionError:
|
||||
# close process once get assert false
|
||||
proc.kill()
|
||||
# if test failed - print logs
|
||||
screen.error(open(clres.stdout.name).read(), crash=False)
|
||||
assert False
|
||||
try:
|
||||
a_utils.validate_arg_result(flag=test_ew_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
|
||||
a_utils.validate_arg_result(flag=test_n_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
except AssertionError:
|
||||
# close process once get assert false
|
||||
proc.kill()
|
||||
# if test failed - print logs
|
||||
screen.error(open(clres.stdout.name).read(), crash=False)
|
||||
assert False
|
||||
|
||||
except OSError as e:
|
||||
# if test launch failed due to OSError - skip test
|
||||
pytest.skip(e)
|
||||
|
||||
proc.kill()
|
||||
|
||||
@@ -201,34 +215,39 @@ def test_preset_n_and_ew_and_onnx(preset_args, clres, start_time=time.time(),
|
||||
|
||||
print(str(run_cmd))
|
||||
|
||||
proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
|
||||
|
||||
try:
|
||||
# Check csv files has been created
|
||||
a_utils.validate_arg_result(flag=test_ew_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
|
||||
|
||||
# Check csv files created same as the number of the workers
|
||||
a_utils.validate_arg_result(flag=test_n_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
try:
|
||||
# Check csv files has been created
|
||||
a_utils.validate_arg_result(flag=test_ew_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
|
||||
# Check checkpoint files
|
||||
a_utils.validate_arg_result(flag=test_s_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
# Check csv files created same as the number of the workers
|
||||
a_utils.validate_arg_result(flag=test_n_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
|
||||
# TODO: add onnx check; issue found #257
|
||||
# Check checkpoint files
|
||||
a_utils.validate_arg_result(flag=test_s_flag,
|
||||
p_valid_params=p_valid_params, clres=clres,
|
||||
process=proc, start_time=start_time,
|
||||
timeout=time_limit)
|
||||
|
||||
except AssertionError:
|
||||
# close process once get assert false
|
||||
proc.kill()
|
||||
# if test failed - print logs
|
||||
screen.error(open(clres.stdout.name).read(), crash=False)
|
||||
assert False
|
||||
# TODO: add onnx check; issue found #257
|
||||
|
||||
except AssertionError:
|
||||
# close process once get assert false
|
||||
proc.kill()
|
||||
# if test failed - print logs
|
||||
screen.error(open(clres.stdout.name).read(), crash=False)
|
||||
assert False
|
||||
|
||||
except OSError as e:
|
||||
# if test launch failed due to OSError - skip test
|
||||
pytest.skip(e)
|
||||
|
||||
proc.kill()
|
||||
|
||||
Reference in New Issue
Block a user