From b82414138dbbe27a2a2bf09f46aaab7b89429e66 Mon Sep 17 00:00:00 2001 From: Gal Novik Date: Sun, 7 Jul 2019 17:11:19 +0300 Subject: [PATCH] Workaround the OSError due to bad address failure on the CI runs (#370) workaround the OSError due to bad address failure on the CI runs --- .circleci/config.yml | 2 +- rl_coach/tests/test_coach_args.py | 149 +++++++++++++++++------------- 2 files changed, 85 insertions(+), 66 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d79f07..641251d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -184,7 +184,7 @@ jobs: name: run functional tests command: | export FUNCTIONAL_PRESETS="'not Mujoco and not CARLA and not Doom and not Starcraft'" - python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test -tc "export FUNCTIONAL_PRESETS=${FUNCTIONAL_PRESETS} && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 + python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test -tc "export FUNCTIONAL_PRESETS=${FUNCTIONAL_PRESETS} && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 8192 no_output_timeout: 45m - run: name: cleanup diff --git a/rl_coach/tests/test_coach_args.py b/rl_coach/tests/test_coach_args.py index c35b5cc..1f42960 100644 --- a/rl_coach/tests/test_coach_args.py +++ b/rl_coach/tests/test_coach_args.py @@ -50,19 +50,24 @@ def test_preset_args(preset_args, flag, clres, start_time=time.time(), run_cmd.extend(test_flag) print(str(run_cmd)) - proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout) - try: - a_utils.validate_arg_result(flag=test_flag, - p_valid_params=p_valid_params, clres=clres, - process=proc, start_time=start_time, - timeout=time_limit) - except AssertionError: - # close process once get assert false - proc.kill() - # if test failed - print logs - screen.error(open(clres.stdout.name).read(), crash=False) - assert False + proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout) + + try: + a_utils.validate_arg_result(flag=test_flag, + p_valid_params=p_valid_params, clres=clres, + process=proc, start_time=start_time, + timeout=time_limit) + except AssertionError: + # close process once get assert false + proc.kill() + # if test failed - print logs + screen.error(open(clres.stdout.name).read(), crash=False) + assert False + + except OSError as e: + # if test launch failed due to OSError - skip test + pytest.skip(e) proc.kill() @@ -102,20 +107,24 @@ def test_preset_seed(preset_args_for_seed, clres, start_time=time.time(), run_cmd.extend(test_flag) print(str(run_cmd)) - for _ in range(seed_num): - proc.append(subprocess.Popen(run_cmd, stdout=clres.stdout, - stderr=clres.stdout)) - try: - a_utils.validate_arg_result(flag=test_flag, - p_valid_params=p_valid_params, clres=clres, - process=proc, start_time=start_time, - timeout=time_limit) - except AssertionError: - close_processes() - # if test failed - print logs - screen.error(open(clres.stdout.name).read(), crash=False) - assert False + for _ in range(seed_num): + proc.append(subprocess.Popen(run_cmd, stdout=clres.stdout, + stderr=clres.stdout)) + try: + a_utils.validate_arg_result(flag=test_flag, + p_valid_params=p_valid_params, clres=clres, + process=proc, start_time=start_time, + timeout=time_limit) + except AssertionError: + close_processes() + # if test failed - print logs + screen.error(open(clres.stdout.name).read(), crash=False) + assert False + + except OSError as e: + # if test launch failed due to OSError - skip test + pytest.skip(e) close_processes() @@ -145,24 +154,29 @@ def test_preset_n_and_ew(preset_args, clres, start_time=time.time(), print(str(run_cmd)) - proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout) - try: - a_utils.validate_arg_result(flag=test_ew_flag, - p_valid_params=p_valid_params, clres=clres, - process=proc, start_time=start_time, - timeout=time_limit) + proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout) - a_utils.validate_arg_result(flag=test_n_flag, - p_valid_params=p_valid_params, clres=clres, - process=proc, start_time=start_time, - timeout=time_limit) - except AssertionError: - # close process once get assert false - proc.kill() - # if test failed - print logs - screen.error(open(clres.stdout.name).read(), crash=False) - assert False + try: + a_utils.validate_arg_result(flag=test_ew_flag, + p_valid_params=p_valid_params, clres=clres, + process=proc, start_time=start_time, + timeout=time_limit) + + a_utils.validate_arg_result(flag=test_n_flag, + p_valid_params=p_valid_params, clres=clres, + process=proc, start_time=start_time, + timeout=time_limit) + except AssertionError: + # close process once get assert false + proc.kill() + # if test failed - print logs + screen.error(open(clres.stdout.name).read(), crash=False) + assert False + + except OSError as e: + # if test launch failed due to OSError - skip test + pytest.skip(e) proc.kill() @@ -201,34 +215,39 @@ def test_preset_n_and_ew_and_onnx(preset_args, clres, start_time=time.time(), print(str(run_cmd)) - proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout) - try: - # Check csv files has been created - a_utils.validate_arg_result(flag=test_ew_flag, - p_valid_params=p_valid_params, clres=clres, - process=proc, start_time=start_time, - timeout=time_limit) + proc = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout) - # Check csv files created same as the number of the workers - a_utils.validate_arg_result(flag=test_n_flag, - p_valid_params=p_valid_params, clres=clres, - process=proc, start_time=start_time, - timeout=time_limit) + try: + # Check csv files has been created + a_utils.validate_arg_result(flag=test_ew_flag, + p_valid_params=p_valid_params, clres=clres, + process=proc, start_time=start_time, + timeout=time_limit) - # Check checkpoint files - a_utils.validate_arg_result(flag=test_s_flag, - p_valid_params=p_valid_params, clres=clres, - process=proc, start_time=start_time, - timeout=time_limit) + # Check csv files created same as the number of the workers + a_utils.validate_arg_result(flag=test_n_flag, + p_valid_params=p_valid_params, clres=clres, + process=proc, start_time=start_time, + timeout=time_limit) - # TODO: add onnx check; issue found #257 + # Check checkpoint files + a_utils.validate_arg_result(flag=test_s_flag, + p_valid_params=p_valid_params, clres=clres, + process=proc, start_time=start_time, + timeout=time_limit) - except AssertionError: - # close process once get assert false - proc.kill() - # if test failed - print logs - screen.error(open(clres.stdout.name).read(), crash=False) - assert False + # TODO: add onnx check; issue found #257 + + except AssertionError: + # close process once get assert false + proc.kill() + # if test failed - print logs + screen.error(open(clres.stdout.name).read(), crash=False) + assert False + + except OSError as e: + # if test launch failed due to OSError - skip test + pytest.skip(e) proc.kill()