From 740359587d1e60bd6f052fe672f318c43bbd7eda Mon Sep 17 00:00:00 2001
From: anabwan <46447582+anabwan@users.noreply.github.com>
Date: Sun, 5 May 2019 08:28:57 +0300
Subject: [PATCH] tests: fixed nightly (#301)

* tests: fixed nightly

* tests: temp testing functional tests

* tests: temp testing functional tests

* tests: add seed to -cp

* test: last fix
---
 .circleci/config.yml               | 6 +++++-
 rl_coach/tests/test_checkpoint.py  | 1 -
 rl_coach/tests/test_coach_args.py  | 6 ++++++
 rl_coach/tests/utils/args_utils.py | 7 ++++---
 4 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8816006..baab360 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -183,7 +183,7 @@ jobs:
       - run:
           name: run functional tests
           command: |
-            export FUNCTIONAL_PRESETS="'not mujoco and not CARLA and not doom'"
+            export FUNCTIONAL_PRESETS="'not Mujoco and not CARLA and not Doom'"
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test -tc "export FUNCTIONAL_PRESETS=${FUNCTIONAL_PRESETS} && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
       - run:
@@ -498,6 +498,10 @@ workflows:
           type: approval
           requires:
             - build_base
+# Keep this for debugging functional_tests for nightly jobs
+#      - functional_tests:
+#          requires:
+#            - e2e_approval
       - build_gym_env:
           requires:
             - e2e_approval
diff --git a/rl_coach/tests/test_checkpoint.py b/rl_coach/tests/test_checkpoint.py
index 1b0eed2..b0753c1 100644
--- a/rl_coach/tests/test_checkpoint.py
+++ b/rl_coach/tests/test_checkpoint.py
@@ -120,7 +120,6 @@ def test_restore_checkpoint(preset_args, clres, framework,
     entities = a_utils.get_files_from_dir(checkpoint_dir)
 
     assert len(entities) > 0
-    assert "checkpoint" in entities
     assert any(".ckpt." in file for file in entities)
 
     # send CTRL+C to close experiment
diff --git a/rl_coach/tests/test_coach_args.py b/rl_coach/tests/test_coach_args.py
index 0f46940..d6aa9c7 100644
--- a/rl_coach/tests/test_coach_args.py
+++ b/rl_coach/tests/test_coach_args.py
@@ -40,6 +40,12 @@ def test_preset_args(preset_args, flag, clres, start_time=time.time(),
 
     # add flags to run command
     test_flag = a_utils.add_one_flag_value(flag=flag)
+
+    if flag[0] == "-cp":
+        seed = ['--seed', '42']
+        seed_flag = a_utils.add_one_flag_value(flag=seed)
+        run_cmd.extend(seed_flag)
+
     run_cmd.extend(test_flag)
     print(str(run_cmd))
 
diff --git a/rl_coach/tests/utils/args_utils.py b/rl_coach/tests/utils/args_utils.py
index 568d398..bf36400 100644
--- a/rl_coach/tests/utils/args_utils.py
+++ b/rl_coach/tests/utils/args_utils.py
@@ -396,7 +396,7 @@ def validate_arg_result(flag, p_valid_params, clres=None, process=None,
                 time.sleep(1)
 
         # get the first value after heat-up
-        time.sleep(1)
+        time.sleep(3)
         results.append(csv["Total steps"].values[-1])
 
         assert int(results[-1]) >= Def.Consts.num_hs, \
@@ -494,9 +494,10 @@ def validate_arg_result(flag, p_valid_params, clres=None, process=None,
                            worker, and check results.
         """
         # wait until files created
-        csv_path = get_csv_path(clres=clres, extra_tries=20)
-
         num_expected_files = int(flag[1])
+        csv_path = get_csv_path(clres=clres, extra_tries=20,
+                                num_expected_files=num_expected_files)
+
         assert len(csv_path) >= num_expected_files, \
             Def.Consts.ASSERT_MSG.format(str(num_expected_files),
                                          str(len(csv_path)))