From acc7f70de3f5e05e61f1d7caf2cd250d0e695012 Mon Sep 17 00:00:00 2001
From: Zach Dwiel <zach.dwiel@intel.com>
Date: Tue, 25 Sep 2018 16:14:47 -0400
Subject: [PATCH] enumerate each preset as its own test

---
 rl_coach/tests/presets/test_presets.py | 76 +++++++++++++++-----------
 1 file changed, 43 insertions(+), 33 deletions(-)

diff --git a/rl_coach/tests/presets/test_presets.py b/rl_coach/tests/presets/test_presets.py
index 2b0bb90..0e6143f 100644
--- a/rl_coach/tests/presets/test_presets.py
+++ b/rl_coach/tests/presets/test_presets.py
@@ -12,45 +12,55 @@ from subprocess import Popen, DEVNULL
 from rl_coach.logger import screen
 
 
+def all_presets():
+    result = []
+    for f in sorted(os.listdir('rl_coach/presets')):
+        if f.endswith('.py') and f != '__init__.py':
+            result.append(f.split('.')[0])
+    return result
+
+
+@pytest.fixture(params=all_presets())
+def preset(request):
+    return request.param
+
+
 @pytest.mark.integration_test
-def test_all_presets_are_running():
-    # os.chdir("../../")
+def test_preset_runs(preset):
     test_failed = False
-    all_presets = sorted([f.split('.')[0] for f in os.listdir('rl_coach/presets') if f.endswith('.py') and f != '__init__.py'])
-    for preset in all_presets:
-        print("Testing preset {}".format(preset))
 
-        # TODO: this is a temporary workaround for presets which define more than a single available level.
-        # we should probably do this in a more robust way
-        level = ""
-        if "Atari" in preset:
-            level = "breakout"
-        elif "Mujoco" in preset:
-            level = "inverted_pendulum"
-        elif "ControlSuite" in preset:
-            level = "pendulum:swingup"
-        params = ["python3", "rl_coach/coach.py", "-p", preset, "-ns", "-e", ".test"]
-        if level != "":
-            params += ["-lvl", level]
+    print("Testing preset {}".format(preset))
 
-        p = Popen(params, stdout=DEVNULL)
+    # TODO: this is a temporary workaround for presets which define more than a single available level.
+    # we should probably do this in a more robust way
+    level = ""
+    if "Atari" in preset:
+        level = "breakout"
+    elif "Mujoco" in preset:
+        level = "inverted_pendulum"
+    elif "ControlSuite" in preset:
+        level = "pendulum:swingup"
 
-        # wait 10 seconds overhead of initialization etc.
-        time.sleep(10)
-        return_value = p.poll()
+    experiment_name = ".test-" + preset
 
-        if return_value is None:
-            screen.success("{} passed successfully".format(preset))
-        else:
-            test_failed = True
-            screen.error("{} failed".format(preset), crash=False)
+    params = ["python3", "rl_coach/coach.py", "-p", preset, "-ns", "-e", experiment_name]
+    if level != "":
+        params += ["-lvl", level]
 
-        p.kill()
-        if os.path.exists("experiments/.test"):
-            shutil.rmtree("experiments/.test")
+    p = Popen(params, stdout=DEVNULL)
+
+    # wait 10 seconds overhead of initialization etc.
+    time.sleep(10)
+    return_value = p.poll()
+
+    if return_value is None:
+        screen.success("{} passed successfully".format(preset))
+    else:
+        test_failed = True
+        screen.error("{} failed".format(preset), crash=False)
+
+    p.kill()
+    if os.path.exists("experiments/" + experiment_name):
+        shutil.rmtree("experiments/" + experiment_name)
 
     assert not test_failed
-
-
-if __name__ == "__main__":
-    test_all_presets_are_running()