enumerate each preset as its own test

rl_coach/tests/presets/test_presets.py

@@ -12,12 +12,23 @@ from subprocess import Popen, DEVNULL
 from rl_coach.logger import screen
 
 
+def all_presets():
+    result = []
+    for f in sorted(os.listdir('rl_coach/presets')):
+        if f.endswith('.py') and f != '__init__.py':
+            result.append(f.split('.')[0])
+    return result
+
+
+@pytest.fixture(params=all_presets())
+def preset(request):
+    return request.param
+
+
 @pytest.mark.integration_test
-def test_all_presets_are_running():
-    # os.chdir("../../")
+def test_preset_runs(preset):
     test_failed = False
-    all_presets = sorted([f.split('.')[0] for f in os.listdir('rl_coach/presets') if f.endswith('.py') and f != '__init__.py'])
-    for preset in all_presets:
+
     print("Testing preset {}".format(preset))
 
     # TODO: this is a temporary workaround for presets which define more than a single available level.
@@ -29,7 +40,10 @@ def test_all_presets_are_running():
         level = "inverted_pendulum"
     elif "ControlSuite" in preset:
         level = "pendulum:swingup"
-        params = ["python3", "rl_coach/coach.py", "-p", preset, "-ns", "-e", ".test"]
+
+    experiment_name = ".test-" + preset
+
+    params = ["python3", "rl_coach/coach.py", "-p", preset, "-ns", "-e", experiment_name]
     if level != "":
         params += ["-lvl", level]
 
@@ -46,11 +60,7 @@ def test_all_presets_are_running():
         screen.error("{} failed".format(preset), crash=False)
 
     p.kill()
-        if os.path.exists("experiments/.test"):
-            shutil.rmtree("experiments/.test")
+    if os.path.exists("experiments/" + experiment_name):
+        shutil.rmtree("experiments/" + experiment_name)
 
     assert not test_failed
-
-
-if __name__ == "__main__":
-    test_all_presets_are_running()