From f14915cada48471e66d9723b43ba6be8dcca9c2d Mon Sep 17 00:00:00 2001
From: anabwan <46447582+anabwan@users.noreply.github.com>
Date: Sun, 21 Apr 2019 13:51:14 +0300
Subject: [PATCH] tests: removed Starcraft from CI (#296)

* tests: removed Starcraft from CI

* tests: fix comment

* tests: fix mujoco
---
 .circleci/config.yml | 184 ++++++++++++++++++++++---------------------
 1 file changed, 94 insertions(+), 90 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3be41b0..8816006 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -117,23 +117,24 @@ jobs:
             docker push ${REGISTRY}/coach-mujoco_environment:${TAG}
           no_output_timeout: 10m
 
-  build_starcraft_env:
-    <<: *executor_prep
-    steps:
-      - checkout
-      - *remote_docker
-      - *restore_cache
-      - *aws_prep
-      - run:
-          name: Build and push starcraft environment container
-          command: |
-            REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com
-            TAG=$(git describe --tags --always --dirty)
-            docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH}
-            docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master
-            docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} -t ${REGISTRY}/coach-starcraft_environment:${TAG} -f docker/Dockerfile.starcraft_environment .
-            docker push ${REGISTRY}/coach-starcraft_environment:${TAG}
-          no_output_timeout: 10m
+#  Starcraft is running too long for evaluation on nightly runs. Consider running in weekly test.
+#  build_starcraft_env:
+#    <<: *executor_prep
+#    steps:
+#      - checkout
+#      - *remote_docker
+#      - *restore_cache
+#      - *aws_prep
+#      - run:
+#          name: Build and push starcraft environment container
+#          command: |
+#            REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com
+#            TAG=$(git describe --tags --always --dirty)
+#            docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH}
+#            docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master
+#            docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} -t ${REGISTRY}/coach-starcraft_environment:${TAG} -f docker/Dockerfile.starcraft_environment .
+#            docker push ${REGISTRY}/coach-starcraft_environment:${TAG}
+#          no_output_timeout: 10m
 
   unit_tests:
     <<: *executor_prep
@@ -252,25 +253,26 @@ jobs:
             kubectl delete ns golden-test-mujoco-${CIRCLE_BUILD_NUM} || true
           when: always
 
-  golden_test_starcraft:
-    <<: *executor_prep
-    steps:
-      - checkout
-      - *remote_docker
-      - *restore_cache
-      - *aws_prep
-      - run:
-          name: run starcraft related golden tests
-          command: |
-            export GOLDEN_PRESETS="Starcraft"
-            python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-starcraft -tc "export GOLDEN_PRESETS=${GOLDEN_PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-starcraft_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
-          no_output_timeout: 30m
-      - run:
-          name: cleanup
-          command: |
-            kubectl delete --all pods --namespace=golden-test-starcraft-${CIRCLE_BUILD_NUM} || true
-            kubectl delete ns golden-test-starcraft-${CIRCLE_BUILD_NUM} || true
-          when: always
+#  Starcraft is running too long for evaluation on nightly runs. Consider running in weekly test.
+#  golden_test_starcraft:
+#    <<: *executor_prep
+#    steps:
+#      - checkout
+#      - *remote_docker
+#      - *restore_cache
+#      - *aws_prep
+#      - run:
+#          name: run starcraft related golden tests
+#          command: |
+#            export GOLDEN_PRESETS="Starcraft"
+#            python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-starcraft -tc "export GOLDEN_PRESETS=${GOLDEN_PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-starcraft_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
+#          no_output_timeout: 30m
+#      - run:
+#          name: cleanup
+#          command: |
+#            kubectl delete --all pods --namespace=golden-test-starcraft-${CIRCLE_BUILD_NUM} || true
+#            kubectl delete ns golden-test-starcraft-${CIRCLE_BUILD_NUM} || true
+#          when: always
 
   functional_test_mujoco:
     <<: *executor_prep
@@ -312,25 +314,26 @@ jobs:
             kubectl delete ns functional-test-doom-${CIRCLE_BUILD_NUM} || true
           when: always
 
-  functional_test_Starcraft:
-    <<: *executor_prep
-    steps:
-      - checkout
-      - *remote_docker
-      - *restore_cache
-      - *aws_prep
-      - run:
-          name: run Starcraft related functional tests
-          command: |
-            export FUNCTIONAL_PRESETS="Starcraft"
-            python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test-starcraft -tc "export FUNCTIONAL_PRESETS='${FUNCTIONAL_PRESETS}' && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-starcraft_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
-          no_output_timeout: 30m
-      - run:
-          name: cleanup
-          command: |
-            kubectl delete --all pods --namespace=functional-test-starcraft-${CIRCLE_BUILD_NUM} || true
-            kubectl delete ns functional-test-starcraft-${CIRCLE_BUILD_NUM} || true
-          when: always
+#  Starcraft is running too long for evaluation on nightly runs. Consider running in weekly test.
+#  functional_test_Starcraft:
+#    <<: *executor_prep
+#    steps:
+#      - checkout
+#      - *remote_docker
+#      - *restore_cache
+#      - *aws_prep
+#      - run:
+#          name: run Starcraft related functional tests
+#          command: |
+#            export FUNCTIONAL_PRESETS="Starcraft"
+#            python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test-starcraft -tc "export FUNCTIONAL_PRESETS='${FUNCTIONAL_PRESETS}' && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-starcraft_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
+#          no_output_timeout: 30m
+#      - run:
+#          name: cleanup
+#          command: |
+#            kubectl delete --all pods --namespace=functional-test-starcraft-${CIRCLE_BUILD_NUM} || true
+#            kubectl delete ns functional-test-starcraft-${CIRCLE_BUILD_NUM} || true
+#          when: always
 
   trace_test_gym:
     <<: *executor_prep
@@ -392,25 +395,26 @@ jobs:
             kubectl delete ns trace-test-mujoco-${CIRCLE_BUILD_NUM} || true
           when: always
 
-  trace_test_starcraft:
-    <<: *executor_prep
-    steps:
-      - checkout
-      - *remote_docker
-      - *restore_cache
-      - *aws_prep
-      - run:
-          name: run starcraft related trace tests
-          command: |
-            export TRACE_PRESETS='Starcraft_CollectMinerals_Arc,Starcraft_CollectMinerals_Deuling_DDQN'
-            python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-starcraft -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-starcraft_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
-          no_output_timeout: 30m
-      - run:
-          name: cleanup
-          command: |
-            kubectl delete --all pods --namespace=trace-test-starcraft-${CIRCLE_BUILD_NUM} || true
-            kubectl delete ns trace-test-starcraft-${CIRCLE_BUILD_NUM} || true
-          when: always
+#  Starcraft is running too long for evaluation on nightly runs. Consider running in weekly test.
+#  trace_test_starcraft:
+#    <<: *executor_prep
+#    steps:
+#      - checkout
+#      - *remote_docker
+#      - *restore_cache
+#      - *aws_prep
+#      - run:
+#          name: run starcraft related trace tests
+#          command: |
+#            export TRACE_PRESETS='Starcraft_CollectMinerals_Arc,Starcraft_CollectMinerals_Deuling_DDQN'
+#            python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-starcraft -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-starcraft_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
+#          no_output_timeout: 30m
+#      - run:
+#          name: cleanup
+#          command: |
+#            kubectl delete --all pods --namespace=trace-test-starcraft-${CIRCLE_BUILD_NUM} || true
+#            kubectl delete ns trace-test-starcraft-${CIRCLE_BUILD_NUM} || true
+#          when: always
 
   multinode_test:
     <<: *executor_prep
@@ -503,9 +507,9 @@ workflows:
       - build_mujoco_env:
           requires:
             - e2e_approval
-      - build_starcraft_env:
-          requires:
-            - e2e_approval
+#      - build_starcraft_env:
+#          requires:
+#            - e2e_approval
       - gym_approval:
           type: approval
           requires:
@@ -518,10 +522,10 @@ workflows:
           type: approval
           requires:
             - golden_test_mujoco
-      - starcraft_approval:
-          type: approval
-          requires:
-            - golden_test_starcraft
+#      - starcraft_approval:
+#          type: approval
+#          requires:
+#            - golden_test_starcraft
       - golden_test_gym:
           requires:
             - build_gym_env
@@ -531,9 +535,9 @@ workflows:
       - golden_test_mujoco:
           requires:
             - build_mujoco_env
-      - golden_test_starcraft:
-          requires:
-            - build_starcraft_env
+#      - golden_test_starcraft:
+#          requires:
+#            - build_starcraft_env
       - trace_test_gym:
           requires:
             - gym_approval
@@ -543,9 +547,9 @@ workflows:
       - trace_test_mujoco:
           requires:
             - mujoco_approval
-      - trace_test_starcraft:
-          requires:
-            - starcraft_approval
+#      - trace_test_starcraft:
+#          requires:
+#            - starcraft_approval
       - container_deploy:
           requires:
             - unit_tests
@@ -568,7 +572,7 @@ workflows:
       - build_gym_env
       - build_doom_env
       - build_mujoco_env
-      # Starcraft disabled from nightly because tests are un-stabled
+      #  Starcraft is running too long for evaluation on nightly runs. Consider running in weekly test.
       #- build_starcraft_env
       - functional_test_doom:
             requires:
@@ -577,7 +581,7 @@ workflows:
             requires:
               - build_mujoco_env
               - functional_test_doom
-      # Starcraft disabled from nightly because tests are un-stabled
+      #  Starcraft is running too long for evaluation on nightly runs. Consider running in weekly test.
       #- functional_test_starcraft:
       #      requires:
       #        - build_starcraft_env
@@ -593,7 +597,7 @@ workflows:
           requires:
             - build_mujoco_env
             - golden_test_doom
-      # Starcraft disabled from nightly because tests are un-stabled
+      #  Starcraft is running too long for evaluation on nightly runs. Consider running in weekly test.
       #- golden_test_starcraft:
       #    requires:
       #      - build_starcraft_env