tests: fix traces and changing workflow jobs (#316)

* tests: fix traces export presets

* tests: increase time for traces

* tests

* remove approval

* fix approval

* fix ap

* change worflow jobs

* fix path

* fix repo path

* change run traces

* adding assert

* fix assert

.circleci/config.yml

@@ -345,10 +345,18 @@ jobs:
       - run:
           name: run gym related trace tests
           command: |
-            aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive
-            export TRACE_PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL'
-            python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-gym -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
-          no_output_timeout: 30m
+            kubectl create ns trace-test-gym-test-${CIRCLE_BUILD_NUM}
+            docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) \
+            /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \
+                          && chmod a+x /usr/local/bin/aws-iam-authenticator \
+                          && aws eks update-kubeconfig --name coach-aws-cicd \
+                          && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
+                          && chmod a+x /usr/local/bin/kubectl \
+                          && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-gym-test-${CIRCLE_BUILD_NUM} \
+                          && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
+                          && TRACE_PRESETS="CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL" \
+                          && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:${TAG} -b coach-trace-gym-aws-test-${CIRCLE_BUILD_NUM}'
+          no_output_timeout: 50m
       - run:
           name: cleanup
           command: |
@@ -366,10 +374,18 @@ jobs:
       - run:
           name: run doom related trace tests
           command: |
-            aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive
-            export TRACE_PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP'
-            python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-doom -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
-          no_output_timeout: 30m
+            kubectl create ns trace-test-doom-test-${CIRCLE_BUILD_NUM}
+            docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) \
+            /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \
+                          && chmod a+x /usr/local/bin/aws-iam-authenticator \
+                          && aws eks update-kubeconfig --name coach-aws-cicd \
+                          && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
+                          && chmod a+x /usr/local/bin/kubectl \
+                          && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-doom-test-${CIRCLE_BUILD_NUM} \
+                          && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
+                          && TRACE_PRESETS="Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP,Doom_Basic_Dueling_DDQN,Doom_Basic_DFP" \
+                          && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:${TAG} -b coach-trace-doom-aws-test-${CIRCLE_BUILD_NUM}'
+          no_output_timeout: 50m
       - run:
           name: cleanup
           command: |
@@ -387,10 +403,18 @@ jobs:
       - run:
           name: run mujoco related trace tests
           command: |
-            aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive
-            export TRACE_PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG'
-            python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-mujoco -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
-          no_output_timeout: 45m
+            kubectl create ns trace-test-mujoco-test-${CIRCLE_BUILD_NUM}
+            docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) \
+            /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \
+                          && chmod a+x /usr/local/bin/aws-iam-authenticator \
+                          && aws eks update-kubeconfig --name coach-aws-cicd \
+                          && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
+                          && chmod a+x /usr/local/bin/kubectl \
+                          && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-mujoco-test-${CIRCLE_BUILD_NUM} \
+                          && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
+                          && TRACE_PRESETS="BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG" \
+                          && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:${TAG} -b coach-trace-mujoco-aws-test-${CIRCLE_BUILD_NUM}'
+          no_output_timeout: 50m
       - run:
           name: cleanup
           command: |
@@ -421,7 +445,7 @@ jobs:
                           && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -ow -ut -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:${TAG} -b coach-trace-gym-aws-test-${CIRCLE_BUILD_NUM}'
             docker ps -a -q | head -n 1  | xargs -I% docker cp %:/root/src/rl_coach/traces . || true
             aws s3 sync ~/repo/traces s3://coach-traces-aws || true
-          no_output_timeout: 30m
+          no_output_timeout: 50m
       - run:
           name: cleanup
           command: |
@@ -448,11 +472,11 @@ jobs:
                           && chmod a+x /usr/local/bin/kubectl \
                           && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-doom-test-${CIRCLE_BUILD_NUM} \
                           && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
-                          && TRACE_PRESETS="Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP" \
+                          && TRACE_PRESETS="Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP,Doom_Basic_Dueling_DDQN,Doom_Basic_DFP" \
                           && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -ow -ut -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:${TAG} -b coach-trace-doom-aws-test-${CIRCLE_BUILD_NUM}'
             docker ps -a -q | head -n 1  | xargs -I% docker cp %:/root/src/rl_coach/traces . || true
             aws s3 sync ~/repo/traces s3://coach-traces-aws || true
-          no_output_timeout: 30m
+          no_output_timeout: 50m
       - run:
           name: cleanup
           command: |
@@ -483,7 +507,7 @@ jobs:
                           && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -ow -ut -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:${TAG} -b coach-trace-mujoco-aws-test-${CIRCLE_BUILD_NUM}'
             docker ps -a -q | head -n 1  | xargs -I% docker cp %:/root/src/rl_coach/traces . || true
             aws s3 sync ~/repo/traces s3://coach-traces-aws || true
-          no_output_timeout: 30m
+          no_output_timeout: 50m
       - run:
           name: cleanup
           command: |
@@ -607,25 +631,6 @@ workflows:
       - build_mujoco_env:
           requires:
             - e2e_approval
-#      - build_starcraft_env:
-#          requires:
-#            - e2e_approval
-      - gym_approval:
-          type: approval
-          requires:
-            - golden_test_gym
-      - doom_approval:
-          type: approval
-          requires:
-            - golden_test_doom
-      - mujoco_approval:
-          type: approval
-          requires:
-            - golden_test_mujoco
-#      - starcraft_approval:
-#          type: approval
-#          requires:
-#            - golden_test_starcraft
       - golden_test_gym:
           requires:
             - build_gym_env
@@ -638,27 +643,36 @@ workflows:
 #      - golden_test_starcraft:
 #          requires:
 #            - build_starcraft_env
+      - update_gym_approval:
+          type: approval
+          requires:
+            - build_gym_env
+      - update_doom_approval:
+          type: approval
+          requires:
+            - build_doom_env
+      - update_mujoco_approval:
+          type: approval
+          requires:
+            - build_mujoco_env
       - update_gym_traces_test:
           requires:
-            - trace_test_gym
-          type: approval
+            - update_gym_approval
       - update_doom_traces_test:
           requires:
-            - trace_test_doom
-          type: approval
+            - update_doom_approval
       - update_mujoco_traces_test:
           requires:
-            - trace_test_mujoco
-          type: approval
+            - update_mujoco_approval
       - trace_test_gym:
           requires:
-            - gym_approval
+            - build_gym_env
       - trace_test_doom:
           requires:
-            - doom_approval
+            - build_doom_env
       - trace_test_mujoco:
           requires:
-            - mujoco_approval
+            - build_mujoco_env
 #      - trace_test_starcraft:
 #          requires:
 #            - starcraft_approval