tests: update traces (#302)

* Traces folder removed from repo and moved to S3
* Traces jobs and update will use directly the S3 files

.circleci/config.yml

@@ -345,6 +345,7 @@ jobs:
       - run:
           name: run gym related trace tests
           command: |
+            aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
             export TRACE_PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-gym -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
@@ -365,6 +366,7 @@ jobs:
       - run:
           name: run doom related trace tests
           command: |
+            aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
             export TRACE_PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-doom -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 30m
@@ -385,6 +387,7 @@ jobs:
       - run:
           name: run mujoco related trace tests
           command: |
+            aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
             export TRACE_PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG'
             python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-mujoco -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
           no_output_timeout: 45m
@@ -395,7 +398,100 @@ jobs:
             kubectl delete ns trace-test-mujoco-${CIRCLE_BUILD_NUM} || true
           when: always
 
-#  Starcraft is running too long for evaluation on nightly runs. Consider running in weekly test.
+  update_gym_traces_test:
+    <<: *executor_prep
+    steps:
+      - checkout
+      - *remote_docker
+      - *restore_cache
+      - *aws_prep
+      - run:
+          name: Update GYM traces
+          command: |
+            kubectl create ns trace-test-gym-test-${CIRCLE_BUILD_NUM}
+            docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) \
+            /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \
+                          && chmod a+x /usr/local/bin/aws-iam-authenticator \
+                          && aws eks update-kubeconfig --name coach-aws-cicd \
+                          && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
+                          && chmod a+x /usr/local/bin/kubectl \
+                          && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-gym-test-${CIRCLE_BUILD_NUM} \
+                          && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
+                          && TRACE_PRESETS="CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL" \
+                          && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -ow -ut -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:${TAG} -b coach-trace-gym-aws-test-${CIRCLE_BUILD_NUM}'
+            docker ps -a -q | head -n 1  | xargs -I% docker cp %:/root/src/rl_coach/traces . || true
+            aws s3 sync ~/repo/traces s3://coach-traces-aws || true
+          no_output_timeout: 30m
+      - run:
+          name: cleanup
+          command: |
+            kubectl delete --all pods --namespace=trace-test-gym-test-${CIRCLE_BUILD_NUM} || true
+            kubectl delete ns trace-test-gym-test-${CIRCLE_BUILD_NUM} || true
+          when: always
+
+  update_doom_traces_test:
+    <<: *executor_prep
+    steps:
+      - checkout
+      - *remote_docker
+      - *restore_cache
+      - *aws_prep
+      - run:
+          name: Update DOOM traces
+          command: |
+            kubectl create ns trace-test-doom-test-${CIRCLE_BUILD_NUM}
+            docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) \
+            /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \
+                          && chmod a+x /usr/local/bin/aws-iam-authenticator \
+                          && aws eks update-kubeconfig --name coach-aws-cicd \
+                          && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
+                          && chmod a+x /usr/local/bin/kubectl \
+                          && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-doom-test-${CIRCLE_BUILD_NUM} \
+                          && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
+                          && TRACE_PRESETS="Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP" \
+                          && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -ow -ut -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:${TAG} -b coach-trace-doom-aws-test-${CIRCLE_BUILD_NUM}'
+            docker ps -a -q | head -n 1  | xargs -I% docker cp %:/root/src/rl_coach/traces . || true
+            aws s3 sync ~/repo/traces s3://coach-traces-aws || true
+          no_output_timeout: 30m
+      - run:
+          name: cleanup
+          command: |
+            kubectl delete --all pods --namespace=trace-test-doom-test-${CIRCLE_BUILD_NUM} || true
+            kubectl delete ns trace-test-doom-test-${CIRCLE_BUILD_NUM} || true
+          when: always
+
+  update_mujoco_traces_test:
+    <<: *executor_prep
+    steps:
+      - checkout
+      - *remote_docker
+      - *restore_cache
+      - *aws_prep
+      - run:
+          name: Update MUJOCO traces
+          command: |
+            kubectl create ns trace-test-mujoco-test-${CIRCLE_BUILD_NUM}
+            docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) \
+            /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \
+                          && chmod a+x /usr/local/bin/aws-iam-authenticator \
+                          && aws eks update-kubeconfig --name coach-aws-cicd \
+                          && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
+                          && chmod a+x /usr/local/bin/kubectl \
+                          && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-mujoco-test-${CIRCLE_BUILD_NUM} \
+                          && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \
+                          && TRACE_PRESETS="BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG" \
+                          && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -ow -ut -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:${TAG} -b coach-trace-mujoco-aws-test-${CIRCLE_BUILD_NUM}'
+            docker ps -a -q | head -n 1  | xargs -I% docker cp %:/root/src/rl_coach/traces . || true
+            aws s3 sync ~/repo/traces s3://coach-traces-aws || true
+          no_output_timeout: 30m
+      - run:
+          name: cleanup
+          command: |
+            kubectl delete --all pods --namespace=trace-test-mujoco-test-${CIRCLE_BUILD_NUM} || true
+            kubectl delete ns trace-test-mujoco-test-${CIRCLE_BUILD_NUM} || true
+          when: always
+
+#  Starcraft is running too long for evaluation on nightly runs. Consider running in weekly test
 #  trace_test_starcraft:
 #    <<: *executor_prep
 #    steps:
@@ -542,6 +638,18 @@ workflows:
 #      - golden_test_starcraft:
 #          requires:
 #            - build_starcraft_env
+      - update_gym_traces_test:
+          requires:
+            - trace_test_gym
+          type: approval
+      - update_doom_traces_test:
+          requires:
+            - trace_test_doom
+          type: approval
+      - update_mujoco_traces_test:
+          requires:
+            - trace_test_mujoco
+          type: approval
       - trace_test_gym:
           requires:
             - gym_approval