Adding framework for multinode tests (#149)

* Currently runs CartPole_ClippedPPO and Mujoco_ClippedPPO with inverted_pendulum level.

.circleci/config.yml

@@ -291,6 +291,41 @@ jobs:
             kubectl delete ns trace-test-mujoco-${CIRCLE_BUILD_NUM} || true
           when: always
 
+  multinode_test:
+    <<: *executor_prep
+    steps:
+      - checkout
+      - *remote_docker
+      - *restore_cache
+      - *aws_prep
+      - *docker_prep
+      - run:
+          name: run multinode test
+          command: |
+            aws s3 mb s3://coach-aws-test-${CIRCLE_BUILD_NUM}
+            kubectl create ns multinode-test-${CIRCLE_BUILD_NUM}
+            docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) \
+            /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \
+                          && chmod a+x /usr/local/bin/aws-iam-authenticator \
+                          && aws eks update-kubeconfig --name coach-aws-cicd \
+                          && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
+                          && chmod a+x /usr/local/bin/kubectl \
+                          && kubectl config set-context $(kubectl config current-context) --namespace=multinode-test-${CIRCLE_BUILD_NUM} \
+                          && python3 rl_coach/tests/test_dist_coach.py -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:${TAG} -b coach-aws-test-${CIRCLE_BUILD_NUM}'
+            docker ps -a -q | head -n 1  | xargs -I% docker cp %:/root/src/experiments . || true
+          no_output_timeout: 30m
+      - store_artifacts:
+          path: ~/repo/experiments
+      - run:
+          name: cleanup
+          command: |
+            kubectl delete --all pods --namespace=golden-test-mujoco-${CIRCLE_BUILD_NUM} || true
+            kubectl delete ns golden-test-mujoco-${CIRCLE_BUILD_NUM} || true
+            aws s3 rm --recursive s3://coach-aws-test-${CIRCLE_BUILD_NUM} || true
+            aws s3 rb s3://coach-aws-test-${CIRCLE_BUILD_NUM} || true
+            kubectl delete ns multinode-test-${CIRCLE_BUILD_NUM} || true
+          when: always
+
   container_deploy:
     <<: *executor_prep
     steps:
@@ -329,6 +364,13 @@ workflows:
       - integration_tests:
           requires:
             - build_base
+      - multinode_approval:
+          type: approval
+          requires:
+            - build_base
+      - multinode_test:
+          requires:
+            - multinode_approval
       - e2e_approval:
           type: approval
           requires: