aliases: - &executor_prep docker: - image: circleci/python:3.7.0-stretch working_directory: ~/repo - &remote_docker # ensure layers of constructed docker containers are cached for reuse between jobs. setup_remote_docker: docker_layer_caching: true - &restore_cache restore_cache: keys: - v1-dependencies-{{ checksum "requirements.txt" }} # fallback to using the latest cache if no exact match is found - v1-dependencies- - &save_cache save_cache: paths: - ./venv key: v1-dependencies-{{ checksum "requirements.txt" }} - &aws_prep run: name: Prepare aws cli command: | sudo pip install awscli pytest kubernetes==8.0.0b1 $(aws ecr get-login --no-include-email --region us-west-2) sudo curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator sudo chmod a+x /usr/local/bin/aws-iam-authenticator aws eks update-kubeconfig --name coach-aws-cicd sudo curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl sudo chmod a+x /usr/local/bin/kubectl version: 2 jobs: build_base: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: Build and push base and main container command: | REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com TAG=$(git describe --tags --always --dirty) docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH} docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} -t ${REGISTRY}/coach-base:${TAG} -f docker/Dockerfile.base . docker push ${REGISTRY}/coach-base:${TAG} docker tag ${REGISTRY}/coach-base:${TAG} coach-base:master docker pull ${REGISTRY}/coach:${MASTER_BRANCH} docker pull ${REGISTRY}/coach:builder-${MASTER_BRANCH} docker build --target builder --cache-from ${REGISTRY}/coach:${MASTER_BRANCH} --cache-from ${REGISTRY}/coach:builder-${MASTER_BRANCH} --build-arg MUJOCO_KEY=${MUJOCO_KEY} -t ${REGISTRY}/coach:builder-${TAG} -f docker/Dockerfile . docker push ${REGISTRY}/coach:builder-${TAG} docker build --cache-from ${REGISTRY}/coach:${MASTER_BRANCH} --cache-from ${REGISTRY}/coach:builder-${TAG} --build-arg MUJOCO_KEY=${MUJOCO_KEY} -t ${REGISTRY}/coach:${TAG} -f docker/Dockerfile . docker push ${REGISTRY}/coach:${TAG} no_output_timeout: 30m build_gym_env: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: Build and push gym environment container command: | REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com TAG=$(git describe --tags --always --dirty) docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH} docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} -t ${REGISTRY}/coach-gym_environment:${TAG} -f docker/Dockerfile.gym_environment . docker push ${REGISTRY}/coach-gym_environment:${TAG} no_output_timeout: 10m build_doom_env: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: Build and push vizdoom environment container command: | REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com TAG=$(git describe --tags --always --dirty) docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH} docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} -t ${REGISTRY}/coach-doom_environment:${TAG} -f docker/Dockerfile.doom_environment . docker push ${REGISTRY}/coach-doom_environment:${TAG} no_output_timeout: 10m build_mujoco_env: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: Build and push mujoco environment container command: | REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com TAG=$(git describe --tags --always --dirty) docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH} docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} --build-arg MUJOCO_KEY=${MUJOCO_KEY} -t ${REGISTRY}/coach-mujoco_environment:${TAG} -f docker/Dockerfile.mujoco_environment . docker push ${REGISTRY}/coach-mujoco_environment:${TAG} no_output_timeout: 10m build_starcraft_env: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: Build and push starcraft environment container command: | REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com TAG=$(git describe --tags --always --dirty) docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH} docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} -t ${REGISTRY}/coach-starcraft_environment:${TAG} -f docker/Dockerfile.starcraft_environment . docker push ${REGISTRY}/coach-starcraft_environment:${TAG} no_output_timeout: 10m unit_tests: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run unit tests command: | python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn unit-test -tc 'make unit_tests_without_docker' -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 - run: name: cleanup command: | kubectl delete --all pods --namespace=unit-test-${CIRCLE_BUILD_NUM} || true kubectl delete ns unit-test-${CIRCLE_BUILD_NUM} || true when: always integration_tests: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run integration tests command: | python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn integration-test -tc 'make integration_tests_without_docker' -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=integration-test-${CIRCLE_BUILD_NUM} || true kubectl delete ns integration-test-${CIRCLE_BUILD_NUM} || true when: always functional_tests: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run functional tests command: | export FUNCTIONAL_PRESETS="'not mujoco and not CARLA and not doom'" python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test -tc "export FUNCTIONAL_PRESETS=${FUNCTIONAL_PRESETS} && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=functional-test-${CIRCLE_BUILD_NUM} || true kubectl delete ns functional-test-${CIRCLE_BUILD_NUM} || true when: always golden_test_gym: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run gym related golden tests command: | export GOLDEN_PRESETS="'CartPole or BitFlip'" python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-gym -tc "export GOLDEN_PRESETS=${GOLDEN_PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=golden-test-gym-${CIRCLE_BUILD_NUM} || true kubectl delete ns golden-test-gym-${CIRCLE_BUILD_NUM} || true when: always golden_test_doom: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run doom related golden tests command: | export GOLDEN_PRESETS="Doom" python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-doom -tc "export GOLDEN_PRESETS=${GOLDEN_PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=golden-test-doom-${CIRCLE_BUILD_NUM} || true kubectl delete ns golden-test-doom-${CIRCLE_BUILD_NUM} || true when: always golden_test_mujoco: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run mujoco related golden tests command: | export GOLDEN_PRESETS="Mujoco" python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-mujoco -tc "export GOLDEN_PRESETS=${GOLDEN_PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 45m - run: name: cleanup command: | kubectl delete --all pods --namespace=golden-test-mujoco-${CIRCLE_BUILD_NUM} || true kubectl delete ns golden-test-mujoco-${CIRCLE_BUILD_NUM} || true when: always golden_test_starcraft: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run starcraft related golden tests command: | export GOLDEN_PRESETS="Starcraft" python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-starcraft -tc "export GOLDEN_PRESETS=${GOLDEN_PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-starcraft_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=golden-test-starcraft-${CIRCLE_BUILD_NUM} || true kubectl delete ns golden-test-starcraft-${CIRCLE_BUILD_NUM} || true when: always functional_test_mujoco: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run mujoco related functional tests command: | export FUNCTIONAL_PRESETS="Mujoco" python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test-mujoco -tc "export FUNCTIONAL_PRESETS=${FUNCTIONAL_PRESETS} && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 45m - run: name: cleanup command: | kubectl delete --all pods --namespace=functional-test-mujoco-${CIRCLE_BUILD_NUM} || true kubectl delete ns functional-test-mujoco-${CIRCLE_BUILD_NUM} || true when: always functional_test_doom: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run doom related functional tests command: | export FUNCTIONAL_PRESETS="Doom" python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test-doom -tc "export FUNCTIONAL_PRESETS=${FUNCTIONAL_PRESETS} && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=functional-test-doom-${CIRCLE_BUILD_NUM} || true kubectl delete ns functional-test-doom-${CIRCLE_BUILD_NUM} || true when: always functional_test_Starcraft: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run Starcraft related functional tests command: | export FUNCTIONAL_PRESETS="Starcraft" python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn functional-test-starcraft -tc "export FUNCTIONAL_PRESETS='${FUNCTIONAL_PRESETS}' && make functional_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-starcraft_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=functional-test-starcraft-${CIRCLE_BUILD_NUM} || true kubectl delete ns functional-test-starcraft-${CIRCLE_BUILD_NUM} || true when: always trace_test_gym: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run gym related trace tests command: | export TRACE_PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL' python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-gym -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=trace-test-gym-${CIRCLE_BUILD_NUM} || true kubectl delete ns trace-test-gym-${CIRCLE_BUILD_NUM} || true when: always trace_test_doom: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run doom related trace tests command: | export TRACE_PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP' python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-doom -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=trace-test-doom-${CIRCLE_BUILD_NUM} || true kubectl delete ns trace-test-doom-${CIRCLE_BUILD_NUM} || true when: always trace_test_mujoco: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run mujoco related trace tests command: | export TRACE_PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG' python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-mujoco -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 45m - run: name: cleanup command: | kubectl delete --all pods --namespace=trace-test-mujoco-${CIRCLE_BUILD_NUM} || true kubectl delete ns trace-test-mujoco-${CIRCLE_BUILD_NUM} || true when: always trace_test_starcraft: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run starcraft related trace tests command: | export TRACE_PRESETS='Starcraft_CollectMinerals_Arc,Starcraft_CollectMinerals_Deuling_DDQN' python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-starcraft -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-starcraft_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 no_output_timeout: 30m - run: name: cleanup command: | kubectl delete --all pods --namespace=trace-test-starcraft-${CIRCLE_BUILD_NUM} || true kubectl delete ns trace-test-starcraft-${CIRCLE_BUILD_NUM} || true when: always multinode_test: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: run multinode test command: | aws s3 mb s3://coach-aws-test-${CIRCLE_BUILD_NUM} kubectl create ns multinode-test-${CIRCLE_BUILD_NUM} docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) \ /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \ && chmod a+x /usr/local/bin/aws-iam-authenticator \ && aws eks update-kubeconfig --name coach-aws-cicd \ && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \ && chmod a+x /usr/local/bin/kubectl \ && kubectl config set-context $(kubectl config current-context) --namespace=multinode-test-${CIRCLE_BUILD_NUM} \ && python3 rl_coach/tests/test_dist_coach.py -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:${TAG} -b coach-aws-test-${CIRCLE_BUILD_NUM}' docker ps -a -q | head -n 1 | xargs -I% docker cp %:/root/src/experiments . || true no_output_timeout: 30m - store_artifacts: path: ~/repo/experiments - run: name: cleanup command: | kubectl delete --all pods --namespace=golden-test-mujoco-${CIRCLE_BUILD_NUM} || true kubectl delete ns golden-test-mujoco-${CIRCLE_BUILD_NUM} || true aws s3 rm --recursive s3://coach-aws-test-${CIRCLE_BUILD_NUM} || true aws s3 rb s3://coach-aws-test-${CIRCLE_BUILD_NUM} || true kubectl delete ns multinode-test-${CIRCLE_BUILD_NUM} || true when: always container_deploy: <<: *executor_prep steps: - checkout - *remote_docker - *restore_cache - *aws_prep - run: name: Tag and push updated base and main container command: | REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com TAG=$(git describe --tags --always --dirty) docker pull ${REGISTRY}/coach-base:${TAG} docker tag ${REGISTRY}/coach-base:${TAG} ${REGISTRY}/coach-base:${MASTER_BRANCH} docker push ${REGISTRY}/coach-base:${MASTER_BRANCH} docker pull ${REGISTRY}/coach:builder-${TAG} docker tag ${REGISTRY}/coach:builder-${TAG} ${REGISTRY}/coach:builder-${MASTER_BRANCH} docker push ${REGISTRY}/coach:builder-${MASTER_BRANCH} docker pull ${REGISTRY}/coach:${TAG} docker tag ${REGISTRY}/coach:${TAG} ${REGISTRY}/coach:${MASTER_BRANCH} docker push ${REGISTRY}/coach:${MASTER_BRANCH} workflows: version: 2 build_and_test: jobs: - build_base - unit_tests: requires: - build_base - integration_tests: requires: - build_base - multinode_approval: type: approval requires: - build_base - multinode_test: requires: - multinode_approval - e2e_approval: type: approval requires: - build_base - build_gym_env: requires: - e2e_approval - build_doom_env: requires: - e2e_approval - build_mujoco_env: requires: - e2e_approval - build_starcraft_env: requires: - e2e_approval - gym_approval: type: approval requires: - golden_test_gym - doom_approval: type: approval requires: - golden_test_doom - mujoco_approval: type: approval requires: - golden_test_mujoco - starcraft_approval: type: approval requires: - golden_test_starcraft - golden_test_gym: requires: - build_gym_env - golden_test_doom: requires: - build_doom_env - golden_test_mujoco: requires: - build_mujoco_env - golden_test_starcraft: requires: - build_starcraft_env - trace_test_gym: requires: - gym_approval - trace_test_doom: requires: - doom_approval - trace_test_mujoco: requires: - mujoco_approval - trace_test_starcraft: requires: - starcraft_approval - container_deploy: requires: - unit_tests - integration_tests filters: branches: only: master nightly: triggers: - schedule: cron: "0 1 * * *" filters: branches: only: - master jobs: - unit_tests - integration_tests - functional_tests - build_gym_env - build_doom_env - build_mujoco_env # Starcraft disabled from nightly because tests are un-stabled #- build_starcraft_env - functional_test_doom: requires: - build_doom_env - functional_test_mujoco: requires: - build_mujoco_env - functional_test_doom # Starcraft disabled from nightly because tests are un-stabled #- functional_test_starcraft: # requires: # - build_starcraft_env # - functional_test_mujoco - golden_test_gym: requires: - build_gym_env - golden_test_doom: requires: - build_doom_env - golden_test_gym - golden_test_mujoco: requires: - build_mujoco_env - golden_test_doom # Starcraft disabled from nightly because tests are un-stabled #- golden_test_starcraft: # requires: # - build_starcraft_env # - golden_test_mujoco # - trace_test_gym: # requires: # - build_gym_env # - trace_test_doom: # requires: # - build_doom_env # - trace_test_gym # - trace_test_mujoco: # requires: # - build_mujoco_env # - trace_test_doom # - trace_test_starcraft: # requires: # - build_mujoco_env # - trace_test_mujoco