diff --git a/.circleci/config.yml b/.circleci/config.yml index 329de50..e578036 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -345,10 +345,18 @@ jobs: - run: name: run gym related trace tests command: | - aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive - export TRACE_PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL' - python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-gym -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 - no_output_timeout: 30m + kubectl create ns trace-test-gym-test-${CIRCLE_BUILD_NUM} + docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) \ + /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \ + && chmod a+x /usr/local/bin/aws-iam-authenticator \ + && aws eks update-kubeconfig --name coach-aws-cicd \ + && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \ + && chmod a+x /usr/local/bin/kubectl \ + && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-gym-test-${CIRCLE_BUILD_NUM} \ + && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \ + && TRACE_PRESETS="CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL" \ + && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:${TAG} -b coach-trace-gym-aws-test-${CIRCLE_BUILD_NUM}' + no_output_timeout: 50m - run: name: cleanup command: | @@ -366,10 +374,18 @@ jobs: - run: name: run doom related trace tests command: | - aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive - export TRACE_PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP' - python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-doom -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 - no_output_timeout: 30m + kubectl create ns trace-test-doom-test-${CIRCLE_BUILD_NUM} + docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) \ + /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \ + && chmod a+x /usr/local/bin/aws-iam-authenticator \ + && aws eks update-kubeconfig --name coach-aws-cicd \ + && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \ + && chmod a+x /usr/local/bin/kubectl \ + && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-doom-test-${CIRCLE_BUILD_NUM} \ + && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \ + && TRACE_PRESETS="Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP,Doom_Basic_Dueling_DDQN,Doom_Basic_DFP" \ + && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:${TAG} -b coach-trace-doom-aws-test-${CIRCLE_BUILD_NUM}' + no_output_timeout: 50m - run: name: cleanup command: | @@ -387,10 +403,18 @@ jobs: - run: name: run mujoco related trace tests command: | - aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive - export TRACE_PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG' - python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-mujoco -tc "export TRACE_PRESETS=${TRACE_PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096 - no_output_timeout: 45m + kubectl create ns trace-test-mujoco-test-${CIRCLE_BUILD_NUM} + docker run -e CIRCLE_BUILD_NUM=$CIRCLE_BUILD_NUM -e TAG=$(git describe --tags --always --dirty) -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) \ + /bin/bash -c 'pip install awscli; curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator \ + && chmod a+x /usr/local/bin/aws-iam-authenticator \ + && aws eks update-kubeconfig --name coach-aws-cicd \ + && curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \ + && chmod a+x /usr/local/bin/kubectl \ + && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-mujoco-test-${CIRCLE_BUILD_NUM} \ + && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \ + && TRACE_PRESETS="BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG" \ + && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:${TAG} -b coach-trace-mujoco-aws-test-${CIRCLE_BUILD_NUM}' + no_output_timeout: 50m - run: name: cleanup command: | @@ -421,7 +445,7 @@ jobs: && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -ow -ut -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:${TAG} -b coach-trace-gym-aws-test-${CIRCLE_BUILD_NUM}' docker ps -a -q | head -n 1 | xargs -I% docker cp %:/root/src/rl_coach/traces . || true aws s3 sync ~/repo/traces s3://coach-traces-aws || true - no_output_timeout: 30m + no_output_timeout: 50m - run: name: cleanup command: | @@ -448,11 +472,11 @@ jobs: && chmod a+x /usr/local/bin/kubectl \ && kubectl config set-context $(kubectl config current-context) --namespace=trace-test-doom-test-${CIRCLE_BUILD_NUM} \ && aws s3 cp s3://coach-traces-aws rl_coach/traces --recursive \ - && TRACE_PRESETS="Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP" \ + && TRACE_PRESETS="Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP,Doom_Basic_Dueling_DDQN,Doom_Basic_DFP" \ && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -ow -ut -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:${TAG} -b coach-trace-doom-aws-test-${CIRCLE_BUILD_NUM}' docker ps -a -q | head -n 1 | xargs -I% docker cp %:/root/src/rl_coach/traces . || true aws s3 sync ~/repo/traces s3://coach-traces-aws || true - no_output_timeout: 30m + no_output_timeout: 50m - run: name: cleanup command: | @@ -483,7 +507,7 @@ jobs: && python3 rl_coach/tests/trace_tests.py -p ${TRACE_PRESETS} -prl -ow -ut -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:${TAG} -b coach-trace-mujoco-aws-test-${CIRCLE_BUILD_NUM}' docker ps -a -q | head -n 1 | xargs -I% docker cp %:/root/src/rl_coach/traces . || true aws s3 sync ~/repo/traces s3://coach-traces-aws || true - no_output_timeout: 30m + no_output_timeout: 50m - run: name: cleanup command: | @@ -607,25 +631,6 @@ workflows: - build_mujoco_env: requires: - e2e_approval -# - build_starcraft_env: -# requires: -# - e2e_approval - - gym_approval: - type: approval - requires: - - golden_test_gym - - doom_approval: - type: approval - requires: - - golden_test_doom - - mujoco_approval: - type: approval - requires: - - golden_test_mujoco -# - starcraft_approval: -# type: approval -# requires: -# - golden_test_starcraft - golden_test_gym: requires: - build_gym_env @@ -638,27 +643,36 @@ workflows: # - golden_test_starcraft: # requires: # - build_starcraft_env + - update_gym_approval: + type: approval + requires: + - build_gym_env + - update_doom_approval: + type: approval + requires: + - build_doom_env + - update_mujoco_approval: + type: approval + requires: + - build_mujoco_env - update_gym_traces_test: requires: - - trace_test_gym - type: approval + - update_gym_approval - update_doom_traces_test: requires: - - trace_test_doom - type: approval + - update_doom_approval - update_mujoco_traces_test: requires: - - trace_test_mujoco - type: approval + - update_mujoco_approval - trace_test_gym: requires: - - gym_approval + - build_gym_env - trace_test_doom: requires: - - doom_approval + - build_doom_env - trace_test_mujoco: requires: - - mujoco_approval + - build_mujoco_env # - trace_test_starcraft: # requires: # - starcraft_approval diff --git a/rl_coach/tests/trace_tests.py b/rl_coach/tests/trace_tests.py index f235eb8..8e0fbc2 100644 --- a/rl_coach/tests/trace_tests.py +++ b/rl_coach/tests/trace_tests.py @@ -85,7 +85,7 @@ def run_trace_based_test(preset_name, num_env_steps, level=None): '-e {test_name} ' '--seed 42 ' '-c ' - '-dcp {template}' + '-dcp {template} ' '--no_summary ' '-cp {custom_param} ' '{level} ' @@ -313,6 +313,10 @@ def main(): else: screen.error(" Summary: " + str(test_count - fail_count) + "/" + str(test_count) + " tests passed successfully", crash=False) + # check fail counts just if update traces is not activated! + if not args.update_traces: + assert fail_count == 0 + if __name__ == '__main__': os.environ['DISABLE_MUJOCO_RENDERING'] = '1'