gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2025-12-18 03:30:19 +01:00
Code Issues Projects Releases Wiki Activity

CI related updates

Browse Source
This commit is contained in:
Scott Leishman
2018-11-27 21:53:46 +00:00
parent fc6604c09c
commit 3601d9bc45
9 changed files with 20 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
.circleci/config.yml
View File

@@ -23,8 +23,6 @@ aliases:
name: Prepare aws cli
command: |
sudo pip install awscli pytest kubernetes==8.0.0b1
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
$(aws ecr get-login --no-include-email --region us-west-2)
sudo curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator
@@ -32,6 +30,11 @@ aliases:
aws eks update-kubeconfig --name coach-aws-cicd
sudo curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
sudo chmod a+x /usr/local/bin/kubectl
- &docker_prep
run:
name: Prepare dockerfiles
command: |
aws s3 cp --recursive s3://coach-ci-dockerfiles docker
version: 2
jobs:
@@ -42,6 +45,7 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: Build and push base and main container
command: |
@@ -72,6 +76,7 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: Build and push gym environment container
command: |
@@ -90,6 +95,7 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: Build and push vizdoom environment container
command: |
@@ -108,6 +114,7 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: Build and push mujoco environment container
command: |
@@ -126,17 +133,14 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: run unit tests
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn unit-test -tc 'make unit_tests_without_docker' -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
- run:
name: cleanup
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
kubectl delete --all pods --namespace=unit-test-${CIRCLE_BUILD_NUM} || true
kubectl delete ns unit-test-${CIRCLE_BUILD_NUM} || true
when: always
@@ -148,17 +152,14 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: run integration tests
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn integration-test -tc 'make integration_tests_without_docker' -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
- run:
name: cleanup
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
kubectl delete --all pods --namespace=integration-test-${CIRCLE_BUILD_NUM} || true
kubectl delete ns integration-test-${CIRCLE_BUILD_NUM} || true
when: always
@@ -170,19 +171,16 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: run gym related golden tests
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
export PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL'
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-gym -tc "export PRESETS=${PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
no_output_timeout: 30m
- run:
name: cleanup
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
kubectl delete --all pods --namespace=golden-test-gym-${CIRCLE_BUILD_NUM} || true
kubectl delete ns golden-test-gym-${CIRCLE_BUILD_NUM} || true
when: always
@@ -194,19 +192,16 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: run doom related golden tests
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
export PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP'
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-doom -tc "export PRESETS=${PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
no_output_timeout: 30m
- run:
name: cleanup
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
kubectl delete --all pods --namespace=golden-test-doom-${CIRCLE_BUILD_NUM} || true
kubectl delete ns golden-test-doom-${CIRCLE_BUILD_NUM} || true
when: always
@@ -218,19 +213,16 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: run mujoco related golden tests
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
export PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG'
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn golden-test-mujoco -tc "export PRESETS=${PRESETS} && make golden_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
no_output_timeout: 30m
- run:
name: cleanup
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
kubectl delete --all pods --namespace=golden-test-mujoco-${CIRCLE_BUILD_NUM} || true
kubectl delete ns golden-test-mujoco-${CIRCLE_BUILD_NUM} || true
when: always
@@ -242,19 +234,16 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: run gym related trace tests
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
export PRESETS='CartPole_A3C,CartPole_Dueling_DDQN,CartPole_NStepQ,CartPole_DQN,CartPole_DFP,CartPole_PG,CartPole_NEC,CartPole_ClippedPPO,CartPole_PAL'
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-gym -tc "export PRESETS=${PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-gym_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
no_output_timeout: 30m
- run:
name: cleanup
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
kubectl delete --all pods --namespace=trace-test-gym-${CIRCLE_BUILD_NUM} || true
kubectl delete ns trace-test-gym-${CIRCLE_BUILD_NUM} || true
when: always
@@ -266,19 +255,16 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: run doom related trace tests
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
export PRESETS='Doom_Basic_DQN,Doom_Basic_A3C,Doom_Health_DFP'
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-doom -tc "export PRESETS=${PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-doom_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
no_output_timeout: 30m
- run:
name: cleanup
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
kubectl delete --all pods --namespace=trace-test-doom-${CIRCLE_BUILD_NUM} || true
kubectl delete ns trace-test-doom-${CIRCLE_BUILD_NUM} || true
when: always
@@ -290,19 +276,16 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: run mujoco related trace tests
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
export PRESETS='BitFlip_DQN_HER,BitFlip_DQN,Mujoco_A3C,Mujoco_A3C_LSTM,Mujoco_PPO,Mujoco_ClippedPPO,Mujoco_DDPG'
python3 rl_coach/tests/test_eks.py -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn trace-test-mujoco -tc "export PRESETS=${PRESETS} && make trace_tests_without_docker" -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach-mujoco_environment:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
no_output_timeout: 30m
- run:
name: cleanup
command: |
export AWS_ACCESS_KEY_ID=`echo ${AWS_ACCESS_KEY_ID} | base64 --decode`
export AWS_SECRET_ACCESS_KEY=`echo ${AWS_SECRET_ACCESS_KEY} | base64 --decode`
kubectl delete --all pods --namespace=trace-test-mujoco-${CIRCLE_BUILD_NUM} || true
kubectl delete ns trace-test-mujoco-${CIRCLE_BUILD_NUM} || true
when: always
@@ -314,6 +297,7 @@ jobs:
- *remote_docker
- *restore_cache
- *aws_prep
- *docker_prep
- run:
name: Tag and push updated base and main container
command: |
@@ -393,4 +377,4 @@ workflows:
- integration_tests
filters:
branches:
only: 0.11.0-release
only: master