diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ea6e487 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,84 @@ +FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 + +# https://github.com/NVIDIA/nvidia-docker/issues/619 +RUN rm /etc/apt/sources.list.d/cuda.list + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get clean autoclean && \ + apt-get autoremove -y + +RUN apt-get update && \ + apt-get install -y python-pip && \ + apt-get clean autoclean && \ + apt-get autoremove -y +RUN pip install pip --upgrade +WORKDIR /root + +# update apt-get +RUN apt-get update && \ + apt-get upgrade -y + +################################ +# Install apt-get Requirements # +################################ + +# General +RUN apt-get install python3-pip cmake zlib1g-dev python3-tk python-opencv -y + +# Boost libraries +RUN apt-get install libboost-all-dev -y + +# Scipy requirements +RUN apt-get install libblas-dev liblapack-dev libatlas-base-dev gfortran -y + +# Pygame requirements +RUN apt-get install libsdl-dev libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev -y +RUN apt-get install libsmpeg-dev libportmidi-dev libavformat-dev libswscale-dev -y + +# Dashboard +RUN apt-get install dpkg-dev build-essential python3.5-dev libjpeg-dev libtiff-dev libsdl1.2-dev libnotify-dev \ + freeglut3 freeglut3-dev libsm-dev libgtk2.0-dev libgtk-3-dev libwebkitgtk-dev libgtk-3-dev \ + libwebkitgtk-3.0-dev libgstreamer-plugins-base1.0-dev -y + +# Gym +RUN apt-get install libav-tools libsdl2-dev swig cmake -y + +# Mujoco_py +RUN apt-get install curl libgl1-mesa-dev libgl1-mesa-glx libglew-dev libosmesa6-dev software-properties-common -y + +# ViZDoom +RUN apt-get install build-essential zlib1g-dev libsdl2-dev libjpeg-dev \ + nasm tar libbz2-dev libgtk2.0-dev cmake git libfluidsynth-dev libgme-dev \ + libopenal-dev timidity libwildmidi-dev unzip wget -y + +# cleanup apt-get +RUN apt-get clean autoclean && \ + apt-get autoremove -y + +############################ +# Install Pip Requirements # +############################ + +RUN pip3 install --upgrade pip + +RUN pip3 install pytest + +# initial installation of coach, so that the docker build won't install everything from scratch +RUN pip3 install rl_coach>=0.10.0 + +# install additional environments +RUN pip3 install gym[atari]==0.10.5 +RUN pip3 install mujoco_py==1.50.1.56 +RUN pip3 install vizdoom==1.1.6 + +COPY . /root/src +WORKDIR /root/src + +RUN pip3 install -e . + +RUN curl -o /usr/local/bin/patchelf https://s3-us-west-2.amazonaws.com/openai-sci-artifacts/manual-builds/patchelf_0.9_amd64.elf \ + && chmod +x /usr/local/bin/patchelf + +RUN chmod 777 /root/src/docker_entrypoint +ENTRYPOINT ["/root/src/docker_entrypoint"] diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..5e7bbd6 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,125 @@ +/* +* NervanaSystems/private-coach Jenkinsfile +*/ + +// Constant Variables for Build +// To define the node label - jenkins -> manage jenkins -> manage nodes -> select requested node -> labels +static final String nodeLabel = 'gpu' +static final String customWorkspace = '/state/ws' +static final Map slackColorMap = [ + 'FAILURE': 'danger', + 'UNSTABLE': 'warning', + 'SUCCESS': 'good' +] +static final Map ansiColorMap = [ + 'FAILURE': '\u001B[31m', + 'WARNING': '\u001B[33m', + 'SUCCESS': '\u001B[32m', + 'END': '\u001B[0m' +] + +// Common Closures for Build +def slackStartMessage = { + try { + slackSend message: "Build ${env.JOB_NAME} started (<${env.BUILD_URL}|LINK>)" + } catch (err) { + echo "BUILD WARNING - Failed to send Slack Message: ${err}" + } +} +def slackEndMessage = { + try { + slackSend color: slackColorMap[currentBuild.currentResult], message: "Build ${env.JOB_NAME} finished with result: ${currentBuild.currentResult} (<${env.BUILD_URL}|LINK>)" + } catch (err) { + echo "BUILD WARNING - Failed to send Slack Message: ${err}" + } +} + + +// Wrap entire build with timestamps to improve Jenkins console log readability +timestamps { + node(nodeLabel) { + + // Send Slack start message + slackStartMessage() + + try { + ansiColor('xterm') { + // Clear previous workspace + deleteDir() + + stage('Checkout') { + // Clone repo at triggered commit + checkout scm + } + + stage('Build') { + // Build docker image + sh 'docker build -t coach --build-arg http_proxy="http://proxy-chain.intel.com:911" --build-arg https_proxy="http://proxy-chain.intel.com:912" -f Dockerfile .' + } + + stage('Test') { + // Unit tests - short and contained functionality tests that take up to 1 minute. + stage('Unit Tests') { + try { + sh 'docker run\ + -e http_proxy="http://proxy-chain.intel.com:911"\ + -e https_proxy="http://proxy-chain.intel.com:912"\ + -e MUJOCO_KEY=\$MUJOCO_KEY coach pytest rl_coach/tests -m unit_test' + } catch (err) { + echo "${ansiColorMap['FAILURE']} BUILD FAILURE - Caught Exception: ${err} ${ansiColorMap['END']}" + currentBuild.result = 'FAILURE' + } + } + + // Integration tests - long functionality tests which can take up to 1 hour. + //stage('Integration Tests') { + // try { + // sh 'docker run\ + // -e http_proxy="http://proxy-chain.intel.com:911"\ + // -e https_proxy="http://proxy-chain.intel.com:912"\ + // -e MUJOCO_KEY=\$MUJOCO_KEY coach pytest rl_coach/tests -m integration_test -s' + // } catch (err) { + // echo "${ansiColorMap['FAILURE']} BUILD FAILURE - Caught Exception: ${err} ${ansiColorMap['END']}" + // currentBuild.result = 'FAILURE' + // } + //} + + // Trace tests - long tests which test for equality to known output + stage('Trace Tests') { + try { + sh 'docker run\ + -e http_proxy="http://proxy-chain.intel.com:911"\ + -e https_proxy="http://proxy-chain.intel.com:912"\ + -e MUJOCO_KEY=\$MUJOCO_KEY coach python3 rl_coach/tests/trace_tests.py -prl\ + -ip Doom_Basic_BC,MontezumaRevenge_BC,Carla_3_Cameras_DDPG,Carla_DDPG,Carla_Dueling_DDQN,Starcraft_CollectMinerals_A3C,Starcraft_CollectMinerals_Dueling_DDQN' + } catch (err) { + echo "${ansiColorMap['FAILURE']} BUILD FAILURE - Caught Exception: ${err} ${ansiColorMap['END']}" + currentBuild.result = 'FAILURE' + } + } + + // Golden tests - long tests which test for performance in terms of score and sample efficiency + stage('Golden Tests') { + try { + sh 'docker run\ + -e http_proxy="http://proxy-chain.intel.com:911"\ + -e https_proxy="http://proxy-chain.intel.com:912"\ + -e MUJOCO_KEY=\$MUJOCO_KEY coach python3 rl_coach/tests/golden_tests.py -np' + } catch (err) { + echo "${ansiColorMap['FAILURE']} BUILD FAILURE - Caught Exception: ${err} ${ansiColorMap['END']}" + currentBuild.result = 'FAILURE' + } + } + + // TODO: run PEP8 style checks? + } + } + } catch (err) { + fail(${err}) + } + + // Send Slack end message + slackEndMessage() + + } +} \ No newline at end of file diff --git a/docker_entrypoint b/docker_entrypoint new file mode 100644 index 0000000..7112943 --- /dev/null +++ b/docker_entrypoint @@ -0,0 +1,21 @@ +#!/bin/sh +set -e + +# download mjpro150 +mkdir /root/.mujoco +cd /root/.mujoco +wget https://www.roboti.us/download/mjpro150_linux.zip +unzip mjpro150_linux.zip + +# copy the mujoco license key into the container +echo $MUJOCO_KEY | base64 --decode > /root/.mujoco/mjkey.txt +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/root/.mujoco/mjpro150/bin + +git clone https://github.com/deepmind/dm_control.git +pip3 install ./dm_control + +export VIZDOOM_ROOT=`pip show vizdoom 2>/dev/null | awk '/Location/{print $2}'`/vizdoom + +cd /root/src/ + +exec "$@"