diff --git a/docker/Dockerfile b/docker/Dockerfile
index 420ac8f..6ac66d0 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,6 +2,7 @@ FROM coach-base:master as builder
 
 # prep some of the more common environments
 # Gym (installed with coach)
+Run pip3 install gym[atari] box2d
 # Mujoco
 RUN mkdir -p ~/.mujoco \
     && wget https://www.roboti.us/download/mjpro150_linux.zip -O mujoco.zip \
@@ -28,5 +29,5 @@ COPY --from=builder /root/.cache /root/.cache
 COPY setup.py /root/src/.
 COPY requirements.txt /root/src/.
 COPY README.md /root/src/.
-RUN pip3 install mujoco_py vizdoom && pip3 install -e .[all] && rm -rf /root/.cache
+RUN pip3 install gym[atari] box2d mujoco_py vizdoom && pip3 install -e .[all] && rm -rf /root/.cache
 COPY . /root/src
diff --git a/requirements.txt b/requirements.txt
index c818202..898283c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,8 +7,7 @@ pygame>=1.9.3
 PyOpenGL>=3.1.0
 scipy>=0.19.0
 scikit-image>=0.13.0
-box2d>=2.3.2
-gym[atari]>=0.10.5
+gym>=0.10.5
 bokeh>=0.13.0
 futures>=3.1.1
 wxPython>=4.0.1