diff --git a/rl_coach/coach.py b/rl_coach/coach.py
index b5c23a2..2542eb4 100644
--- a/rl_coach/coach.py
+++ b/rl_coach/coach.py
@@ -81,6 +81,7 @@ def start_graph(graph_manager: 'GraphManager', task_parameters: 'TaskParameters'
         graph_manager.evaluate(EnvironmentSteps(sys.maxsize))
     else:
         graph_manager.improve()
+    graph_manager.close()
 
 
 def handle_distributed_coach_tasks(graph_manager, args):
diff --git a/rl_coach/environments/environment.py b/rl_coach/environments/environment.py
index 941224d..e29795e 100644
--- a/rl_coach/environments/environment.py
+++ b/rl_coach/environments/environment.py
@@ -492,3 +492,11 @@ class Environment(EnvironmentInterface):
 
     def get_target_success_rate(self) -> float:
         return self.target_success_rate
+
+    def close(self) -> None:
+        """
+        Clean up steps.
+
+        :return: None
+        """
+        pass
diff --git a/rl_coach/environments/gym_environment.py b/rl_coach/environments/gym_environment.py
index adf096d..a3407bf 100644
--- a/rl_coach/environments/gym_environment.py
+++ b/rl_coach/environments/gym_environment.py
@@ -515,3 +515,11 @@ class GymEnvironment(Environment):
 
     def get_target_success_rate(self) -> float:
         return self.target_success_rate
+
+    def close(self) -> None:
+        """
+        Clean up to close rendering windows.
+
+        :return: None
+        """
+        self.env.close()
diff --git a/rl_coach/graph_managers/graph_manager.py b/rl_coach/graph_managers/graph_manager.py
index 1166be3..180681d 100644
--- a/rl_coach/graph_managers/graph_manager.py
+++ b/rl_coach/graph_managers/graph_manager.py
@@ -687,3 +687,12 @@ class GraphManager(object):
             return self.data_store
 
         return data_store_creator(param)
+
+    def close(self) -> None:
+        """
+        Clean up to close environments.
+
+        :return: None
+        """
+        for env in self.environments:
+            env.close()