parameter noise exploration - using Noisy Nets

rl_coach/agents/actor_critic_agent.py

@@ -24,11 +24,12 @@ from rl_coach.architectures.tensorflow_components.heads.policy_head import Polic
 from rl_coach.architectures.tensorflow_components.heads.v_head import VHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
 from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, \
-    AgentParameters, InputEmbedderParameters
+    AgentParameters
 from rl_coach.logger import screen
 from rl_coach.memories.episodic.single_episode_buffer import SingleEpisodeBufferParameters
 from rl_coach.spaces import DiscreteActionSpace
 from rl_coach.utils import last_sample
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
 
 
 class ActorCriticAlgorithmParameters(AlgorithmParameters):

rl_coach/agents/bc_agent.py

@@ -21,10 +21,11 @@ import numpy as np
 from rl_coach.agents.imitation_agent import ImitationAgent
 from rl_coach.architectures.tensorflow_components.heads.policy_head import PolicyHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
-from rl_coach.base_parameters import AgentParameters, AlgorithmParameters, NetworkParameters, InputEmbedderParameters, \
+from rl_coach.base_parameters import AgentParameters, AlgorithmParameters, NetworkParameters, \
     MiddlewareScheme
 from rl_coach.exploration_policies.e_greedy import EGreedyParameters
 from rl_coach.memories.episodic.episodic_experience_replay import EpisodicExperienceReplayParameters
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
 
 
 class BCAlgorithmParameters(AlgorithmParameters):

rl_coach/agents/bootstrapped_dqn_agent.py

@@ -18,9 +18,11 @@ from typing import Union
 
 import numpy as np
 
-from rl_coach.agents.dqn_agent import DQNAgentParameters, DQNNetworkParameters
+from rl_coach.agents.dqn_agent import DQNNetworkParameters, DQNAlgorithmParameters
 from rl_coach.agents.value_optimization_agent import ValueOptimizationAgent
+from rl_coach.base_parameters import AgentParameters
 from rl_coach.exploration_policies.bootstrapped import BootstrappedParameters
+from rl_coach.memories.non_episodic.experience_replay import ExperienceReplayParameters
 
 
 class BootstrappedDQNNetworkParameters(DQNNetworkParameters):
@@ -30,11 +32,12 @@ class BootstrappedDQNNetworkParameters(DQNNetworkParameters):
         self.rescale_gradient_from_head_by_factor = [1.0/self.num_output_head_copies]*self.num_output_head_copies
 
 
-class BootstrappedDQNAgentParameters(DQNAgentParameters):
+class BootstrappedDQNAgentParameters(AgentParameters):
     def __init__(self):
-        super().__init__()
-        self.network_wrappers = {"main": BootstrappedDQNNetworkParameters()}
-        self.exploration = BootstrappedParameters()
+        super().__init__(algorithm=DQNAlgorithmParameters(),
+                         exploration=BootstrappedParameters(),
+                         memory=ExperienceReplayParameters(),
+                         networks={"main": BootstrappedDQNNetworkParameters()})
 
     @property
     def path(self):

rl_coach/agents/categorical_dqn_agent.py

@@ -18,7 +18,7 @@ from typing import Union
 
 import numpy as np
 
-from rl_coach.agents.dqn_agent import DQNNetworkParameters, DQNAlgorithmParameters
+from rl_coach.agents.dqn_agent import DQNNetworkParameters, DQNAlgorithmParameters, DQNAgentParameters
 from rl_coach.agents.value_optimization_agent import ValueOptimizationAgent
 from rl_coach.architectures.tensorflow_components.heads.categorical_q_head import CategoricalQHeadParameters
 from rl_coach.base_parameters import AgentParameters
@@ -49,12 +49,12 @@ class CategoricalDQNExplorationParameters(EGreedyParameters):
         self.evaluation_epsilon = 0.001
 
 
-class CategoricalDQNAgentParameters(AgentParameters):
+class CategoricalDQNAgentParameters(DQNAgentParameters):
     def __init__(self):
-        super().__init__(algorithm=CategoricalDQNAlgorithmParameters(),
-                         exploration=CategoricalDQNExplorationParameters(),
-                         memory=ExperienceReplayParameters(),
-                         networks={"main": CategoricalDQNNetworkParameters()})
+        super().__init__()
+        self.algorithm = CategoricalDQNAlgorithmParameters()
+        self.exploration = CategoricalDQNExplorationParameters()
+        self.network_wrappers = {"main": CategoricalDQNNetworkParameters()}
 
     @property
     def path(self):

rl_coach/agents/clipped_ppo_agent.py

@@ -27,7 +27,8 @@ from rl_coach.architectures.tensorflow_components.heads.ppo_head import PPOHeadP
 from rl_coach.architectures.tensorflow_components.heads.v_head import VHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
 from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, \
-    AgentParameters, InputEmbedderParameters
+    AgentParameters
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
 from rl_coach.core_types import EnvironmentSteps, Batch, EnvResponse, StateType
 from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters
 from rl_coach.logger import screen

rl_coach/agents/ddpg_agent.py

@@ -22,11 +22,12 @@ import numpy as np
 
 from rl_coach.agents.actor_critic_agent import ActorCriticAgent
 from rl_coach.agents.agent import Agent
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
 from rl_coach.architectures.tensorflow_components.heads.ddpg_actor_head import DDPGActorHeadParameters
 from rl_coach.architectures.tensorflow_components.heads.v_head import VHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
 from rl_coach.base_parameters import NetworkParameters, AlgorithmParameters, \
-    AgentParameters, InputEmbedderParameters, EmbedderScheme
+    AgentParameters, EmbedderScheme
 from rl_coach.core_types import ActionInfo, EnvironmentSteps
 from rl_coach.exploration_policies.ou_process import OUProcessParameters
 from rl_coach.memories.episodic.episodic_experience_replay import EpisodicExperienceReplayParameters

rl_coach/agents/dfp_agent.py

@@ -26,8 +26,9 @@ from rl_coach.architectures.tensorflow_components.heads.measurements_prediction_
     MeasurementsPredictionHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
 from rl_coach.base_parameters import AlgorithmParameters, AgentParameters, NetworkParameters, \
-    InputEmbedderParameters, MiddlewareScheme
+     MiddlewareScheme
 from rl_coach.core_types import ActionInfo, EnvironmentSteps, RunPhase
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
 from rl_coach.exploration_policies.e_greedy import EGreedyParameters
 from rl_coach.memories.episodic.episodic_experience_replay import EpisodicExperienceReplayParameters
 from rl_coach.memories.memory import MemoryGranularity

rl_coach/agents/dqn_agent.py

@@ -22,7 +22,8 @@ from rl_coach.agents.value_optimization_agent import ValueOptimizationAgent
 from rl_coach.architectures.tensorflow_components.heads.q_head import QHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
 from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, AgentParameters, \
-    InputEmbedderParameters, MiddlewareScheme
+    MiddlewareScheme
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
 from rl_coach.core_types import EnvironmentSteps
 from rl_coach.exploration_policies.e_greedy import EGreedyParameters
 from rl_coach.memories.non_episodic.experience_replay import ExperienceReplayParameters

rl_coach/agents/human_agent.py

@@ -25,8 +25,9 @@ from rl_coach.agents.agent import Agent
 from rl_coach.agents.bc_agent import BCNetworkParameters
 from rl_coach.architectures.tensorflow_components.heads.policy_head import PolicyHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
-from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, InputEmbedderParameters, EmbedderScheme, \
+from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, EmbedderScheme, \
     AgentParameters
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
 from rl_coach.core_types import ActionInfo
 from rl_coach.exploration_policies.e_greedy import EGreedyParameters
 from rl_coach.logger import screen

rl_coach/agents/n_step_q_agent.py

@@ -22,8 +22,9 @@ from rl_coach.agents.policy_optimization_agent import PolicyOptimizationAgent
 from rl_coach.agents.value_optimization_agent import ValueOptimizationAgent
 from rl_coach.architectures.tensorflow_components.heads.q_head import QHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
-from rl_coach.base_parameters import AlgorithmParameters, AgentParameters, NetworkParameters, \
-    InputEmbedderParameters
+from rl_coach.base_parameters import AlgorithmParameters, AgentParameters, NetworkParameters
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
+
 from rl_coach.core_types import EnvironmentSteps
 from rl_coach.exploration_policies.e_greedy import EGreedyParameters
 from rl_coach.memories.episodic.single_episode_buffer import SingleEpisodeBufferParameters

rl_coach/agents/naf_agent.py

@@ -22,7 +22,9 @@ from rl_coach.agents.value_optimization_agent import ValueOptimizationAgent
 from rl_coach.architectures.tensorflow_components.heads.naf_head import NAFHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
 from rl_coach.base_parameters import AlgorithmParameters, AgentParameters, \
-    NetworkParameters, InputEmbedderParameters
+    NetworkParameters
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
+
 from rl_coach.core_types import ActionInfo, EnvironmentSteps
 from rl_coach.exploration_policies.ou_process import OUProcessParameters
 from rl_coach.memories.episodic.episodic_experience_replay import EpisodicExperienceReplayParameters

rl_coach/agents/nec_agent.py

@@ -23,8 +23,9 @@ import numpy as np
 from rl_coach.agents.value_optimization_agent import ValueOptimizationAgent
 from rl_coach.architectures.tensorflow_components.heads.dnd_q_head import DNDQHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
-from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, AgentParameters, \
-    InputEmbedderParameters
+from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, AgentParameters
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
+
 from rl_coach.core_types import RunPhase, EnvironmentSteps, Episode, StateType
 from rl_coach.exploration_policies.e_greedy import EGreedyParameters
 from rl_coach.logger import screen

rl_coach/agents/policy_gradients_agent.py

@@ -22,7 +22,9 @@ from rl_coach.agents.policy_optimization_agent import PolicyOptimizationAgent, P
 from rl_coach.architectures.tensorflow_components.heads.policy_head import PolicyHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
 from rl_coach.base_parameters import NetworkParameters, AlgorithmParameters, \
-    AgentParameters, InputEmbedderParameters
+    AgentParameters
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
+
 from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters
 from rl_coach.logger import screen
 from rl_coach.memories.episodic.single_episode_buffer import SingleEpisodeBufferParameters

rl_coach/agents/ppo_agent.py

@@ -26,7 +26,9 @@ from rl_coach.architectures.tensorflow_components.heads.ppo_head import PPOHeadP
 from rl_coach.architectures.tensorflow_components.heads.v_head import VHeadParameters
 from rl_coach.architectures.tensorflow_components.middlewares.fc_middleware import FCMiddlewareParameters
 from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, \
-    AgentParameters, InputEmbedderParameters, DistributedTaskParameters
+    AgentParameters, DistributedTaskParameters
+from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedderParameters
+
 from rl_coach.core_types import EnvironmentSteps, Batch
 from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters
 from rl_coach.logger import screen