parameter noise exploration - using Noisy Nets

rl_coach/architectures/tensorflow_components/embedders/embedder.py

@@ -19,11 +19,40 @@ from typing import List, Union
 import numpy as np
 import tensorflow as tf
 
-from rl_coach.architectures.tensorflow_components.architecture import batchnorm_activation_dropout
-from rl_coach.base_parameters import EmbedderScheme
+from rl_coach.architectures.tensorflow_components.architecture import batchnorm_activation_dropout, Dense
+from rl_coach.base_parameters import EmbedderScheme, NetworkComponentParameters
+
 from rl_coach.core_types import InputEmbedding
 
 
+class InputEmbedderParameters(NetworkComponentParameters):
+    def __init__(self, activation_function: str='relu', scheme: Union[List, EmbedderScheme]=EmbedderScheme.Medium,
+                 batchnorm: bool=False, dropout=False, name: str='embedder', input_rescaling=None, input_offset=None,
+                 input_clipping=None, dense_layer=Dense):
+        super().__init__(dense_layer=dense_layer)
+        self.activation_function = activation_function
+        self.scheme = scheme
+        self.batchnorm = batchnorm
+        self.dropout = dropout
+
+        if input_rescaling is None:
+            input_rescaling = {'image': 255.0, 'vector': 1.0}
+        if input_offset is None:
+            input_offset = {'image': 0.0, 'vector': 0.0}
+
+        self.input_rescaling = input_rescaling
+        self.input_offset = input_offset
+        self.input_clipping = input_clipping
+        self.name = name
+
+    @property
+    def path(self):
+        return {
+            "image": 'image_embedder:ImageEmbedder',
+            "vector": 'vector_embedder:VectorEmbedder'
+        }
+
+
 class InputEmbedder(object):
     """
     An input embedder is the first part of the network, which takes the input from the state and produces a vector
@@ -32,7 +61,7 @@ class InputEmbedder(object):
     """
     def __init__(self, input_size: List[int], activation_function=tf.nn.relu,
                  scheme: EmbedderScheme=None, batchnorm: bool=False, dropout: bool=False,
-                 name: str= "embedder", input_rescaling=1.0, input_offset=0.0, input_clipping=None):
+                 name: str= "embedder", input_rescaling=1.0, input_offset=0.0, input_clipping=None, dense_layer=Dense):
         self.name = name
         self.input_size = input_size
         self.activation_function = activation_function
@@ -47,6 +76,7 @@ class InputEmbedder(object):
         self.input_rescaling = input_rescaling
         self.input_offset = input_offset
         self.input_clipping = input_clipping
+        self.dense_layer = dense_layer
 
     def __call__(self, prev_input_placeholder=None):
         with tf.variable_scope(self.get_name()):

rl_coach/architectures/tensorflow_components/embedders/image_embedder.py

@@ -18,7 +18,7 @@ from typing import List
 
 import tensorflow as tf
 
-from rl_coach.architectures.tensorflow_components.architecture import Conv2d
+from rl_coach.architectures.tensorflow_components.architecture import Conv2d, Dense
 from rl_coach.architectures.tensorflow_components.embedders.embedder import InputEmbedder
 from rl_coach.base_parameters import EmbedderScheme
 from rl_coach.core_types import InputImageEmbedding
@@ -30,45 +30,49 @@ class ImageEmbedder(InputEmbedder):
     The embedder is intended for image like inputs, where the channels are expected to be the last axis.
     The embedder also allows custom rescaling of the input prior to the neural network.
     """
-    schemes = {
-        EmbedderScheme.Empty:
-            [],
-
-        EmbedderScheme.Shallow:
-            [
-                Conv2d([32, 3, 1])
-            ],
-
-        # atari dqn
-        EmbedderScheme.Medium:
-            [
-                Conv2d([32, 8, 4]),
-                Conv2d([64, 4, 2]),
-                Conv2d([64, 3, 1])
-            ],
-
-        # carla
-        EmbedderScheme.Deep: \
-            [
-                Conv2d([32, 5, 2]),
-                Conv2d([32, 3, 1]),
-                Conv2d([64, 3, 2]),
-                Conv2d([64, 3, 1]),
-                Conv2d([128, 3, 2]),
-                Conv2d([128, 3, 1]),
-                Conv2d([256, 3, 2]),
-                Conv2d([256, 3, 1])
-            ]
-    }
 
     def __init__(self, input_size: List[int], activation_function=tf.nn.relu,
                  scheme: EmbedderScheme=EmbedderScheme.Medium, batchnorm: bool=False, dropout: bool=False,
-                 name: str= "embedder", input_rescaling: float=255.0, input_offset: float=0.0, input_clipping=None):
+                 name: str= "embedder", input_rescaling: float=255.0, input_offset: float=0.0, input_clipping=None,
+                 dense_layer=Dense):
         super().__init__(input_size, activation_function, scheme, batchnorm, dropout, name, input_rescaling,
-                         input_offset, input_clipping)
+                         input_offset, input_clipping, dense_layer=dense_layer)
         self.return_type = InputImageEmbedding
         if len(input_size) != 3 and scheme != EmbedderScheme.Empty:
             raise ValueError("Image embedders expect the input size to have 3 dimensions. The given size is: {}"
                              .format(input_size))
 
+    @property
+    def schemes(self):
+        return {
+            EmbedderScheme.Empty:
+                [],
+
+            EmbedderScheme.Shallow:
+                [
+                    Conv2d([32, 3, 1])
+                ],
+
+            # atari dqn
+            EmbedderScheme.Medium:
+                [
+                    Conv2d([32, 8, 4]),
+                    Conv2d([64, 4, 2]),
+                    Conv2d([64, 3, 1])
+                ],
+
+            # carla
+            EmbedderScheme.Deep: \
+                [
+                    Conv2d([32, 5, 2]),
+                    Conv2d([32, 3, 1]),
+                    Conv2d([64, 3, 2]),
+                    Conv2d([64, 3, 1]),
+                    Conv2d([128, 3, 2]),
+                    Conv2d([128, 3, 1]),
+                    Conv2d([256, 3, 2]),
+                    Conv2d([256, 3, 1])
+                ]
+        }
+
 

rl_coach/architectures/tensorflow_components/embedders/vector_embedder.py

@@ -29,36 +29,40 @@ class VectorEmbedder(InputEmbedder):
     An input embedder that is intended for inputs that can be represented as vectors.
     The embedder flattens the input, applies several dense layers to it and returns the output.
     """
-    schemes = {
-        EmbedderScheme.Empty:
-            [],
-
-        EmbedderScheme.Shallow:
-            [
-                Dense([128])
-            ],
-
-        # dqn
-        EmbedderScheme.Medium:
-            [
-                Dense([256])
-            ],
-
-        # carla
-        EmbedderScheme.Deep: \
-            [
-                Dense([128]),
-                Dense([128]),
-                Dense([128])
-            ]
-    }
 
     def __init__(self, input_size: List[int], activation_function=tf.nn.relu,
                  scheme: EmbedderScheme=EmbedderScheme.Medium, batchnorm: bool=False, dropout: bool=False,
-                 name: str= "embedder", input_rescaling: float=1.0, input_offset:float=0.0, input_clipping=None):
+                 name: str= "embedder", input_rescaling: float=1.0, input_offset:float=0.0, input_clipping=None,
+                 dense_layer=Dense):
         super().__init__(input_size, activation_function, scheme, batchnorm, dropout, name,
-                         input_rescaling, input_offset, input_clipping)
+                         input_rescaling, input_offset, input_clipping, dense_layer=dense_layer)
 
         self.return_type = InputVectorEmbedding
         if len(self.input_size) != 1 and scheme != EmbedderScheme.Empty:
             raise ValueError("The input size of a vector embedder must contain only a single dimension")
+
+    @property
+    def schemes(self):
+        return {
+            EmbedderScheme.Empty:
+                [],
+
+            EmbedderScheme.Shallow:
+                [
+                    self.dense_layer([128])
+                ],
+
+            # dqn
+            EmbedderScheme.Medium:
+                [
+                    self.dense_layer([256])
+                ],
+
+            # carla
+            EmbedderScheme.Deep: \
+                [
+                    self.dense_layer([128]),
+                    self.dense_layer([128]),
+                    self.dense_layer([128])
+                ]
+        }