gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2026-02-13 20:35:48 +01:00
Code Issues Projects Releases Wiki Activity

SAC algorithm (#282)

Browse Source 
* SAC algorithm

* SAC - updates to agent (learn_from_batch), sac_head and sac_q_head to fix problem in gradient calculation. Now SAC agents is able to train.
gym_environment - fixing an error in access to gym.spaces

* Soft Actor Critic - code cleanup

* code cleanup

* V-head initialization fix

* SAC benchmarks

* SAC Documentation

* typo fix

* documentation fixes

* documentation and version update

* README typo
This commit is contained in:
guyk1971
2019-05-01 18:37:49 +03:00
committed by shadiendrawis
parent 33dc29ee99
commit 74db141d5e
92 changed files with 2812 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
rl_coach/architectures/tensorflow_components/heads/__init__.py
View File

@@ -12,6 +12,8 @@ from .quantile_regression_q_head import QuantileRegressionQHead
from .rainbow_q_head import RainbowQHead
from .v_head import VHead
from .acer_policy_head import ACERPolicyHead
from .sac_head import SACPolicyHead
from .sac_q_head import SACQHead
from .classification_head import ClassificationHead
from .cil_head import RegressionHead
@@ -30,6 +32,8 @@ __all__ = [
'RainbowQHead',
'VHead',
'ACERPolicyHead',
'ClassificationHead'
'SACPolicyHead',
'SACQHead',
'ClassificationHead',
'RegressionHead'
]

107
rl_coach/architectures/tensorflow_components/heads/sac_head.py Normal file
View File

@@ -0,0 +1,107 @@
#
# Copyright (c) 2019 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import tensorflow as tf
from rl_coach.architectures.tensorflow_components.layers import Dense
from rl_coach.architectures.tensorflow_components.heads.head import Head
from rl_coach.base_parameters import AgentParameters
from rl_coach.core_types import ActionProbabilities
from rl_coach.spaces import SpacesDefinition
from rl_coach.utils import eps
LOG_SIG_CAP_MAX = 2
LOG_SIG_CAP_MIN = -20
class SACPolicyHead(Head):
def __init__(self, agent_parameters: AgentParameters, spaces: SpacesDefinition, network_name: str,
head_idx: int = 0, loss_weight: float = 1., is_local: bool = True, activation_function: str='relu',
squash: bool = True, dense_layer=Dense):
super().__init__(agent_parameters, spaces, network_name, head_idx, loss_weight, is_local, activation_function,
dense_layer=dense_layer)
self.name = 'sac_policy_head'
self.return_type = ActionProbabilities
self.num_actions = self.spaces.action.shape # continuous actions
self.squash = squash # squashing using tanh
def _build_module(self, input_layer):
self.given_raw_actions = tf.placeholder(tf.float32, [None, self.num_actions], name="actions")
self.input = [self.given_raw_actions]
self.output = []
# build the network
self._build_continuous_net(input_layer, self.spaces.action)
def _squash_correction(self,actions):
'''
correct squash operation (in case of bounded actions) according to appendix C in the paper.
NOTE : this correction assume the squash is done with tanh.
:param actions: unbounded actions
:return: the correction to be applied to the log_prob of the actions, assuming tanh squash
'''
if not self.squash:
return 0
return tf.reduce_sum(tf.log(1 - tf.tanh(actions) ** 2 + eps), axis=1)
def _build_continuous_net(self, input_layer, action_space):
num_actions = action_space.shape[0]
self.policy_mu_and_logsig = self.dense_layer(2*num_actions)(input_layer, name='policy_mu_logsig')
self.policy_mean = tf.identity(self.policy_mu_and_logsig[..., :num_actions], name='policy_mean')
self.policy_log_std = tf.clip_by_value(self.policy_mu_and_logsig[..., num_actions:],
LOG_SIG_CAP_MIN, LOG_SIG_CAP_MAX,name='policy_log_std')
self.output.append(self.policy_mean) # output[0]
self.output.append(self.policy_log_std) # output[1]
# define the distributions for the policy
# Tensorflow's multivariate normal distribution supports reparameterization
tfd = tf.contrib.distributions
self.policy_distribution = tfd.MultivariateNormalDiag(loc=self.policy_mean,
scale_diag=tf.exp(self.policy_log_std))
# define network outputs
# note that tensorflow supports reparametrization.
# i.e. policy_action_sample is a tensor through which gradients can flow
self.raw_actions = self.policy_distribution.sample()
if self.squash:
self.actions = tf.tanh(self.raw_actions)
# correct log_prob in case of squash (see appendix C in the paper)
squash_correction = self._squash_correction(self.raw_actions)
else:
self.actions = self.raw_actions
squash_correction = 0
# policy_action_logprob is a tensor through which gradients can flow
self.sampled_actions_logprob = self.policy_distribution.log_prob(self.raw_actions) - squash_correction
self.sampled_actions_logprob_mean = tf.reduce_mean(self.sampled_actions_logprob)
self.output.append(self.raw_actions) # output[2] : sampled raw action (before squash)
self.output.append(self.actions) # output[3] : squashed (if needed) version of sampled raw_actions
self.output.append(self.sampled_actions_logprob) # output[4]: log prob of sampled action (squash corrected)
self.output.append(self.sampled_actions_logprob_mean) # output[5]: mean of log prob of sampled actions (squash corrected)
def __str__(self):
result = [
"policy head:"
"\t\tDense (num outputs = 256)",
"\t\tDense (num outputs = 256)",
"\t\tDense (num outputs = {0})".format(2*self.num_actions),
"policy_mu = output[:num_actions], policy_std = output[num_actions:]"
]
return '\n'.join(result)

116
rl_coach/architectures/tensorflow_components/heads/sac_q_head.py Normal file
View File

@@ -0,0 +1,116 @@
#
# Copyright (c) 2019 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import tensorflow as tf
from rl_coach.architectures.tensorflow_components.layers import Dense
from rl_coach.architectures.tensorflow_components.heads.head import Head
from rl_coach.base_parameters import AgentParameters
from rl_coach.core_types import QActionStateValue
from rl_coach.spaces import SpacesDefinition, BoxActionSpace
class SACQHead(Head):
def __init__(self, agent_parameters: AgentParameters, spaces: SpacesDefinition, network_name: str,
head_idx: int = 0, loss_weight: float = 1., is_local: bool = True, activation_function: str='relu',
dense_layer=Dense):
super().__init__(agent_parameters, spaces, network_name, head_idx, loss_weight, is_local, activation_function,
dense_layer=dense_layer)
self.name = 'q_values_head'
if isinstance(self.spaces.action, BoxActionSpace):
self.num_actions = self.spaces.action.shape # continuous actions
else:
raise ValueError(
'SACQHead does not support action spaces of type: {class_name}'.format(
class_name=self.spaces.action.__class__.__name__,
)
)
self.return_type = QActionStateValue
# extract the topology from the SACQHeadParameters
self.network_layers_sizes = agent_parameters.network_wrappers['q'].heads_parameters[0].network_layers_sizes
def _build_module(self, input_layer):
# SAC Q network is basically 2 networks running in parallel on the same input (state , action)
# state is the observation fed through the input_layer, action is fed through placeholder to the header
# each is calculating q value : q1(s,a) and q2(s,a)
# the output of the head is min(q1,q2)
self.actions = tf.placeholder(tf.float32, [None, self.num_actions], name="actions")
self.target = tf.placeholder(tf.float32, [None, 1], name="q_targets")
self.input = [self.actions]
self.output = []
# Note (1) : in the author's implementation of sac (in rllab) they summarize the embedding of observation and
# action (broadcasting the bias) in the first layer of the network.
# build q1 network head
with tf.variable_scope("q1_head"):
layer_size = self.network_layers_sizes[0]
qi_obs_emb = self.dense_layer(layer_size)(input_layer, activation=self.activation_function)
qi_act_emb = self.dense_layer(layer_size)(self.actions, activation=self.activation_function)
qi_output = qi_obs_emb + qi_act_emb # merging the inputs by summarizing them (see Note (1))
for layer_size in self.network_layers_sizes[1:]:
qi_output = self.dense_layer(layer_size)(qi_output, activation=self.activation_function)
# the output layer
self.q1_output = self.dense_layer(1)(qi_output, name='q1_output')
# build q2 network head
with tf.variable_scope("q2_head"):
layer_size = self.network_layers_sizes[0]
qi_obs_emb = self.dense_layer(layer_size)(input_layer, activation=self.activation_function)
qi_act_emb = self.dense_layer(layer_size)(self.actions, activation=self.activation_function)
qi_output = qi_obs_emb + qi_act_emb # merging the inputs by summarizing them (see Note (1))
for layer_size in self.network_layers_sizes[1:]:
qi_output = self.dense_layer(layer_size)(qi_output, activation=self.activation_function)
# the output layer
self.q2_output = self.dense_layer(1)(qi_output, name='q2_output')
# take the minimum as the network's output. this is the log_target (in the original implementation)
self.q_output = tf.minimum(self.q1_output, self.q2_output, name='q_output')
# the policy gradients
# self.q_output_mean = tf.reduce_mean(self.q1_output) # option 1: use q1
self.q_output_mean = tf.reduce_mean(self.q_output) # option 2: use min(q1,q2)
self.output.append(self.q_output)
self.output.append(self.q_output_mean)
# defining the loss
self.q1_loss = 0.5*tf.reduce_mean(tf.square(self.q1_output - self.target))
self.q2_loss = 0.5*tf.reduce_mean(tf.square(self.q2_output - self.target))
# eventually both losses are depends on different parameters so we can sum them up
self.loss = self.q1_loss+self.q2_loss
tf.losses.add_loss(self.loss)
def __str__(self):
result = [
"q1 output"
"\t\tDense (num outputs = 256)",
"\t\tDense (num outputs = 256)",
"\t\tDense (num outputs = 1)",
"q2 output"
"\t\tDense (num outputs = 256)",
"\t\tDense (num outputs = 256)",
"\t\tDense (num outputs = 1)",
"min(Q1,Q2)"
]
return '\n'.join(result)

11
rl_coach/architectures/tensorflow_components/heads/v_head.py
View File

@@ -26,7 +26,7 @@ from rl_coach.spaces import SpacesDefinition
class VHead(Head):
def __init__(self, agent_parameters: AgentParameters, spaces: SpacesDefinition, network_name: str,
head_idx: int = 0, loss_weight: float = 1., is_local: bool = True, activation_function: str='relu',
dense_layer=Dense):
dense_layer=Dense, initializer='normalized_columns'):
super().__init__(agent_parameters, spaces, network_name, head_idx, loss_weight, is_local, activation_function,
dense_layer=dense_layer)
self.name = 'v_values_head'
@@ -37,10 +37,15 @@ class VHead(Head):
else:
self.loss_type = tf.losses.mean_squared_error
self.initializer = initializer
def _build_module(self, input_layer):
# Standard V Network
self.output = self.dense_layer(1)(input_layer, name='output',
kernel_initializer=normalized_columns_initializer(1.0))
if self.initializer == 'normalized_columns':
self.output = self.dense_layer(1)(input_layer, name='output',
kernel_initializer=normalized_columns_initializer(1.0))
elif self.initializer == 'xavier' or self.initializer is None:
self.output = self.dense_layer(1)(input_layer, name='output')
def __str__(self):
result = [