# rl_coach.agents.soft_actor_critic_agent — Reinforcement Learning Coach 0.12.0 documentation

# Source code for rl_coach.agents.soft_actor_critic_agent

    #
    # Copyright (c) 2019 Intel Corporation
    #
    # Licensed under the Apache License, Version 2.0 (the "License");
    # you may not use this file except in compliance with the License.
    # You may obtain a copy of the License at
    #
    #      http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    #

    from typing import Union
    import copy
    import numpy as np
    from collections import OrderedDict

    from rl_coach.agents.agent import Agent
    from rl_coach.agents.policy_optimization_agent import PolicyOptimizationAgent

    from rl_coach.architectures.head_parameters import SACQHeadParameters,SACPolicyHeadParameters,VHeadParameters
    from rl_coach.architectures.middleware_parameters import FCMiddlewareParameters
    from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, AgentParameters, EmbedderScheme, MiddlewareScheme
    from rl_coach.core_types import ActionInfo, EnvironmentSteps, RunPhase
    from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters
    from rl_coach.memories.non_episodic.experience_replay import ExperienceReplayParameters
    from rl_coach.architectures.embedder_parameters import InputEmbedderParameters
    from rl_coach.spaces import BoxActionSpace

    # There are 3 networks in SAC implementation. All have the same topology but parameters are not shared.
    # The networks are:
    # 1. State Value Network - SACValueNetwork
    # 2. Soft Q Value Network - SACCriticNetwork
    # 3. Policy Network - SACPolicyNetwork - currently supporting only Gaussian Policy

    # 1. State Value Network - SACValueNetwork
    # this is the state value network in SAC.
    # The network is trained to predict (regression) the state value in the max-entropy settings
    # The objective to be minimized is given in equation (5) in the paper:
    #
    # J(psi)= E_(s~D)[0.5*(V_psi(s)-y(s))^2]
    # where y(s) = E_(a~pi)[Q_theta(s,a)-log(pi(a|s))]

    # Default parameters for value network:
    # topology :
    #   input embedder : EmbedderScheme.Medium (Dense(256)) , relu activation
    #   middleware : EmbedderScheme.Medium (Dense(256)) , relu activation

    class SACValueNetworkParameters(NetworkParameters):
        def __init__(self):
            super().__init__()
            self.input_embedders_parameters = {'observation': InputEmbedderParameters(activation_function='relu')}
            self.middleware_parameters = FCMiddlewareParameters(activation_function='relu')
            self.heads_parameters = [VHeadParameters(initializer='xavier')]
            self.rescale_gradient_from_head_by_factor = [1]
            self.optimizer_type = 'Adam'
            self.batch_size = 256
            self.async_training = False
            self.learning_rate = 0.0003     # 3e-4 see appendix D in the paper
            self.create_target_network = True   # tau is set in SoftActorCriticAlgorithmParameters.rate_for_copying_weights_to_target

    # 2. Soft Q Value Network - SACCriticNetwork
    # the whole network is built in the SACQHeadParameters. we use empty input embedder and middleware
    class SACCriticNetworkParameters(NetworkParameters):
        def __init__(self):
            super().__init__()
            self.input_embedders_parameters = {'observation': InputEmbedderParameters(scheme=EmbedderScheme.Empty)}
            self.middleware_parameters = FCMiddlewareParameters(scheme=MiddlewareScheme.Empty)
            self.heads_parameters = [SACQHeadParameters()]      # SACQHeadParameters includes the topology of the head
            self.rescale_gradient_from_head_by_factor = [1]
            self.optimizer_type = 'Adam'
            self.batch_size = 256
            self.async_training = False
            self.learning_rate = 0.0003
            self.create_target_network = False

    # 3. policy Network
    # Default parameters for policy network:
    # topology :
    #   input embedder : EmbedderScheme.Medium (Dense(256)) , relu activation
    #   middleware : EmbedderScheme = [Dense(256)] , relu activation --> scheme should be overridden in preset
    class SACPolicyNetworkParameters(NetworkParameters):
        def __init__(self):
            super().__init__()
            self.input_embedders_parameters = {'observation': InputEmbedderParameters(activation_function='relu')}
            self.middleware_parameters = FCMiddlewareParameters(activation_function='relu')
            self.heads_parameters = [SACPolicyHeadParameters()]
            self.rescale_gradient_from_head_by_factor = [1]
            self.optimizer_type = 'Adam'
            self.batch_size = 256
            self.async_training = False
            self.learning_rate = 0.0003
            self.create_target_network = False
            self.l2_regularization = 0      # weight decay regularization. not used in the original paper

    # Algorithm Parameters

    class SoftActorCriticAlgorithmParameters(AlgorithmParameters):
        """
        :param num_steps_between_copying_online_weights_to_target: (StepMethod)
            The number of steps between copying the online network weights to the target network weights.

        :param rate_for_copying_weights_to_target: (float)
            When copying the online network weights to the target network weights, a soft update will be used, which
            weight the new online network weights by rate_for_copying_weights_to_target. (Tau as defined in the paper)

        :param use_deterministic_for_evaluation: (bool)
            If True, during the evaluation phase, action are chosen deterministically according to the policy mean
            and not sampled from the policy distribution.
        """
        def __init__(self):
            super().__init__()
            self.num_steps_between_copying_online_weights_to_target = EnvironmentSteps(1)
            self.rate_for_copying_weights_to_target = 0.005
            self.use_deterministic_for_evaluation = True    # evaluate agent using deterministic policy (i.e. take the mean value)

    class SoftActorCriticAgentParameters(AgentParameters):
        def __init__(self):
            super().__init__(algorithm=SoftActorCriticAlgorithmParameters(),
                             exploration=AdditiveNoiseParameters(),
                             memory=ExperienceReplayParameters(),   # SAC doesnt use episodic related data
                             # network wrappers:
                             networks=OrderedDict([("policy", SACPolicyNetworkParameters()),
                                                   ("q", SACCriticNetworkParameters()),
                                                   ("v", SACValueNetworkParameters())]))

        @property
        def path(self):
            return 'rl_coach.agents.soft_actor_critic_agent:SoftActorCriticAgent'

    # Soft Actor Critic - https://arxiv.org/abs/1801.01290
    class SoftActorCriticAgent(PolicyOptimizationAgent):
        def __init__(self, agent_parameters, parent: Union['LevelManager', 'CompositeAgent']=None):
            super().__init__(agent_parameters, parent)
            self.last_gradient_update_step_idx = 0

            # register signals to track (in learn_from_batch)
            self.policy_means = self.register_signal('Policy_mu_avg')
            self.policy_logsig = self.register_signal('Policy_logsig')
            self.policy_logprob_sampled = self.register_signal('Policy_logp_sampled')
            self.policy_grads = self.register_signal('Policy_grads_sumabs')

            self.q1_values = self.register_signal("Q1")
            self.TD_err1 = self.register_signal("TD err1")
            self.q2_values = self.register_signal("Q2")
            self.TD_err2 = self.register_signal("TD err2")
            self.v_tgt_ns = self.register_signal('V_tgt_ns')
            self.v_onl_ys = self.register_signal('V_onl_ys')
            self.action_signal = self.register_signal("actions")

        def learn_from_batch(self, batch):
            #########################################
            # need to update the following networks:
            # 1. actor (policy)
            # 2. state value (v)
            # 3. critic (q1 and q2)
            # 4. target network - probably already handled by V

            #########################################
            # define the networks to be used

            # State Value Network
            value_network = self.networks['v']
            value_network_keys = self.ap.network_wrappers['v'].input_embedders_parameters.keys()

            # Critic Network
            q_network = self.networks['q'].online_network
            q_head = q_network.output_heads[0]
            q_network_keys = self.ap.network_wrappers['q'].input_embedders_parameters.keys()

            # Actor (policy) Network
            policy_network = self.networks['policy'].online_network
            policy_network_keys = self.ap.network_wrappers['policy'].input_embedders_parameters.keys()

            ##########################################
            # 1. updating the actor - according to (13) in the paper
            policy_inputs = copy.copy(batch.states(policy_network_keys))
            policy_results = policy_network.predict(policy_inputs)

            policy_mu, policy_std, sampled_raw_actions, sampled_actions, sampled_actions_logprob, \
            sampled_actions_logprob_mean = policy_results

            self.policy_means.add_sample(policy_mu)
            self.policy_logsig.add_sample(policy_std)
            self.policy_logprob_sampled.add_sample(sampled_actions_logprob_mean)

            # get the state-action values for the replayed states and their corresponding actions from the policy
            q_inputs = copy.copy(batch.states(q_network_keys))
            q_inputs['output_0_0'] = sampled_actions
            log_target = q_network.predict(q_inputs)[0].squeeze()

            # log internal q values
            q1_vals, q2_vals = q_network.predict(q_inputs, outputs=[q_head.q1_output, q_head.q2_output])
            self.q1_values.add_sample(q1_vals)
            self.q2_values.add_sample(q2_vals)

            # calculate the gradients according to (13)
            # get the gradients of log_prob w.r.t the weights (parameters) - indicated as phi in the paper
            initial_feed_dict = {policy_network.gradients_weights_ph[5]: np.array(1.0)}
            dlogp_dphi = policy_network.predict(policy_inputs,
                                                outputs=policy_network.weighted_gradients[5],
                                                initial_feed_dict=initial_feed_dict)

            # calculate dq_da
            dq_da = q_network.predict(q_inputs,
                                      outputs=q_network.gradients_wrt_inputs[1]['output_0_0'])

            # calculate da_dphi
            initial_feed_dict = {policy_network.gradients_weights_ph[3]: dq_da}
            dq_dphi = policy_network.predict(policy_inputs,
                                             outputs=policy_network.weighted_gradients[3],
                                             initial_feed_dict=initial_feed_dict)

            # now given dlogp_dphi, dq_dphi we need to calculate the policy gradients according to (13)
            policy_grads = [dlogp_dphi[l] - dq_dphi[l] for l in range(len(dlogp_dphi))]

            # apply the gradients to policy networks
            policy_network.apply_gradients(policy_grads)
            grads_sumabs = np.sum([np.sum(np.abs(policy_grads[l])) for l in range(len(policy_grads))])
            self.policy_grads.add_sample(grads_sumabs)

            ##########################################
            # 2. updating the state value online network weights
            # done by calculating the targets for the v head according to (5) in the paper
            # value_targets = log_targets-sampled_actions_logprob
            value_inputs = copy.copy(batch.states(value_network_keys))
            value_targets = log_target - sampled_actions_logprob

            self.v_onl_ys.add_sample(value_targets)

            # call value_network apply gradients with this target
            value_loss = value_network.online_network.train_on_batch(value_inputs, value_targets[:,None])[0]

            ##########################################
            # 3. updating the critic (q networks)
            # updating q networks according to (7) in the paper

            # define the input to the q network: state has been already updated previously, but now we need
            # the actions from the batch (and not those sampled by the policy)
            q_inputs['output_0_0'] = batch.actions(len(batch.actions().shape) == 1)

            # define the targets : scale_reward * reward + (1-terminal)*discount*v_target_next_state
            # define v_target_next_state
            value_inputs = copy.copy(batch.next_states(value_network_keys))
            v_target_next_state = value_network.target_network.predict(value_inputs)
            self.v_tgt_ns.add_sample(v_target_next_state)
            # Note: reward is assumed to be rescaled by RewardRescaleFilter in the preset parameters
            TD_targets = batch.rewards(expand_dims=True) + \
                         (1.0 - batch.game_overs(expand_dims=True)) * self.ap.algorithm.discount * v_target_next_state

            # call critic network update
            result = q_network.train_on_batch(q_inputs, TD_targets, additional_fetches=[q_head.q1_loss, q_head.q2_loss])
            total_loss, losses, unclipped_grads = result[:3]
            q1_loss, q2_loss = result[3]
            self.TD_err1.add_sample(q1_loss)
            self.TD_err2.add_sample(q2_loss)

            ##########################################
            # 4. updating the value target network
            # I just need to set the parameter rate_for_copying_weights_to_target in the agent parameters to be 1-tau
            # where tau is the hyper parameter as defined in sac original implementation

            return total_loss, losses, unclipped_grads

        def get_prediction(self, states):
            """
            get the mean and stdev of the policy distribution given 'states'
            :param states: the states for which we need to sample actions from the policy
            :return: mean and stdev
            """
            tf_input_state = self.prepare_batch_for_inference(states, 'policy')
            return self.networks['policy'].online_network.predict(tf_input_state)

        def train(self):
            # since the algorithm works with experience replay buffer (non-episodic),
            # we cant use the policy optimization train method. we need Agent.train
            # note that since in Agent.train there is no apply_gradients, we need to do it in learn from batch
            return Agent.train(self)

        def choose_action(self, curr_state):
            """
            choose_action - chooses the most likely action
            if 'deterministic' - take the mean of the policy which is the prediction of the policy network.
            else - use the exploration policy
            :param curr_state:
            :return: action wrapped in ActionInfo
            """
            if not isinstance(self.spaces.action, BoxActionSpace):
                raise ValueError("SAC works only for continuous control problems")
            # convert to batch so we can run it through the network
            tf_input_state = self.prepare_batch_for_inference(curr_state, 'policy')
            # use the online network for prediction
            policy_network = self.networks['policy'].online_network
            policy_head = policy_network.output_heads[0]
            result = policy_network.predict(tf_input_state,
                                            outputs=[policy_head.policy_mean, policy_head.actions])
            action_mean, action_sample = result

            # if using deterministic policy, take the mean values. else, use exploration policy to sample from the pdf
            if self.phase == RunPhase.TEST and self.ap.algorithm.use_deterministic_for_evaluation:
                action = action_mean[0]
            else:
                action = action_sample[0]

            self.action_signal.add_sample(action)

            action_info = ActionInfo(action=action)
            return action_info