diff --git a/agents/clipped_ppo_agent.py b/agents/clipped_ppo_agent.py index de8cd37..88a70b0 100644 --- a/agents/clipped_ppo_agent.py +++ b/agents/clipped_ppo_agent.py @@ -114,7 +114,6 @@ class ClippedPPOAgent(ActorCriticAgent): # otherwise, it has both a mean and standard deviation for input_index, input in enumerate(old_policy_distribution): inputs['output_0_{}'.format(input_index + 1)] = input - # print('old_policy_distribution.shape', len(old_policy_distribution)) total_loss, policy_losses, unclipped_grads, fetch_result =\ self.main_network.online_network.accumulate_gradients( inputs, [total_return, advantages], additional_fetches=fetches) diff --git a/agents/imitation_agent.py b/agents/imitation_agent.py index f7f5e06..f893fbe 100644 --- a/agents/imitation_agent.py +++ b/agents/imitation_agent.py @@ -31,12 +31,7 @@ class ImitationAgent(Agent): def choose_action(self, curr_state, phase=RunPhase.TRAIN): # convert to batch so we can run it through the network - observation = np.expand_dims(np.array(curr_state['observation']), 0) - if self.tp.agent.use_measurements: - measurements = np.expand_dims(np.array(curr_state['measurements']), 0) - prediction = self.main_network.online_network.predict([observation, measurements]) - else: - prediction = self.main_network.online_network.predict(observation) + prediction = self.main_network.online_network.predict(self.tf_input_state(curr_state)) # get action values and extract the best action from it action_values = self.extract_action_values(prediction) diff --git a/agents/naf_agent.py b/agents/naf_agent.py index 3ef563e..65ca83c 100644 --- a/agents/naf_agent.py +++ b/agents/naf_agent.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2017 Intel Corporation +# Copyright (c) 2017 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,10 @@ # limitations under the License. # -from agents.value_optimization_agent import * +import numpy as np + +from agents.value_optimization_agent import ValueOptimizationAgent +from utils import RunPhase, Signal # Normalized Advantage Functions - https://arxiv.org/pdf/1603.00748.pdf @@ -31,14 +34,17 @@ class NAFAgent(ValueOptimizationAgent): current_states, next_states, actions, rewards, game_overs, _ = self.extract_batch(batch) # TD error = r + discount*v_st_plus_1 - q_st - v_st_plus_1 = self.main_network.sess.run(self.main_network.target_network.output_heads[0].V, - feed_dict={self.main_network.target_network.inputs[0]: next_states}) + v_st_plus_1 = self.main_network.target_network.predict( + next_states, + self.main_network.target_network.output_heads[0].V, + squeeze_output=False, + ) TD_targets = np.expand_dims(rewards, -1) + (1.0 - np.expand_dims(game_overs, -1)) * self.tp.agent.discount * v_st_plus_1 if len(actions.shape) == 1: actions = np.expand_dims(actions, -1) - result = self.main_network.train_and_sync_networks([current_states, actions], TD_targets) + result = self.main_network.train_and_sync_networks({**current_states, 'output_0_0': actions}, TD_targets) total_loss = result[0] return total_loss @@ -47,21 +53,21 @@ class NAFAgent(ValueOptimizationAgent): assert not self.env.discrete_controls, 'NAF works only for continuous control problems' # convert to batch so we can run it through the network - observation = np.expand_dims(np.array(curr_state['observation']), 0) + # observation = np.expand_dims(np.array(curr_state['observation']), 0) naf_head = self.main_network.online_network.output_heads[0] - action_values = self.main_network.sess.run(naf_head.mu, - feed_dict={self.main_network.online_network.inputs[0]: observation}) + action_values = self.main_network.online_network.predict( + self.tf_input_state(curr_state), + outputs=naf_head.mu, + squeeze_output=False, + ) if phase == RunPhase.TRAIN: action = self.exploration_policy.get_action(action_values) else: action = action_values - Q, L, A, mu, V = self.main_network.sess.run( - [naf_head.Q, naf_head.L, naf_head.A, naf_head.mu, naf_head.V], - feed_dict={ - self.main_network.online_network.inputs[0]: observation, - self.main_network.online_network.inputs[1]: action_values - } + Q, L, A, mu, V = self.main_network.online_network.predict( + {**self.tf_input_state(curr_state), 'output_0_0': action_values}, + outputs=[naf_head.Q, naf_head.L, naf_head.A, naf_head.mu, naf_head.V], ) # store the q values statistics for logging diff --git a/agents/policy_gradients_agent.py b/agents/policy_gradients_agent.py index 53c8fc4..3a592d1 100644 --- a/agents/policy_gradients_agent.py +++ b/agents/policy_gradients_agent.py @@ -64,17 +64,16 @@ class PolicyGradientsAgent(PolicyOptimizationAgent): self.returns_mean.add_sample(np.mean(total_returns)) self.returns_variance.add_sample(np.std(total_returns)) - result = self.main_network.online_network.accumulate_gradients([current_states, actions], targets) + result = self.main_network.online_network.accumulate_gradients({**current_states, 'output_0_0': actions}, targets) total_loss = result[0] return total_loss def choose_action(self, curr_state, phase=RunPhase.TRAIN): # convert to batch so we can run it through the network - observation = np.expand_dims(np.array(curr_state['observation']), 0) if self.env.discrete_controls: # DISCRETE - action_values = self.main_network.online_network.predict(observation).squeeze() + action_values = self.main_network.online_network.predict(self.tf_input_state(curr_state)).squeeze() if phase == RunPhase.TRAIN: action = self.exploration_policy.get_action(action_values) else: @@ -83,7 +82,7 @@ class PolicyGradientsAgent(PolicyOptimizationAgent): self.entropy.add_sample(-np.sum(action_values * np.log(action_values + eps))) else: # CONTINUOUS - result = self.main_network.online_network.predict(observation) + result = self.main_network.online_network.predict(self.tf_input_state(curr_state)) action_values = result[0].squeeze() if phase == RunPhase.TRAIN: action = self.exploration_policy.get_action(action_values) diff --git a/agents/ppo_agent.py b/agents/ppo_agent.py index daef1b1..4a37e69 100644 --- a/agents/ppo_agent.py +++ b/agents/ppo_agent.py @@ -26,7 +26,7 @@ class PPOAgent(ActorCriticAgent): self.critic_network = self.main_network # define the policy network - tuning_parameters.agent.input_types = [InputTypes.Observation] + tuning_parameters.agent.input_types = {'observation': InputTypes.Observation} tuning_parameters.agent.output_types = [OutputTypes.PPO] tuning_parameters.agent.optimizer_type = 'Adam' tuning_parameters.agent.l2_regularization = 0 @@ -53,7 +53,7 @@ class PPOAgent(ActorCriticAgent): # * Found not to have any impact * # current_states_with_timestep = self.concat_state_and_timestep(batch) - current_state_values = self.critic_network.online_network.predict(current_state).squeeze() + current_state_values = self.critic_network.online_network.predict(current_states).squeeze() # calculate advantages advantages = [] @@ -102,7 +102,10 @@ class PPOAgent(ActorCriticAgent): batch_size = self.tp.batch_size for i in range(len(dataset) // batch_size): # split to batches for first order optimization techniques - current_states_batch = current_states[i * batch_size:(i + 1) * batch_size] + current_states_batch = { + k: v[i * batch_size:(i + 1) * batch_size] + for k, v in current_states.items() + } total_return_batch = total_return[i * batch_size:(i + 1) * batch_size] old_policy_values = force_list(self.critic_network.target_network.predict( current_states_batch).squeeze()) @@ -114,10 +117,11 @@ class PPOAgent(ActorCriticAgent): inputs = copy.copy(current_states_batch) for input_index, input in enumerate(old_policy_values): - inputs['output_0_{}'.format(input_index)] = input + name = 'output_0_{}'.format(input_index) + if name in self.critic_network.online_network.inputs: + inputs[name] = input - value_loss = self.critic_network.online_network.\ - accumulate_gradients(inputs, targets) + value_loss = self.critic_network.online_network.accumulate_gradients(inputs, targets) self.critic_network.apply_gradients_to_online_network() if self.tp.distributed: self.critic_network.apply_gradients_to_global_network() @@ -151,15 +155,23 @@ class PPOAgent(ActorCriticAgent): actions = np.expand_dims(actions, -1) # get old policy probabilities and distribution - old_policy = force_list(self.policy_network.target_network.predict([current_states])) + old_policy = force_list(self.policy_network.target_network.predict(current_states)) # calculate gradients and apply on both the local policy network and on the global policy network fetches = [self.policy_network.online_network.output_heads[0].kl_divergence, self.policy_network.online_network.output_heads[0].entropy] + inputs = copy.copy(current_states) + # TODO: why is this output 0 and not output 1? + inputs['output_0_0'] = actions + # TODO: does old_policy_distribution really need to be represented as a list? + # A: yes it does, in the event of discrete controls, it has just a mean + # otherwise, it has both a mean and standard deviation + for input_index, input in enumerate(old_policy): + inputs['output_0_{}'.format(input_index + 1)] = input total_loss, policy_losses, unclipped_grads, fetch_result =\ self.policy_network.online_network.accumulate_gradients( - [current_states, actions] + old_policy, [advantages], additional_fetches=fetches) + inputs, [advantages], additional_fetches=fetches) self.policy_network.apply_gradients_to_online_network() if self.tp.distributed: @@ -253,13 +265,9 @@ class PPOAgent(ActorCriticAgent): return np.append(value_loss, policy_loss) def choose_action(self, curr_state, phase=RunPhase.TRAIN): - # convert to batch so we can run it through the network - observation = curr_state['observation'] - observation = np.expand_dims(np.array(observation), 0) - if self.env.discrete_controls: # DISCRETE - action_values = self.policy_network.online_network.predict(observation).squeeze() + action_values = self.policy_network.online_network.predict(self.tf_input_state(curr_state)).squeeze() if phase == RunPhase.TRAIN: action = self.exploration_policy.get_action(action_values) @@ -269,7 +277,7 @@ class PPOAgent(ActorCriticAgent): # self.entropy.add_sample(-np.sum(action_values * np.log(action_values))) else: # CONTINUOUS - action_values_mean, action_values_std = self.policy_network.online_network.predict(observation) + action_values_mean, action_values_std = self.policy_network.online_network.predict(self.tf_input_state(curr_state)) action_values_mean = action_values_mean.squeeze() action_values_std = action_values_std.squeeze() if phase == RunPhase.TRAIN: diff --git a/architectures/tensorflow_components/architecture.py b/architectures/tensorflow_components/architecture.py index 82a898e..5008b32 100644 --- a/architectures/tensorflow_components/architecture.py +++ b/architectures/tensorflow_components/architecture.py @@ -296,16 +296,16 @@ class TensorFlowArchitecture(Architecture): return feed_dict - def predict(self, inputs, outputs=None): + def predict(self, inputs, outputs=None, squeeze_output=True): """ Run a forward pass of the network using the given input :param inputs: The input for the network :param outputs: The output for the network, defaults to self.outputs + :param squeeze_output: call squeeze_list on output :return: The network output WARNING: must only call once per state since each call is assumed by LSTM to be a new time step. """ - # TODO: rename self.inputs -> self.input_placeholders feed_dict = self._feed_dict(inputs) if outputs is None: outputs = self.outputs @@ -318,7 +318,10 @@ class TensorFlowArchitecture(Architecture): else: output = self.tp.sess.run(outputs, feed_dict) - return squeeze_list(output) + if squeeze_output: + output = squeeze_list(output) + + return output # def train_on_batch(self, inputs, targets, scaler=1., additional_fetches=None): # """