gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2025-12-17 19:20:19 +01:00
Code Issues Projects Releases Wiki Activity

fix more agents

Browse Source
This commit is contained in:
Zach Dwiel
2018-02-16 20:06:51 -05:00
parent 98f57a0d87
commit 8248caf35e
6 changed files with 52 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
agents/clipped_ppo_agent.py
View File

@@ -114,7 +114,6 @@ class ClippedPPOAgent(ActorCriticAgent):
# otherwise, it has both a mean and standard deviation # otherwise, it has both a mean and standard deviation
for input_index, input in enumerate(old_policy_distribution): for input_index, input in enumerate(old_policy_distribution):
inputs['output_0_{}'.format(input_index + 1)] = input inputs['output_0_{}'.format(input_index + 1)] = input
# print('old_policy_distribution.shape', len(old_policy_distribution))
total_loss, policy_losses, unclipped_grads, fetch_result =\ total_loss, policy_losses, unclipped_grads, fetch_result =\
self.main_network.online_network.accumulate_gradients( self.main_network.online_network.accumulate_gradients(
inputs, [total_return, advantages], additional_fetches=fetches) inputs, [total_return, advantages], additional_fetches=fetches)

7
agents/imitation_agent.py
View File

@@ -31,12 +31,7 @@ class ImitationAgent(Agent):
def choose_action(self, curr_state, phase=RunPhase.TRAIN): def choose_action(self, curr_state, phase=RunPhase.TRAIN):
# convert to batch so we can run it through the network # convert to batch so we can run it through the network
observation = np.expand_dims(np.array(curr_state['observation']), 0) prediction = self.main_network.online_network.predict(self.tf_input_state(curr_state))
if self.tp.agent.use_measurements:
measurements = np.expand_dims(np.array(curr_state['measurements']), 0)
prediction = self.main_network.online_network.predict([observation, measurements])
else:
prediction = self.main_network.online_network.predict(observation)
# get action values and extract the best action from it # get action values and extract the best action from it
action_values = self.extract_action_values(prediction) action_values = self.extract_action_values(prediction)

32
agents/naf_agent.py
View File

@@ -14,7 +14,10 @@
# limitations under the License. # limitations under the License.
# #
from agents.value_optimization_agent import * import numpy as np
from agents.value_optimization_agent import ValueOptimizationAgent
from utils import RunPhase, Signal
# Normalized Advantage Functions - https://arxiv.org/pdf/1603.00748.pdf # Normalized Advantage Functions - https://arxiv.org/pdf/1603.00748.pdf
@@ -31,14 +34,17 @@ class NAFAgent(ValueOptimizationAgent):
current_states, next_states, actions, rewards, game_overs, _ = self.extract_batch(batch) current_states, next_states, actions, rewards, game_overs, _ = self.extract_batch(batch)
# TD error = r + discount*v_st_plus_1 - q_st # TD error = r + discount*v_st_plus_1 - q_st
v_st_plus_1 = self.main_network.sess.run(self.main_network.target_network.output_heads[0].V, v_st_plus_1 = self.main_network.target_network.predict(
feed_dict={self.main_network.target_network.inputs[0]: next_states}) next_states,
self.main_network.target_network.output_heads[0].V,
squeeze_output=False,
)
TD_targets = np.expand_dims(rewards, -1) + (1.0 - np.expand_dims(game_overs, -1)) * self.tp.agent.discount * v_st_plus_1 TD_targets = np.expand_dims(rewards, -1) + (1.0 - np.expand_dims(game_overs, -1)) * self.tp.agent.discount * v_st_plus_1
if len(actions.shape) == 1: if len(actions.shape) == 1:
actions = np.expand_dims(actions, -1) actions = np.expand_dims(actions, -1)
result = self.main_network.train_and_sync_networks([current_states, actions], TD_targets) result = self.main_network.train_and_sync_networks({**current_states, 'output_0_0': actions}, TD_targets)
total_loss = result[0] total_loss = result[0]
return total_loss return total_loss
@@ -47,21 +53,21 @@ class NAFAgent(ValueOptimizationAgent):
assert not self.env.discrete_controls, 'NAF works only for continuous control problems' assert not self.env.discrete_controls, 'NAF works only for continuous control problems'
# convert to batch so we can run it through the network # convert to batch so we can run it through the network
observation = np.expand_dims(np.array(curr_state['observation']), 0) # observation = np.expand_dims(np.array(curr_state['observation']), 0)
naf_head = self.main_network.online_network.output_heads[0] naf_head = self.main_network.online_network.output_heads[0]
action_values = self.main_network.sess.run(naf_head.mu, action_values = self.main_network.online_network.predict(
feed_dict={self.main_network.online_network.inputs[0]: observation}) self.tf_input_state(curr_state),
outputs=naf_head.mu,
squeeze_output=False,
)
if phase == RunPhase.TRAIN: if phase == RunPhase.TRAIN:
action = self.exploration_policy.get_action(action_values) action = self.exploration_policy.get_action(action_values)
else: else:
action = action_values action = action_values
Q, L, A, mu, V = self.main_network.sess.run( Q, L, A, mu, V = self.main_network.online_network.predict(
[naf_head.Q, naf_head.L, naf_head.A, naf_head.mu, naf_head.V], {**self.tf_input_state(curr_state), 'output_0_0': action_values},
feed_dict={ outputs=[naf_head.Q, naf_head.L, naf_head.A, naf_head.mu, naf_head.V],
self.main_network.online_network.inputs[0]: observation,
self.main_network.online_network.inputs[1]: action_values
}
) )
# store the q values statistics for logging # store the q values statistics for logging

7
agents/policy_gradients_agent.py
View File

@@ -64,17 +64,16 @@ class PolicyGradientsAgent(PolicyOptimizationAgent):
self.returns_mean.add_sample(np.mean(total_returns)) self.returns_mean.add_sample(np.mean(total_returns))
self.returns_variance.add_sample(np.std(total_returns)) self.returns_variance.add_sample(np.std(total_returns))
result = self.main_network.online_network.accumulate_gradients([current_states, actions], targets) result = self.main_network.online_network.accumulate_gradients({**current_states, 'output_0_0': actions}, targets)
total_loss = result[0] total_loss = result[0]
return total_loss return total_loss
def choose_action(self, curr_state, phase=RunPhase.TRAIN): def choose_action(self, curr_state, phase=RunPhase.TRAIN):
# convert to batch so we can run it through the network # convert to batch so we can run it through the network
observation = np.expand_dims(np.array(curr_state['observation']), 0)
if self.env.discrete_controls: if self.env.discrete_controls:
# DISCRETE # DISCRETE
action_values = self.main_network.online_network.predict(observation).squeeze() action_values = self.main_network.online_network.predict(self.tf_input_state(curr_state)).squeeze()
if phase == RunPhase.TRAIN: if phase == RunPhase.TRAIN:
action = self.exploration_policy.get_action(action_values) action = self.exploration_policy.get_action(action_values)
else: else:
@@ -83,7 +82,7 @@ class PolicyGradientsAgent(PolicyOptimizationAgent):
self.entropy.add_sample(-np.sum(action_values * np.log(action_values + eps))) self.entropy.add_sample(-np.sum(action_values * np.log(action_values + eps)))
else: else:
# CONTINUOUS # CONTINUOUS
result = self.main_network.online_network.predict(observation) result = self.main_network.online_network.predict(self.tf_input_state(curr_state))
action_values = result[0].squeeze() action_values = result[0].squeeze()
if phase == RunPhase.TRAIN: if phase == RunPhase.TRAIN:
action = self.exploration_policy.get_action(action_values) action = self.exploration_policy.get_action(action_values)

36
agents/ppo_agent.py
View File

@@ -26,7 +26,7 @@ class PPOAgent(ActorCriticAgent):
self.critic_network = self.main_network self.critic_network = self.main_network
# define the policy network # define the policy network
tuning_parameters.agent.input_types = [InputTypes.Observation] tuning_parameters.agent.input_types = {'observation': InputTypes.Observation}
tuning_parameters.agent.output_types = [OutputTypes.PPO] tuning_parameters.agent.output_types = [OutputTypes.PPO]
tuning_parameters.agent.optimizer_type = 'Adam' tuning_parameters.agent.optimizer_type = 'Adam'
tuning_parameters.agent.l2_regularization = 0 tuning_parameters.agent.l2_regularization = 0
@@ -53,7 +53,7 @@ class PPOAgent(ActorCriticAgent):
# * Found not to have any impact * # * Found not to have any impact *
# current_states_with_timestep = self.concat_state_and_timestep(batch) # current_states_with_timestep = self.concat_state_and_timestep(batch)
current_state_values = self.critic_network.online_network.predict(current_state).squeeze() current_state_values = self.critic_network.online_network.predict(current_states).squeeze()
# calculate advantages # calculate advantages
advantages = [] advantages = []
@@ -102,7 +102,10 @@ class PPOAgent(ActorCriticAgent):
batch_size = self.tp.batch_size batch_size = self.tp.batch_size
for i in range(len(dataset) // batch_size): for i in range(len(dataset) // batch_size):
# split to batches for first order optimization techniques # split to batches for first order optimization techniques
current_states_batch = current_states[i * batch_size:(i + 1) * batch_size] current_states_batch = {
k: v[i * batch_size:(i + 1) * batch_size]
for k, v in current_states.items()
}
total_return_batch = total_return[i * batch_size:(i + 1) * batch_size] total_return_batch = total_return[i * batch_size:(i + 1) * batch_size]
old_policy_values = force_list(self.critic_network.target_network.predict( old_policy_values = force_list(self.critic_network.target_network.predict(
current_states_batch).squeeze()) current_states_batch).squeeze())
@@ -114,10 +117,11 @@ class PPOAgent(ActorCriticAgent):
inputs = copy.copy(current_states_batch) inputs = copy.copy(current_states_batch)
for input_index, input in enumerate(old_policy_values): for input_index, input in enumerate(old_policy_values):
inputs['output_0_{}'.format(input_index)] = input name = 'output_0_{}'.format(input_index)
if name in self.critic_network.online_network.inputs:
inputs[name] = input
value_loss = self.critic_network.online_network.\ value_loss = self.critic_network.online_network.accumulate_gradients(inputs, targets)
accumulate_gradients(inputs, targets)
self.critic_network.apply_gradients_to_online_network() self.critic_network.apply_gradients_to_online_network()
if self.tp.distributed: if self.tp.distributed:
self.critic_network.apply_gradients_to_global_network() self.critic_network.apply_gradients_to_global_network()
@@ -151,15 +155,23 @@ class PPOAgent(ActorCriticAgent):
actions = np.expand_dims(actions, -1) actions = np.expand_dims(actions, -1)
# get old policy probabilities and distribution # get old policy probabilities and distribution
old_policy = force_list(self.policy_network.target_network.predict([current_states])) old_policy = force_list(self.policy_network.target_network.predict(current_states))
# calculate gradients and apply on both the local policy network and on the global policy network # calculate gradients and apply on both the local policy network and on the global policy network
fetches = [self.policy_network.online_network.output_heads[0].kl_divergence, fetches = [self.policy_network.online_network.output_heads[0].kl_divergence,
self.policy_network.online_network.output_heads[0].entropy] self.policy_network.online_network.output_heads[0].entropy]
inputs = copy.copy(current_states)
# TODO: why is this output 0 and not output 1?
inputs['output_0_0'] = actions
# TODO: does old_policy_distribution really need to be represented as a list?
# A: yes it does, in the event of discrete controls, it has just a mean
# otherwise, it has both a mean and standard deviation
for input_index, input in enumerate(old_policy):
inputs['output_0_{}'.format(input_index + 1)] = input
total_loss, policy_losses, unclipped_grads, fetch_result =\ total_loss, policy_losses, unclipped_grads, fetch_result =\
self.policy_network.online_network.accumulate_gradients( self.policy_network.online_network.accumulate_gradients(
[current_states, actions] + old_policy, [advantages], additional_fetches=fetches) inputs, [advantages], additional_fetches=fetches)
self.policy_network.apply_gradients_to_online_network() self.policy_network.apply_gradients_to_online_network()
if self.tp.distributed: if self.tp.distributed:
@@ -253,13 +265,9 @@ class PPOAgent(ActorCriticAgent):
return np.append(value_loss, policy_loss) return np.append(value_loss, policy_loss)
def choose_action(self, curr_state, phase=RunPhase.TRAIN): def choose_action(self, curr_state, phase=RunPhase.TRAIN):
# convert to batch so we can run it through the network
observation = curr_state['observation']
observation = np.expand_dims(np.array(observation), 0)
if self.env.discrete_controls: if self.env.discrete_controls:
# DISCRETE # DISCRETE
action_values = self.policy_network.online_network.predict(observation).squeeze() action_values = self.policy_network.online_network.predict(self.tf_input_state(curr_state)).squeeze()
if phase == RunPhase.TRAIN: if phase == RunPhase.TRAIN:
action = self.exploration_policy.get_action(action_values) action = self.exploration_policy.get_action(action_values)
@@ -269,7 +277,7 @@ class PPOAgent(ActorCriticAgent):
# self.entropy.add_sample(-np.sum(action_values * np.log(action_values))) # self.entropy.add_sample(-np.sum(action_values * np.log(action_values)))
else: else:
# CONTINUOUS # CONTINUOUS
action_values_mean, action_values_std = self.policy_network.online_network.predict(observation) action_values_mean, action_values_std = self.policy_network.online_network.predict(self.tf_input_state(curr_state))
action_values_mean = action_values_mean.squeeze() action_values_mean = action_values_mean.squeeze()
action_values_std = action_values_std.squeeze() action_values_std = action_values_std.squeeze()
if phase == RunPhase.TRAIN: if phase == RunPhase.TRAIN:

9
architectures/tensorflow_components/architecture.py
View File

@@ -296,16 +296,16 @@ class TensorFlowArchitecture(Architecture):
return feed_dict return feed_dict
def predict(self, inputs, outputs=None): def predict(self, inputs, outputs=None, squeeze_output=True):
""" """
Run a forward pass of the network using the given input Run a forward pass of the network using the given input
:param inputs: The input for the network :param inputs: The input for the network
:param outputs: The output for the network, defaults to self.outputs :param outputs: The output for the network, defaults to self.outputs
:param squeeze_output: call squeeze_list on output
:return: The network output :return: The network output
WARNING: must only call once per state since each call is assumed by LSTM to be a new time step. WARNING: must only call once per state since each call is assumed by LSTM to be a new time step.
""" """
# TODO: rename self.inputs -> self.input_placeholders
feed_dict = self._feed_dict(inputs) feed_dict = self._feed_dict(inputs)
if outputs is None: if outputs is None:
outputs = self.outputs outputs = self.outputs
@@ -318,7 +318,10 @@ class TensorFlowArchitecture(Architecture):
else: else:
output = self.tp.sess.run(outputs, feed_dict) output = self.tp.sess.run(outputs, feed_dict)
return squeeze_list(output) if squeeze_output:
output = squeeze_list(output)
return output
# def train_on_batch(self, inputs, targets, scaler=1., additional_fetches=None): # def train_on_batch(self, inputs, targets, scaler=1., additional_fetches=None):
# """ # """