gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2025-12-17 11:10:20 +01:00
Code Issues Projects Releases Wiki Activity

fix more agents

Browse Source
This commit is contained in:
Zach Dwiel
2018-02-16 20:06:51 -05:00
parent 98f57a0d87
commit 8248caf35e
6 changed files with 52 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
agents/clipped_ppo_agent.py
View File

@@ -114,7 +114,6 @@ class ClippedPPOAgent(ActorCriticAgent):
# otherwise, it has both a mean and standard deviation
for input_index, input in enumerate(old_policy_distribution):
inputs['output_0_{}'.format(input_index + 1)] = input
# print('old_policy_distribution.shape', len(old_policy_distribution))
total_loss, policy_losses, unclipped_grads, fetch_result =\
self.main_network.online_network.accumulate_gradients(
inputs, [total_return, advantages], additional_fetches=fetches)

7
agents/imitation_agent.py
View File

@@ -31,12 +31,7 @@ class ImitationAgent(Agent):
def choose_action(self, curr_state, phase=RunPhase.TRAIN):
# convert to batch so we can run it through the network
observation = np.expand_dims(np.array(curr_state['observation']), 0)
if self.tp.agent.use_measurements:
measurements = np.expand_dims(np.array(curr_state['measurements']), 0)
prediction = self.main_network.online_network.predict([observation, measurements])
else:
prediction = self.main_network.online_network.predict(observation)
prediction = self.main_network.online_network.predict(self.tf_input_state(curr_state))
# get action values and extract the best action from it
action_values = self.extract_action_values(prediction)

32
agents/naf_agent.py
View File

@@ -14,7 +14,10 @@
# limitations under the License.
#
from agents.value_optimization_agent import *
import numpy as np
from agents.value_optimization_agent import ValueOptimizationAgent
from utils import RunPhase, Signal
# Normalized Advantage Functions - https://arxiv.org/pdf/1603.00748.pdf
@@ -31,14 +34,17 @@ class NAFAgent(ValueOptimizationAgent):
current_states, next_states, actions, rewards, game_overs, _ = self.extract_batch(batch)
# TD error = r + discount*v_st_plus_1 - q_st
v_st_plus_1 = self.main_network.sess.run(self.main_network.target_network.output_heads[0].V,
feed_dict={self.main_network.target_network.inputs[0]: next_states})
v_st_plus_1 = self.main_network.target_network.predict(
next_states,
self.main_network.target_network.output_heads[0].V,
squeeze_output=False,
)
TD_targets = np.expand_dims(rewards, -1) + (1.0 - np.expand_dims(game_overs, -1)) * self.tp.agent.discount * v_st_plus_1
if len(actions.shape) == 1:
actions = np.expand_dims(actions, -1)
result = self.main_network.train_and_sync_networks([current_states, actions], TD_targets)
result = self.main_network.train_and_sync_networks({**current_states, 'output_0_0': actions}, TD_targets)
total_loss = result[0]
return total_loss
@@ -47,21 +53,21 @@ class NAFAgent(ValueOptimizationAgent):
assert not self.env.discrete_controls, 'NAF works only for continuous control problems'
# convert to batch so we can run it through the network
observation = np.expand_dims(np.array(curr_state['observation']), 0)
# observation = np.expand_dims(np.array(curr_state['observation']), 0)
naf_head = self.main_network.online_network.output_heads[0]
action_values = self.main_network.sess.run(naf_head.mu,
feed_dict={self.main_network.online_network.inputs[0]: observation})
action_values = self.main_network.online_network.predict(
self.tf_input_state(curr_state),
outputs=naf_head.mu,
squeeze_output=False,
)
if phase == RunPhase.TRAIN:
action = self.exploration_policy.get_action(action_values)
else:
action = action_values
Q, L, A, mu, V = self.main_network.sess.run(
[naf_head.Q, naf_head.L, naf_head.A, naf_head.mu, naf_head.V],
feed_dict={
self.main_network.online_network.inputs[0]: observation,
self.main_network.online_network.inputs[1]: action_values
}
Q, L, A, mu, V = self.main_network.online_network.predict(
{**self.tf_input_state(curr_state), 'output_0_0': action_values},
outputs=[naf_head.Q, naf_head.L, naf_head.A, naf_head.mu, naf_head.V],
)
# store the q values statistics for logging

7
agents/policy_gradients_agent.py
View File

@@ -64,17 +64,16 @@ class PolicyGradientsAgent(PolicyOptimizationAgent):
self.returns_mean.add_sample(np.mean(total_returns))
self.returns_variance.add_sample(np.std(total_returns))
result = self.main_network.online_network.accumulate_gradients([current_states, actions], targets)
result = self.main_network.online_network.accumulate_gradients({**current_states, 'output_0_0': actions}, targets)
total_loss = result[0]
return total_loss
def choose_action(self, curr_state, phase=RunPhase.TRAIN):
# convert to batch so we can run it through the network
observation = np.expand_dims(np.array(curr_state['observation']), 0)
if self.env.discrete_controls:
# DISCRETE
action_values = self.main_network.online_network.predict(observation).squeeze()
action_values = self.main_network.online_network.predict(self.tf_input_state(curr_state)).squeeze()
if phase == RunPhase.TRAIN:
action = self.exploration_policy.get_action(action_values)
else:
@@ -83,7 +82,7 @@ class PolicyGradientsAgent(PolicyOptimizationAgent):
self.entropy.add_sample(-np.sum(action_values * np.log(action_values + eps)))
else:
# CONTINUOUS
result = self.main_network.online_network.predict(observation)
result = self.main_network.online_network.predict(self.tf_input_state(curr_state))
action_values = result[0].squeeze()
if phase == RunPhase.TRAIN:
action = self.exploration_policy.get_action(action_values)

36
agents/ppo_agent.py
View File

@@ -26,7 +26,7 @@ class PPOAgent(ActorCriticAgent):
self.critic_network = self.main_network
# define the policy network
tuning_parameters.agent.input_types = [InputTypes.Observation]
tuning_parameters.agent.input_types = {'observation': InputTypes.Observation}
tuning_parameters.agent.output_types = [OutputTypes.PPO]
tuning_parameters.agent.optimizer_type = 'Adam'
tuning_parameters.agent.l2_regularization = 0
@@ -53,7 +53,7 @@ class PPOAgent(ActorCriticAgent):
# * Found not to have any impact *
# current_states_with_timestep = self.concat_state_and_timestep(batch)
current_state_values = self.critic_network.online_network.predict(current_state).squeeze()
current_state_values = self.critic_network.online_network.predict(current_states).squeeze()
# calculate advantages
advantages = []
@@ -102,7 +102,10 @@ class PPOAgent(ActorCriticAgent):
batch_size = self.tp.batch_size
for i in range(len(dataset) // batch_size):
# split to batches for first order optimization techniques
current_states_batch = current_states[i * batch_size:(i + 1) * batch_size]
current_states_batch = {
k: v[i * batch_size:(i + 1) * batch_size]
for k, v in current_states.items()
}
total_return_batch = total_return[i * batch_size:(i + 1) * batch_size]
old_policy_values = force_list(self.critic_network.target_network.predict(
current_states_batch).squeeze())
@@ -114,10 +117,11 @@ class PPOAgent(ActorCriticAgent):
inputs = copy.copy(current_states_batch)
for input_index, input in enumerate(old_policy_values):
inputs['output_0_{}'.format(input_index)] = input
name = 'output_0_{}'.format(input_index)
if name in self.critic_network.online_network.inputs:
inputs[name] = input
value_loss = self.critic_network.online_network.\
accumulate_gradients(inputs, targets)
value_loss = self.critic_network.online_network.accumulate_gradients(inputs, targets)
self.critic_network.apply_gradients_to_online_network()
if self.tp.distributed:
self.critic_network.apply_gradients_to_global_network()
@@ -151,15 +155,23 @@ class PPOAgent(ActorCriticAgent):
actions = np.expand_dims(actions, -1)
# get old policy probabilities and distribution
old_policy = force_list(self.policy_network.target_network.predict([current_states]))
old_policy = force_list(self.policy_network.target_network.predict(current_states))
# calculate gradients and apply on both the local policy network and on the global policy network
fetches = [self.policy_network.online_network.output_heads[0].kl_divergence,
self.policy_network.online_network.output_heads[0].entropy]
inputs = copy.copy(current_states)
# TODO: why is this output 0 and not output 1?
inputs['output_0_0'] = actions
# TODO: does old_policy_distribution really need to be represented as a list?
# A: yes it does, in the event of discrete controls, it has just a mean
# otherwise, it has both a mean and standard deviation
for input_index, input in enumerate(old_policy):
inputs['output_0_{}'.format(input_index + 1)] = input
total_loss, policy_losses, unclipped_grads, fetch_result =\
self.policy_network.online_network.accumulate_gradients(
[current_states, actions] + old_policy, [advantages], additional_fetches=fetches)
inputs, [advantages], additional_fetches=fetches)
self.policy_network.apply_gradients_to_online_network()
if self.tp.distributed:
@@ -253,13 +265,9 @@ class PPOAgent(ActorCriticAgent):
return np.append(value_loss, policy_loss)
def choose_action(self, curr_state, phase=RunPhase.TRAIN):
# convert to batch so we can run it through the network
observation = curr_state['observation']
observation = np.expand_dims(np.array(observation), 0)
if self.env.discrete_controls:
# DISCRETE
action_values = self.policy_network.online_network.predict(observation).squeeze()
action_values = self.policy_network.online_network.predict(self.tf_input_state(curr_state)).squeeze()
if phase == RunPhase.TRAIN:
action = self.exploration_policy.get_action(action_values)
@@ -269,7 +277,7 @@ class PPOAgent(ActorCriticAgent):
# self.entropy.add_sample(-np.sum(action_values * np.log(action_values)))
else:
# CONTINUOUS
action_values_mean, action_values_std = self.policy_network.online_network.predict(observation)
action_values_mean, action_values_std = self.policy_network.online_network.predict(self.tf_input_state(curr_state))
action_values_mean = action_values_mean.squeeze()
action_values_std = action_values_std.squeeze()
if phase == RunPhase.TRAIN:

9
architectures/tensorflow_components/architecture.py
View File

@@ -296,16 +296,16 @@ class TensorFlowArchitecture(Architecture):
return feed_dict
def predict(self, inputs, outputs=None):
def predict(self, inputs, outputs=None, squeeze_output=True):
"""
Run a forward pass of the network using the given input
:param inputs: The input for the network
:param outputs: The output for the network, defaults to self.outputs
:param squeeze_output: call squeeze_list on output
:return: The network output
WARNING: must only call once per state since each call is assumed by LSTM to be a new time step.
"""
# TODO: rename self.inputs -> self.input_placeholders
feed_dict = self._feed_dict(inputs)
if outputs is None:
outputs = self.outputs
@@ -318,7 +318,10 @@ class TensorFlowArchitecture(Architecture):
else:
output = self.tp.sess.run(outputs, feed_dict)
return squeeze_list(output)
if squeeze_output:
output = squeeze_list(output)
return output
# def train_on_batch(self, inputs, targets, scaler=1., additional_fetches=None):
# """