gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2025-12-18 03:30:19 +01:00
Code Issues Projects Releases Wiki Activity

fix more agents

Browse Source
This commit is contained in:
Zach Dwiel
2018-02-16 20:06:51 -05:00
parent 98f57a0d87
commit 8248caf35e
6 changed files with 52 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
agents/naf_agent.py
View File

@@ -1,5 +1,5 @@
#
# Copyright (c) 2017 Intel Corporation
# Copyright (c) 2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,7 +14,10 @@
# limitations under the License.
#
from agents.value_optimization_agent import *
import numpy as np
from agents.value_optimization_agent import ValueOptimizationAgent
from utils import RunPhase, Signal
# Normalized Advantage Functions - https://arxiv.org/pdf/1603.00748.pdf
@@ -31,14 +34,17 @@ class NAFAgent(ValueOptimizationAgent):
current_states, next_states, actions, rewards, game_overs, _ = self.extract_batch(batch)
# TD error = r + discount*v_st_plus_1 - q_st
v_st_plus_1 = self.main_network.sess.run(self.main_network.target_network.output_heads[0].V,
feed_dict={self.main_network.target_network.inputs[0]: next_states})
v_st_plus_1 = self.main_network.target_network.predict(
next_states,
self.main_network.target_network.output_heads[0].V,
squeeze_output=False,
)
TD_targets = np.expand_dims(rewards, -1) + (1.0 - np.expand_dims(game_overs, -1)) * self.tp.agent.discount * v_st_plus_1
if len(actions.shape) == 1:
actions = np.expand_dims(actions, -1)
result = self.main_network.train_and_sync_networks([current_states, actions], TD_targets)
result = self.main_network.train_and_sync_networks({**current_states, 'output_0_0': actions}, TD_targets)
total_loss = result[0]
return total_loss
@@ -47,21 +53,21 @@ class NAFAgent(ValueOptimizationAgent):
assert not self.env.discrete_controls, 'NAF works only for continuous control problems'
# convert to batch so we can run it through the network
observation = np.expand_dims(np.array(curr_state['observation']), 0)
# observation = np.expand_dims(np.array(curr_state['observation']), 0)
naf_head = self.main_network.online_network.output_heads[0]
action_values = self.main_network.sess.run(naf_head.mu,
feed_dict={self.main_network.online_network.inputs[0]: observation})
action_values = self.main_network.online_network.predict(
self.tf_input_state(curr_state),
outputs=naf_head.mu,
squeeze_output=False,
)
if phase == RunPhase.TRAIN:
action = self.exploration_policy.get_action(action_values)
else:
action = action_values
Q, L, A, mu, V = self.main_network.sess.run(
[naf_head.Q, naf_head.L, naf_head.A, naf_head.mu, naf_head.V],
feed_dict={
self.main_network.online_network.inputs[0]: observation,
self.main_network.online_network.inputs[1]: action_values
}
Q, L, A, mu, V = self.main_network.online_network.predict(
{**self.tf_input_state(curr_state), 'output_0_0': action_values},
outputs=[naf_head.Q, naf_head.L, naf_head.A, naf_head.mu, naf_head.V],
)
# store the q values statistics for logging