gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2025-12-17 19:20:19 +01:00
Code Issues Projects Releases Wiki Activity
Files
42a9ec132d39d0b381a8dbdcc1c1be4caf3a90a8
coach/presets.py
Roman Dobosz 42a9ec132d Merge branch 'master' into imports
2018-04-24 07:43:04 +02:00

1431 lines
56 KiB
Python
Raw Blame History

#
# Copyright (c) 2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import ast
import json
import sys
import agents
import configurations as conf
import environments as env
import exploration_policies as ep
import presets
def json_to_preset(json_path):
with open(json_path, 'r') as json_file:
run_dict = json.loads(json_file.read())
if run_dict['preset'] is None:
tuning_parameters = conf.Preset(eval('agents.' + run_dict['agent_type']),
eval('env.' + run_dict['environment_type']),
eval('ep.' + run_dict['exploration_policy_type']))
else:
tuning_parameters = eval('presets.' + run_dict['preset'])()
# Override existing parts of the preset
if run_dict['agent_type'] is not None:
tuning_parameters.agent = eval('agents.' + run_dict['agent_type'])()
if run_dict['environment_type'] is not None:
tuning_parameters.env = eval('env.' + run_dict['environment_type'])()
if run_dict['exploration_policy_type'] is not None:
tuning_parameters.exploration = eval('ep.' + run_dict['exploration_policy_type'])()
# human control
if run_dict['play']:
tuning_parameters.agent.type = 'HumanAgent'
tuning_parameters.env.human_control = True
tuning_parameters.num_heatup_steps = 0
if run_dict['level']:
tuning_parameters.env.level = run_dict['level']
if run_dict['custom_parameter'] is not None:
unstripped_key_value_pairs = [pair.split('=') for pair in run_dict['custom_parameter'].split(';')]
stripped_key_value_pairs = [tuple([pair[0].strip(), ast.literal_eval(pair[1].strip())]) for pair in
unstripped_key_value_pairs]
# load custom parameters into run_dict
for key, value in stripped_key_value_pairs:
run_dict[key] = value
for key in ['agent_type', 'environment_type', 'exploration_policy_type', 'preset', 'custom_parameter']:
run_dict.pop(key, None)
# load parameters from run_dict to tuning_parameters
for key, value in run_dict.items():
if ((sys.version_info[0] == 2 and type(value) == unicode) or
(sys.version_info[0] == 3 and type(value) == str)):
value = '"{}"'.format(value)
exec('tuning_parameters.{} = {}'.format(key, value)) in globals(), locals()
return tuning_parameters
class Doom_Basic_DQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DQN, conf.Doom, conf.ExplorationParameters)
self.env.level = 'basic'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.agent.num_steps_between_copying_online_weights_to_target = 1000
self.num_heatup_steps = 1000
class Doom_Basic_QRDQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.QuantileRegressionDQN, conf.Doom, conf.ExplorationParameters)
self.env.level = 'basic'
self.agent.num_steps_between_copying_online_weights_to_target = 1000
self.learning_rate = 0.00025
self.agent.num_episodes_in_experience_replay = 200
self.num_heatup_steps = 1000
class Doom_Basic_OneStepQ(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NStepQ, conf.Doom, conf.ExplorationParameters)
self.env.level = 'basic'
self.learning_rate = 0.00025
self.num_heatup_steps = 0
self.agent.num_steps_between_copying_online_weights_to_target = 100
self.agent.optimizer_type = 'Adam'
self.clip_gradients = 1000
self.agent.targets_horizon = '1-Step'
class Doom_Basic_NStepQ(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NStepQ, conf.Doom, conf.ExplorationParameters)
self.env.level = 'basic'
self.learning_rate = 0.000025
self.num_heatup_steps = 0
self.agent.num_steps_between_copying_online_weights_to_target = 1000
self.agent.optimizer_type = 'Adam'
self.clip_gradients = 1000
class Doom_Basic_A2C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.Doom, conf.CategoricalExploration)
self.env.level = 'basic'
self.agent.policy_gradient_rescaler = 'A_VALUE'
self.learning_rate = 0.00025
self.num_heatup_steps = 100
self.env.reward_scaling = 100.
class Doom_Basic_Dueling_DDQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDQN, conf.Doom, conf.ExplorationParameters)
self.env.level = 'basic'
self.agent.output_types = [conf.OutputTypes.DuelingQ]
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.agent.num_steps_between_copying_online_weights_to_target = 1000
self.num_heatup_steps = 1000
class Doom_Basic_Dueling_DQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DuelingDQN, conf.Doom, conf.ExplorationParameters)
self.env.level = 'basic'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.agent.num_steps_between_copying_online_weights_to_target = 1000
self.num_heatup_steps = 1000
class CartPole_Dueling_DDQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDQN, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.agent.output_types = [conf.OutputTypes.DuelingQ]
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.agent.num_steps_between_copying_online_weights_to_target = 100
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 3000
self.agent.discount = 1.0
self.test = True
self.test_max_step_threshold = 100
self.test_min_return_threshold = 150
class Doom_Health_MMC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.MMC, conf.Doom, conf.ExplorationParameters)
self.env.level = 'HEALTH_GATHERING'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.agent.num_steps_between_copying_online_weights_to_target = 1000
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 10000
class CartPole_MMC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.MMC, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.agent.num_steps_between_copying_online_weights_to_target = 100
self.learning_rate = 0.00025
self.agent.num_episodes_in_experience_replay = 200
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 3000
self.agent.discount = 1.0
self.test = True
self.test_max_step_threshold = 90
self.test_min_return_threshold = 150
class CartPole_PAL(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PAL, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.agent.num_steps_between_copying_online_weights_to_target = 100
self.learning_rate = 0.00025
self.agent.num_episodes_in_experience_replay = 200
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 3000
self.agent.discount = 1.0
self.test = True
self.test_max_step_threshold = 100
self.test_min_return_threshold = 150
class CartPole_DFP(conf.Preset):
def __init__(self):
Preset.__init__(self, conf.DFP, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.0001
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 10000
self.agent.use_accumulated_reward_as_measurement = True
self.agent.goal_vector = [1.0]
class Doom_Basic_DFP(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DFP, conf.Doom, conf.ExplorationParameters)
self.env.level = 'BASIC'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.0001
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 10000
self.agent.use_accumulated_reward_as_measurement = True
self.agent.goal_vector = [0.0, 1.0]
# self.agent.num_consecutive_playing_steps = 10
class Doom_Health_DFP(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DFP, conf.Doom, conf.ExplorationParameters)
self.env.level = 'HEALTH_GATHERING'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 10000
self.agent.use_accumulated_reward_as_measurement = True
class Doom_Deadly_Corridor_Bootstrapped_DQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.BootstrappedDQN, conf.Doom, conf.BootstrappedDQNExploration)
self.env.level = 'deadly_corridor'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.agent.num_steps_between_copying_online_weights_to_target = 1000
self.num_heatup_steps = 1000
class CartPole_Bootstrapped_DQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.BootstrappedDQN, conf.GymVectorObservation, conf.BootstrappedDQNExploration)
self.env.level = 'CartPole-v0'
self.agent.num_steps_between_copying_online_weights_to_target = 200
self.learning_rate = 0.00025
self.agent.num_episodes_in_experience_replay = 200
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 3000
self.agent.discount = 1.0
self.test = True
self.test_max_step_threshold = 200
self.test_min_return_threshold = 150
class CartPole_PG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PolicyGradient, conf.GymVectorObservation, conf.CategoricalExploration)
self.env.level = 'CartPole-v0'
self.agent.policy_gradient_rescaler = 'FUTURE_RETURN_NORMALIZED_BY_TIMESTEP'
self.learning_rate = 0.001
self.num_heatup_steps = 100
self.agent.discount = 1.0
self.test = True
self.test_max_step_threshold = 150
self.test_min_return_threshold = 150
class CartPole_PPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PPO, conf.GymVectorObservation, conf.CategoricalExploration)
self.env.level = 'CartPole-v0'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 512
self.agent.discount = 0.99
self.batch_size = 128
self.agent.policy_gradient_rescaler = 'A_VALUE'
self.agent.optimizer_type = 'LBFGS'
self.env.normalize_observation = True
self.test = True
self.test_max_step_threshold = 200
self.test_min_return_threshold = 150
class CartPole_ClippedPPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.GymVectorObservation, conf.CategoricalExploration)
self.env.level = 'CartPole-v0'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 512
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
self.test = True
self.test_max_step_threshold = 200
self.test_min_return_threshold = 150
class CartPole_A2C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.CategoricalExploration)
self.env.level = 'CartPole-v0'
self.agent.policy_gradient_rescaler = 'A_VALUE'
self.learning_rate = 0.001
self.num_heatup_steps = 0
self.env.reward_scaling = 200.
self.agent.discount = 1.0
self.test = True
self.test_max_step_threshold = 300
self.test_min_return_threshold = 150
class CartPole_OneStepQ(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NStepQ, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.agent.num_steps_between_copying_online_weights_to_target = 100
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.discount = 1.0
self.agent.targets_horizon = '1-Step'
class CartPole_NStepQ(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NStepQ, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.agent.num_steps_between_copying_online_weights_to_target = 100
self.learning_rate = 0.0001
self.exploration.epsilon_decay_steps = 10000
self.num_heatup_steps = 0
self.agent.discount = 0.99
self.agent.num_steps_between_gradient_updates = 5
self.test = True
self.test_max_step_threshold = 2000
self.test_min_return_threshold = 150
self.test_num_workers = 8
class CartPole_DQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DQN, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.agent.num_steps_between_copying_online_weights_to_target = 100
self.learning_rate = 0.00025
self.agent.num_episodes_in_experience_replay = 200
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 3000
self.agent.discount = 1.0
self.test = True
self.test_max_step_threshold = 150
self.test_min_return_threshold = 150
class CartPole_C51(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.CategoricalDQN, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.agent.num_steps_between_copying_online_weights_to_target = 100
self.learning_rate = 0.00025
self.agent.num_episodes_in_experience_replay = 200
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 3000
self.agent.discount = 1.0
# self.env.reward_scaling = 20.
self.agent.v_min = 0.0
self.agent.v_max = 200.0
self.test = True
self.test_max_step_threshold = 150
self.test_min_return_threshold = 150
class CartPole_QRDQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.QuantileRegressionDQN, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.agent.num_steps_between_copying_online_weights_to_target = 100
self.learning_rate = 0.00025
self.agent.num_episodes_in_experience_replay = 200
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 3000
self.agent.discount = 1.0
# The below preset matches the hyper-parameters setting as in the original DQN paper.
# This a very resource intensive preset, and might easily blow up your RAM (> 100GB of usage).
# Try reducing the number of transitions in the experience replay (50e3 might be a reasonable number to start with),
# so to make sure it fits your RAM.
class Breakout_DQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DQN, conf.Atari, conf.ExplorationParameters)
self.env.level = 'BreakoutDeterministic-v4'
self.agent.num_steps_between_copying_online_weights_to_target = 10000
self.learning_rate = 0.00025
self.agent.num_transitions_in_experience_replay = 1000000
self.exploration.initial_epsilon = 1.0
self.exploration.final_epsilon = 0.1
self.exploration.epsilon_decay_steps = 1000000
self.exploration.evaluation_policy = 'EGreedy'
self.exploration.evaluation_epsilon = 0.05
self.num_heatup_steps = 50000
self.agent.num_consecutive_playing_steps = 4
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 25
self.agent.replace_mse_with_huber_loss = True
# self.env.crop_observation = True # TODO: remove
# self.rescaling_interpolation_type = 'nearest' # TODO: remove
class Breakout_DDQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDQN, conf.Atari, conf.ExplorationParameters)
self.env.level = 'BreakoutDeterministic-v4'
self.agent.num_steps_between_copying_online_weights_to_target = 30000
self.learning_rate = 0.00025
self.agent.num_transitions_in_experience_replay = 1000000
self.exploration.initial_epsilon = 1.0
self.exploration.final_epsilon = 0.01
self.exploration.epsilon_decay_steps = 1000000
self.exploration.evaluation_policy = 'EGreedy'
self.exploration.evaluation_epsilon = 0.001
self.num_heatup_steps = 50000
self.agent.num_consecutive_playing_steps = 4
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 25
self.agent.replace_mse_with_huber_loss = True
class Breakout_Dueling_DDQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDQN, conf.Atari, conf.ExplorationParameters)
self.env.level = 'BreakoutDeterministic-v4'
self.agent.output_types = [conf.OutputTypes.DuelingQ]
self.agent.num_steps_between_copying_online_weights_to_target = 30000
self.learning_rate = 0.00025
self.agent.num_transitions_in_experience_replay = 1000000
self.exploration.initial_epsilon = 1.0
self.exploration.final_epsilon = 0.01
self.exploration.epsilon_decay_steps = 1000000
self.exploration.evaluation_policy = 'EGreedy'
self.exploration.evaluation_epsilon = 0.001
self.num_heatup_steps = 50000
self.agent.num_consecutive_playing_steps = 4
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 25
self.agent.replace_mse_with_huber_loss = True
class Alien_DQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DQN, conf.Atari, conf.ExplorationParameters)
self.env.level = 'AlienDeterministic-v4'
self.agent.num_steps_between_copying_online_weights_to_target = 10000
self.learning_rate = 0.00025
self.agent.num_transitions_in_experience_replay = 1000000
self.exploration.initial_epsilon = 1.0
self.exploration.final_epsilon = 0.1
self.exploration.epsilon_decay_steps = 1000000
self.exploration.evaluation_policy = 'EGreedy'
self.exploration.evaluation_epsilon = 0.05
self.num_heatup_steps = 50000
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 5
class Breakout_C51(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.CategoricalDQN, conf.Atari, conf.ExplorationParameters)
self.env.level = 'BreakoutDeterministic-v4'
self.agent.num_steps_between_copying_online_weights_to_target = 10000
self.learning_rate = 0.00025
self.agent.num_transitions_in_experience_replay = 1000000
self.exploration.initial_epsilon = 1.0
self.exploration.final_epsilon = 0.01
self.exploration.epsilon_decay_steps = 1000000
self.env.reward_clipping_max = 1.0
self.env.reward_clipping_min = -1.0
self.exploration.evaluation_policy = 'EGreedy'
self.exploration.evaluation_epsilon = 0.001
self.num_heatup_steps = 50000
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 5000000
class Breakout_QRDQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.QuantileRegressionDQN, conf.Atari, conf.ExplorationParameters)
self.env.level = 'BreakoutDeterministic-v4'
self.agent.num_steps_between_copying_online_weights_to_target = 10000
self.learning_rate = 0.00025
self.agent.num_transitions_in_experience_replay = 1000000
self.exploration.initial_epsilon = 1.0
self.exploration.final_epsilon = 0.01
self.exploration.epsilon_decay_steps = 1000000
self.exploration.evaluation_policy = 'EGreedy'
self.exploration.evaluation_epsilon = 0.001
self.num_heatup_steps = 50000
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 50
class Atari_DQN_TestBench(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DQN, conf.Atari, conf.ExplorationParameters)
self.env.level = 'BreakoutDeterministic-v4'
self.agent.num_steps_between_copying_online_weights_to_target = 10000
self.learning_rate = 0.00025
self.agent.num_transitions_in_experience_replay = 1000000
self.exploration.initial_epsilon = 1.0
self.exploration.final_epsilon = 0.1
self.exploration.epsilon_decay_steps = 1000000
self.exploration.evaluation_policy = 'EGreedy'
self.exploration.evaluation_epsilon = 0.05
self.num_heatup_steps = 10000
self.evaluation_episodes = 25
self.evaluate_every_x_episodes = 1000
self.num_training_iterations = 500
class Doom_Basic_PG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PolicyGradient, conf.Doom, conf.CategoricalExploration)
self.env.level = 'basic'
self.agent.policy_gradient_rescaler = 'FUTURE_RETURN_NORMALIZED_BY_TIMESTEP'
self.learning_rate = 0.00001
self.num_heatup_steps = 0
self.agent.beta_entropy = 0.01
class InvertedPendulum_PG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PolicyGradient, conf.GymVectorObservation, conf.AdditiveNoiseExploration)
self.env.level = 'InvertedPendulum-v1'
self.agent.policy_gradient_rescaler = 'FUTURE_RETURN_NORMALIZED_BY_TIMESTEP'
self.learning_rate = 0.001
self.num_heatup_steps = 0
class Pendulum_PG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PolicyGradient, conf.GymVectorObservation, conf.AdditiveNoiseExploration)
self.env.level = 'Pendulum-v0'
self.agent.policy_gradient_rescaler = 'FUTURE_RETURN_NORMALIZED_BY_TIMESTEP'
self.learning_rate = 0.001
self.num_heatup_steps = 0
self.agent.apply_gradients_every_x_episodes = 10
class Pendulum_DDPG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDPG, conf.GymVectorObservation, conf.AdditiveNoiseExploration)
self.env.level = 'Pendulum-v0'
self.learning_rate = 0.001
self.num_heatup_steps = 1000
self.env.normalize_observation = False
self.test = True
self.test_max_step_threshold = 100
self.test_min_return_threshold = -250
class InvertedPendulum_DDPG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDPG, conf.GymVectorObservation, conf.OUExploration)
self.env.level = 'InvertedPendulum-v1'
self.learning_rate = 0.00025
self.num_heatup_steps = 100
self.env.normalize_observation = True
class InvertedPendulum_PPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'InvertedPendulum-v1'
self.learning_rate = 0.001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 5000
self.agent.discount = 0.99
self.batch_size = 128
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.96
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.agent.shared_optimizer = False
self.agent.async_training = True
self.env.normalize_observation = True
class Pendulum_ClippedPPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'Pendulum-v0'
self.learning_rate = 0.00005
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
self.agent.beta_entropy = 0.01
class Hopper_DPPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'Hopper-v1'
self.learning_rate = 0.00001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 5000
self.agent.discount = 0.99
self.batch_size = 128
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.96
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.agent.async_training = True
self.env.normalize_observation = True
class InvertedPendulum_ClippedPPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'InvertedPendulum-v1'
self.learning_rate = 0.00005
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
class Humanoid_ClippedPPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'Humanoid-v1'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
class Hopper_ClippedPPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'Hopper-v1'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
class InvertedPendulum_ClippedPPO_Roboschool(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.Roboschool, conf.ExplorationParameters)
self.env.level = 'RoboschoolInvertedPendulum-v1'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
class HalfCheetah_ClippedPPO_Roboschool(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.Roboschool, conf.ExplorationParameters)
self.env.level = 'RoboschoolHalfCheetah-v1'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
class Hopper_ClippedPPO_Roboschool(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.Roboschool, conf.ExplorationParameters)
self.env.level = 'RoboschoolHopper-v1'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
class Ant_ClippedPPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'Ant-v1'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
class Hopper_ClippedPPO_Distributed(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'Hopper-v1'
self.learning_rate = 0.00001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 10000
self.agent.discount = 0.99
self.batch_size = 128
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'LBFGS'
self.env.normalize_observation = True
class Hopper_DDPG_Roboschool(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDPG, conf.Roboschool, conf.OUExploration)
self.env.level = 'RoboschoolHopper-v1'
self.learning_rate = 0.00025
self.num_heatup_steps = 100
class Hopper_PPO_Roboschool(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PPO, conf.Roboschool, conf.ExplorationParameters)
self.env.level = 'RoboschoolHopper-v1'
self.learning_rate = 0.001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 5000
self.agent.discount = 0.99
self.batch_size = 128
self.agent.policy_gradient_rescaler = 'GENERALIZED_ADVANTAGE_ESTIMATION'
self.agent.gae_lambda = 0.96
self.visualization.dump_csv = True
self.agent.optimizer_type = 'LBFGS'
class Hopper_DDPG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDPG, conf.GymVectorObservation, conf.OUExploration)
self.env.level = 'Hopper-v1'
self.learning_rate = 0.00025
self.num_heatup_steps = 100
self.env.normalize_observation = True
class Hopper_DDDPG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDDPG, conf.GymVectorObservation, conf.OUExploration)
self.env.level = 'Hopper-v1'
self.learning_rate = 0.00025
self.num_heatup_steps = 100
self.env.normalize_observation = True
class Hopper_PPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'Hopper-v1'
self.learning_rate = 0.001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 5000
self.agent.discount = 0.99
self.batch_size = 128
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.96
self.visualization.dump_csv = True
self.agent.optimizer_type = 'LBFGS'
# self.clip_gradients = 2
self.env.normalize_observation = True
class Walker_PPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.PPO, conf.GymVectorObservation, conf.AdditiveNoiseExploration)
self.env.level = 'Walker2d-v1'
self.learning_rate = 0.001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 5000
self.agent.discount = 0.99
self.batch_size = 128
self.agent.policy_gradient_rescaler = 'A_VALUE'
self.agent.gae_lambda = 0.96
self.visualization.dump_csv = True
self.agent.optimizer_type = 'LBFGS'
self.env.normalize_observation = True
class HalfCheetah_DDPG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDPG, conf.GymVectorObservation, conf.OUExploration)
self.env.level = 'HalfCheetah-v1'
self.learning_rate = 0.00025
self.num_heatup_steps = 1000
self.env.normalize_observation = True
class Ant_DDPG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDPG, conf.GymVectorObservation, conf.OUExploration)
self.env.level = 'Ant-v1'
self.learning_rate = 0.00025
self.num_heatup_steps = 1000
self.env.normalize_observation = True
class Pendulum_NAF(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NAF, conf.GymVectorObservation, conf.AdditiveNoiseExploration)
self.env.level = 'Pendulum-v0'
self.learning_rate = 0.001
self.num_heatup_steps = 1000
self.batch_size = 100
# self.env.reward_scaling = 1000
self.test = True
self.test_max_step_threshold = 100
self.test_min_return_threshold = -250
class InvertedPendulum_NAF(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NAF, conf.GymVectorObservation, conf.AdditiveNoiseExploration)
self.env.level = 'InvertedPendulum-v1'
self.learning_rate = 0.001
self.num_heatup_steps = 1000
self.batch_size = 100
class Hopper_NAF(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NAF, conf.GymVectorObservation, conf.AdditiveNoiseExploration)
self.env.level = 'Hopper-v1'
self.learning_rate = 0.0005
self.num_heatup_steps = 1000
self.batch_size = 100
self.agent.async_training = True
self.env.normalize_observation = True
class CartPole_NEC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NEC, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'CartPole-v0'
self.learning_rate = 0.00025
self.agent.num_episodes_in_experience_replay = 200
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 1000
self.exploration.final_epsilon = 0.1
self.agent.discount = 0.99
self.seed = 0
self.test = True
self.test_max_step_threshold = 200
self.test_min_return_threshold = 150
class Doom_Basic_NEC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NEC, conf.Doom, conf.ExplorationParameters)
self.env.level = 'basic'
self.learning_rate = 0.00001
self.agent.num_transitions_in_experience_replay = 100000
# self.exploration.initial_epsilon = 0.1 # TODO: try exploration
# self.exploration.final_epsilon = 0.1
# self.exploration.epsilon_decay_steps = 1000000
self.num_heatup_steps = 200
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 5
self.seed = 123
class Montezuma_NEC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NEC, conf.Atari, conf.ExplorationParameters)
self.env.level = 'MontezumaRevenge-v0'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.num_heatup_steps = 1000
self.agent.num_playing_steps_between_two_training_steps = 1
class Breakout_NEC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NEC, conf.Atari, conf.ExplorationParameters)
self.env.level = 'BreakoutDeterministic-v4'
self.agent.num_steps_between_copying_online_weights_to_target = 10000
self.learning_rate = 0.00001
self.agent.num_transitions_in_experience_replay = 1000000
self.exploration.initial_epsilon = 0.1
self.exploration.final_epsilon = 0.1
self.exploration.epsilon_decay_steps = 1000000
self.exploration.evaluation_policy = 'EGreedy'
self.exploration.evaluation_epsilon = 0.05
self.num_heatup_steps = 1000
self.env.reward_clipping_max = None
self.env.reward_clipping_min = None
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 25
self.seed = 123
class Doom_Health_NEC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NEC, conf.Doom, conf.ExplorationParameters)
self.env.level = 'HEALTH_GATHERING'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 10000
self.agent.num_playing_steps_between_two_training_steps = 1
class Doom_Health_DQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DQN, conf.Doom, conf.ExplorationParameters)
self.env.level = 'HEALTH_GATHERING'
self.agent.num_episodes_in_experience_replay = 200
self.learning_rate = 0.00025
self.num_heatup_steps = 1000
self.exploration.epsilon_decay_steps = 10000
self.agent.num_steps_between_copying_online_weights_to_target = 1000
class Pong_NEC_LSTM(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NEC, conf.Atari, conf.ExplorationParameters)
self.env.level = 'PongDeterministic-v4'
self.learning_rate = 0.001
self.agent.num_transitions_in_experience_replay = 1000000
self.agent.middleware_type = conf.MiddlewareTypes.LSTM
self.exploration.initial_epsilon = 0.5
self.exploration.final_epsilon = 0.1
self.exploration.epsilon_decay_steps = 1000000
self.num_heatup_steps = 500
class Pong_NEC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NEC, conf.Atari, conf.ExplorationParameters)
self.env.level = 'PongDeterministic-v4'
self.learning_rate = 0.00001
self.agent.num_transitions_in_experience_replay = 100000
self.exploration.initial_epsilon = 0.1 # TODO: try exploration
self.exploration.final_epsilon = 0.1
self.exploration.epsilon_decay_steps = 1000000
self.num_heatup_steps = 2000
self.env.reward_clipping_max = None
self.env.reward_clipping_min = None
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 5
self.env.crop_observation = True # TODO: remove
self.env.random_initialization_steps = 1 # TODO: remove
# self.seed = 123
class Alien_NEC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.NEC, conf.Atari, conf.ExplorationParameters)
self.env.level = 'AlienDeterministic-v4'
self.learning_rate = 0.0001
self.agent.num_transitions_in_experience_replay = 100000
self.exploration.initial_epsilon = 0.1 # TODO: try exploration
self.exploration.final_epsilon = 0.1
self.exploration.epsilon_decay_steps = 1000000
self.num_heatup_steps = 3000
self.env.reward_clipping_max = None
self.env.reward_clipping_min = None
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 5
self.seed = 123
class Pong_DQN(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DQN, conf.Atari, conf.ExplorationParameters)
self.env.level = 'PongDeterministic-v4'
self.agent.num_steps_between_copying_online_weights_to_target = 10000
self.learning_rate = 0.00025
self.agent.num_transitions_in_experience_replay = 1000000
self.exploration.initial_epsilon = 1.0
self.exploration.final_epsilon = 0.1
self.exploration.epsilon_decay_steps = 1000000
self.exploration.evaluation_policy = 'EGreedy'
self.exploration.evaluation_epsilon = 0.05
self.num_heatup_steps = 50000
self.evaluation_episodes = 1
self.evaluate_every_x_episodes = 5
self.seed = 123
class CartPole_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.CategoricalExploration)
self.env.level = 'CartPole-v0'
self.agent.policy_gradient_rescaler = 'GAE'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.env.reward_scaling = 200.
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.01
self.agent.num_steps_between_gradient_updates = 5
self.agent.middleware_type = conf.MiddlewareTypes.FC
self.test = True
self.test_max_step_threshold = 1000
self.test_min_return_threshold = 150
self.test_num_workers = 8
class MountainCar_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.CategoricalExploration)
self.env.level = 'MountainCar-v0'
self.agent.policy_gradient_rescaler = 'GAE'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.env.reward_scaling = 200.
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.01
self.agent.num_steps_between_gradient_updates = 5
self.agent.middleware_type = conf.MiddlewareTypes.FC
class InvertedPendulum_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.EntropyExploration)
self.env.level = 'InvertedPendulum-v1'
self.agent.policy_gradient_rescaler = 'A_VALUE'
self.agent.optimizer_type = 'Adam'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.env.reward_scaling = 200.
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 30
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.005
self.clip_gradients = 40
self.agent.middleware_type = conf.MiddlewareTypes.FC
class Hopper_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.EntropyExploration)
self.env.level = 'Hopper-v1'
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.optimizer_type = 'Adam'
self.learning_rate = 0.00002
self.num_heatup_steps = 0
self.env.reward_scaling = 20.
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 20
self.agent.gae_lambda = 0.98
self.agent.beta_entropy = 0.005
self.clip_gradients = 40
self.agent.middleware_type = conf.MiddlewareTypes.FC
class HopperIceWall_A3C(Hopper_A3C):
def __init__(self):
Hopper_A3C.__init__(self)
self.env.level = 'HopperIceWall-v0'
class HopperStairs_A3C(Hopper_A3C):
def __init__(self):
Hopper_A3C.__init__(self)
self.env.level = 'HopperStairs-v0'
class HopperBullet_A3C(Hopper_A3C):
def __init__(self):
Hopper_A3C.__init__(self)
self.env.level = 'HopperBulletEnv-v0'
class Kuka_ClippedPPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'KukaBulletEnv-v0'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
class Minitaur_ClippedPPO(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ClippedPPO, conf.GymVectorObservation, conf.ExplorationParameters)
self.env.level = 'MinitaurBulletEnv-v0'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.num_consecutive_training_steps = 1
self.agent.num_consecutive_playing_steps = 2048
self.agent.discount = 0.99
self.batch_size = 64
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.gae_lambda = 0.95
self.visualization.dump_csv = True
self.agent.optimizer_type = 'Adam'
self.env.normalize_observation = True
class Walker_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.EntropyExploration)
self.env.level = 'Walker2d-v1'
self.agent.policy_gradient_rescaler = 'A_VALUE'
self.agent.optimizer_type = 'Adam'
self.learning_rate = 0.00002
self.num_heatup_steps = 0
self.env.reward_scaling = 20.
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 20
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.005
self.clip_gradients = 40
self.agent.middleware_type = conf.MiddlewareTypes.FC
class Ant_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.EntropyExploration)
self.env.level = 'Ant-v1'
self.agent.policy_gradient_rescaler = 'A_VALUE'
self.agent.optimizer_type = 'Adam'
self.learning_rate = 0.00002
self.num_heatup_steps = 0
self.env.reward_scaling = 20.
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 20
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.005
self.clip_gradients = 40
self.agent.middleware_type = conf.MiddlewareTypes.FC
self.env.normalize_observation = True
class AntBullet_A3C(Ant_A3C):
def __init__(self):
Ant_A3C.__init__(self)
self.env.level = 'AntBulletEnv-v0'
class AntMaze_A3C(Ant_A3C):
def __init__(self):
Ant_A3C.__init__(self)
self.env.level = 'AntMaze-v0'
class Humanoid_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.EntropyExploration)
self.env.level = 'Humanoid-v1'
self.agent.policy_gradient_rescaler = 'A_VALUE'
self.agent.optimizer_type = 'Adam'
self.learning_rate = 0.00002
self.num_heatup_steps = 0
self.env.reward_scaling = 20.
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 20
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.005
self.clip_gradients = 40
self.agent.middleware_type = conf.MiddlewareTypes.FC
self.env.normalize_observation = True
class Pendulum_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.EntropyExploration)
self.env.level = 'Pendulum-v0'
self.agent.policy_gradient_rescaler = 'GAE'
self.agent.optimizer_type = 'Adam'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.agent.discount = 0.99
self.agent.num_steps_between_gradient_updates = 5
self.agent.gae_lambda = 1
class BipedalWalker_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.GymVectorObservation, conf.EntropyExploration)
self.env.level = 'BipedalWalker-v2'
self.agent.policy_gradient_rescaler = 'A_VALUE'
self.agent.optimizer_type = 'RMSProp'
self.learning_rate = 0.00002
self.num_heatup_steps = 0
self.env.reward_scaling = 50.
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 10
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.005
self.clip_gradients = None
self.agent.middleware_type = conf.MiddlewareTypes.FC
class Doom_Basic_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.Doom, conf.CategoricalExploration)
self.env.level = 'basic'
self.agent.policy_gradient_rescaler = 'GAE'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.env.reward_scaling = 100.
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 30
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.01
self.clip_gradients = 40
self.agent.middleware_type = conf.MiddlewareTypes.FC
class Pong_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.Atari, conf.CategoricalExploration)
self.env.level = 'PongDeterministic-v4'
self.agent.policy_gradient_rescaler = 'GAE'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
self.env.reward_scaling = 1.
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 20
self.agent.gae_lambda = 1.
self.agent.beta_entropy = 0.01
self.clip_gradients = 40.0
self.agent.middleware_type = conf.MiddlewareTypes.FC
class Breakout_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.Atari, conf.CategoricalExploration)
self.env.level = 'BreakoutDeterministic-v4'
self.agent.policy_gradient_rescaler = 'GAE'
self.learning_rate = 0.0001
self.num_heatup_steps = 200
self.env.reward_scaling = 1.
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 20
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.05
self.clip_gradients = 40.0
self.agent.middleware_type = conf.MiddlewareTypes.FC
class Carla_A3C(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.ActorCritic, conf.Carla, conf.EntropyExploration)
self.agent.embedder_complexity = conf.EmbedderComplexity.Deep
self.agent.policy_gradient_rescaler = 'GAE'
self.learning_rate = 0.0001
self.num_heatup_steps = 0
# self.env.reward_scaling = 1.0e9
self.agent.discount = 0.99
self.agent.apply_gradients_every_x_episodes = 1
self.agent.num_steps_between_gradient_updates = 30
self.agent.gae_lambda = 1
self.agent.beta_entropy = 0.01
self.clip_gradients = 40
self.agent.middleware_type = conf.MiddlewareTypes.FC
class Carla_DDPG(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.DDPG, conf.Carla, conf.OUExploration)
self.agent.embedder_complexity = conf.EmbedderComplexity.Deep
self.learning_rate = 0.0001
self.num_heatup_steps = 1000
self.agent.num_consecutive_training_steps = 5
class Carla_BC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.BC, conf.Carla, conf.ExplorationParameters)
self.agent.embedder_complexity = conf.EmbedderComplexity.Deep
self.agent.load_memory_from_file_path = 'datasets/carla_town1.p'
self.learning_rate = 0.0005
self.num_heatup_steps = 0
self.evaluation_episodes = 5
self.batch_size = 120
self.evaluate_every_x_training_iterations = 5000
class Doom_Basic_BC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.BC, conf.Doom, conf.ExplorationParameters)
self.env.level = 'basic'
self.agent.load_memory_from_file_path = 'datasets/doom_basic.p'
self.learning_rate = 0.0005
self.num_heatup_steps = 0
self.evaluation_episodes = 5
self.batch_size = 120
self.evaluate_every_x_training_iterations = 100
self.num_training_iterations = 2000
class Doom_Defend_BC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.BC, conf.Doom, conf.ExplorationParameters)
self.env.level = 'defend'
self.agent.load_memory_from_file_path = 'datasets/doom_defend.p'
self.learning_rate = 0.0005
self.num_heatup_steps = 0
self.evaluation_episodes = 5
self.batch_size = 120
self.evaluate_every_x_training_iterations = 100
class Doom_Deathmatch_BC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.BC, conf.Doom, conf.ExplorationParameters)
self.env.level = 'deathmatch'
self.agent.load_memory_from_file_path = 'datasets/doom_deathmatch.p'
self.learning_rate = 0.0005
self.num_heatup_steps = 0
self.evaluation_episodes = 5
self.batch_size = 120
self.evaluate_every_x_training_iterations = 100
class MontezumaRevenge_BC(conf.Preset):
def __init__(self):
conf.Preset.__init__(self, conf.BC, conf.Atari, conf.ExplorationParameters)
self.env.level = 'MontezumaRevenge-v0'
self.agent.load_memory_from_file_path = 'datasets/montezuma_revenge.p'
self.learning_rate = 0.0005
self.num_heatup_steps = 0
self.evaluation_episodes = 5
self.batch_size = 120
self.evaluate_every_x_training_iterations = 100
self.exploration.evaluation_epsilon = 0.05
self.exploration.evaluation_policy = 'EGreedy'
self.env.frame_skip = 1
Reference in New Issue View Git Blame Copy Permalink