gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2026-02-16 14:05:46 +01:00
Code Issues Projects Releases Wiki Activity

Create a dataset using an agent (#306)

Browse Source 
Generate a dataset using an agent (allowing to select between this and a random dataset)
This commit is contained in:
Gal Leibovich
2019-05-28 09:34:49 +03:00
committed by GitHub
parent 342b7184bc
commit 9e9c4fd332
26 changed files with 351 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
rl_coach/presets/CartPole_DQN_BatchRL_BCQ.py
View File

@@ -1,4 +1,5 @@
from copy import deepcopy
from rl_coach.agents.dqn_agent import DQNAgentParameters
from rl_coach.architectures.tensorflow_components.layers import Dense
from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters
@@ -45,10 +46,10 @@ agent_params.network_wrappers['main'].batch_size = 128
agent_params.algorithm.num_steps_between_copying_online_weights_to_target = TrainingSteps(
DATASET_SIZE / agent_params.network_wrappers['main'].batch_size)
#
# agent_params.algorithm.num_steps_between_copying_online_weights_to_target = TrainingSteps(
# 3)
agent_params.algorithm.num_steps_between_copying_online_weights_to_target = TrainingSteps(
100)
# agent_params.algorithm.num_steps_between_copying_online_weights_to_target = EnvironmentSteps(100)
agent_params.algorithm.num_consecutive_playing_steps = EnvironmentSteps(0)
agent_params.algorithm.discount = 0.98
# can use either a kNN or a NN based model for predicting which actions not to max over in the bellman equation
@@ -79,17 +80,49 @@ agent_params.network_wrappers['imitation_model'].middleware_parameters.scheme =
# ER size
agent_params.memory = EpisodicExperienceReplayParameters()
agent_params.memory.max_size = (MemoryGranularity.Transitions, DATASET_SIZE)
# E-Greedy schedule
agent_params.exploration.epsilon_schedule = LinearSchedule(0, 0, 10000)
agent_params.exploration.evaluation_epsilon = 0
# Input filtering
agent_params.input_filter = InputFilter()
agent_params.input_filter.add_reward_filter('rescale', RewardRescaleFilter(1/200.))
# Experience Generating Agent parameters
experience_generating_agent_params = DQNAgentParameters()
# schedule parameters
experience_generating_schedule_params = ScheduleParameters()
experience_generating_schedule_params.heatup_steps = EnvironmentSteps(1000)
experience_generating_schedule_params.improve_steps = TrainingSteps(
DATASET_SIZE - experience_generating_schedule_params.heatup_steps.num_steps)
experience_generating_schedule_params.steps_between_evaluation_periods = EnvironmentEpisodes(10)
experience_generating_schedule_params.evaluation_steps = EnvironmentEpisodes(1)
# DQN params
experience_generating_agent_params.algorithm.num_steps_between_copying_online_weights_to_target = EnvironmentSteps(100)
experience_generating_agent_params.algorithm.discount = 0.99
experience_generating_agent_params.algorithm.num_consecutive_playing_steps = EnvironmentSteps(1)
# NN configuration
experience_generating_agent_params.network_wrappers['main'].learning_rate = 0.00025
experience_generating_agent_params.network_wrappers['main'].replace_mse_with_huber_loss = False
# ER size
experience_generating_agent_params.memory = EpisodicExperienceReplayParameters()
experience_generating_agent_params.memory.max_size = \
(MemoryGranularity.Transitions,
experience_generating_schedule_params.heatup_steps.num_steps +
experience_generating_schedule_params.improve_steps.num_steps)
# E-Greedy schedule
experience_generating_agent_params.exploration.epsilon_schedule = LinearSchedule(1.0, 0.01, 10000)
################
# Environment #
################
@@ -101,11 +134,14 @@ env_params = GymVectorEnvironment(level='CartPole-v0')
preset_validation_params = PresetValidationParameters()
preset_validation_params.test = True
preset_validation_params.min_reward_threshold = 150
preset_validation_params.max_episodes_to_achieve_reward = 2000
preset_validation_params.max_episodes_to_achieve_reward = 50
graph_manager = BatchRLGraphManager(agent_params=agent_params, env_params=env_params,
graph_manager = BatchRLGraphManager(agent_params=agent_params,
experience_generating_agent_params=experience_generating_agent_params,
experience_generating_schedule_params=experience_generating_schedule_params,
env_params=env_params,
schedule_params=schedule_params,
vis_params=VisualizationParameters(dump_signals_to_csv_every_x_episodes=1),
preset_validation_params=preset_validation_params,
reward_model_num_epochs=30,
train_to_eval_ratio=0.8)
train_to_eval_ratio=0.4)