1
0
mirror of https://github.com/gryf/coach.git synced 2025-12-17 19:20:19 +01:00

Save filters' internal state (#127)

* save filters internal state

* moving the restore to be made from within NumpyRunningStats
This commit is contained in:
Gal Leibovich
2018-11-20 17:21:48 +02:00
committed by GitHub
parent 67eb9e4c28
commit a112ee69f6
13 changed files with 173 additions and 14 deletions

View File

@@ -565,7 +565,7 @@ class GraphManager(object):
self.verify_graph_was_created()
# TODO: find better way to load checkpoints that were saved with a global network into the online network
if hasattr(self.task_parameters, 'checkpoint_restore_dir') and self.task_parameters.checkpoint_restore_dir:
if self.task_parameters.checkpoint_restore_dir:
checkpoint = get_checkpoint_state(self.task_parameters.checkpoint_restore_dir)
screen.log_title("Loading checkpoint: {}".format(checkpoint.model_checkpoint_path))
@@ -577,6 +577,8 @@ class GraphManager(object):
else:
raise ValueError('Invalid framework {}'.format(self.task_parameters.framework_type))
[manager.restore_checkpoint(self.task_parameters.checkpoint_restore_dir) for manager in self.level_managers]
def occasionally_save_checkpoint(self):
# only the chief process saves checkpoints
if self.task_parameters.checkpoint_save_secs \