gryf/coach
1
0
Fork 0
mirror of https://github.com/gryf/coach.git synced 2025-12-18 19:50:17 +01:00
Code Issues Projects Releases Wiki Activity

S3 optimization - save only the latest checkpoint. (#148)

Browse Source
This commit is contained in:
Balaji Subramaniam
2018-11-23 22:17:36 -08:00
committed by Ajay Deshpande
parent 13d2679af4
commit bf2036b284
1 changed files with 17 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
rl_coach/data_stores/s3_data_store.py
View File

@@ -54,15 +54,20 @@ class S3DataStore(DataStore):
try: try:
# remove lock file if it exists # remove lock file if it exists
self.mc.remove_object(self.params.bucket_name, SyncFiles.LOCKFILE.value) self.mc.remove_object(self.params.bucket_name, SyncFiles.LOCKFILE.value)
# Acquire lock # Acquire lock
self.mc.put_object(self.params.bucket_name, SyncFiles.LOCKFILE.value, io.BytesIO(b''), 0) self.mc.put_object(self.params.bucket_name, SyncFiles.LOCKFILE.value, io.BytesIO(b''), 0)
state_file = CheckpointStateFile(os.path.abspath(self.params.checkpoint_dir))
if state_file.exists():
ckpt_state = state_file.read()
checkpoint_file = None checkpoint_file = None
for root, dirs, files in os.walk(self.params.checkpoint_dir): for root, dirs, files in os.walk(self.params.checkpoint_dir):
for filename in files: for filename in files:
if filename == CheckpointStateFile.checkpoint_state_filename: if filename == CheckpointStateFile.checkpoint_state_filename:
checkpoint_file = (root, filename) checkpoint_file = (root, filename)
continue continue
if filename.startswith(ckpt_state.name):
abs_name = os.path.abspath(os.path.join(root, filename)) abs_name = os.path.abspath(os.path.join(root, filename))
rel_name = os.path.relpath(abs_name, self.params.checkpoint_dir) rel_name = os.path.relpath(abs_name, self.params.checkpoint_dir)
self.mc.fput_object(self.params.bucket_name, rel_name, abs_name) self.mc.fput_object(self.params.bucket_name, rel_name, abs_name)