pre-release 0.10.0

rl_coach/schedules.py

@@ -0,0 +1,125 @@
+#
+# Copyright (c) 2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import List, Tuple
+
+import numpy as np
+
+from rl_coach.core_types import EnvironmentSteps
+
+
+class Schedule(object):
+    def __init__(self, initial_value: float):
+        self.initial_value = initial_value
+        self.current_value = initial_value
+
+    def step(self):
+        raise NotImplementedError("")
+
+
+class ConstantSchedule(Schedule):
+    def __init__(self, initial_value: float):
+        super().__init__(initial_value)
+
+    def step(self):
+        pass
+
+
+class LinearSchedule(Schedule):
+    """
+    A simple linear schedule which decreases or increases over time from an initial to a final value
+    """
+    def __init__(self, initial_value: float, final_value: float, decay_steps: int):
+        """
+        :param initial_value: the initial value
+        :param final_value: the final value
+        :param decay_steps: the number of steps that are required to decay the initial value to the final value
+        """
+        super().__init__(initial_value)
+        self.final_value = final_value
+        self.decay_steps = decay_steps
+        self.decay_delta = (initial_value - final_value) / float(decay_steps)
+
+    def step(self):
+        self.current_value -= self.decay_delta
+        # decreasing schedule
+        if self.final_value < self.initial_value:
+            self.current_value = np.clip(self.current_value, self.final_value, self.initial_value)
+        # increasing schedule
+        if self.final_value > self.initial_value:
+            self.current_value = np.clip(self.current_value, self.initial_value, self.final_value)
+
+
+class PieceWiseSchedule(Schedule):
+    """
+    A schedule which consists of multiple sub-schedules, where each one is used for a defined number of steps
+    """
+    def __init__(self, schedules: List[Tuple[Schedule, EnvironmentSteps]]):
+        """
+        :param schedules: a list of schedules to apply serially. Each element of the list should be a tuple of
+                          2 elements - a schedule and the number of steps to run it in terms of EnvironmentSteps
+        """
+        super().__init__(schedules[0][0].initial_value)
+        self.schedules = schedules
+        self.current_schedule = schedules[0]
+        self.current_schedule_idx = 0
+        self.current_schedule_step_count = 0
+
+    def step(self):
+        self.current_schedule[0].step()
+
+        if self.current_schedule_idx < len(self.schedules) - 1 \
+                and self.current_schedule_step_count >= self.current_schedule[1].num_steps:
+            self.current_schedule_idx += 1
+            self.current_schedule = self.schedules[self.current_schedule_idx]
+            self.current_schedule_step_count = 0
+
+        self.current_value = self.current_schedule[0].current_value
+        self.current_schedule_step_count += 1
+
+
+class ExponentialSchedule(Schedule):
+    """
+    A simple exponential schedule which decreases or increases over time from an initial to a final value
+    """
+    def __init__(self, initial_value: float, final_value: float, decay_coefficient: float):
+        """
+        :param initial_value: the initial value
+        :param final_value: the final value
+        :param decay_coefficient: the exponential decay coefficient
+        """
+        super().__init__(initial_value)
+        self.initial_value = initial_value
+        self.final_value = final_value
+        self.decay_coefficient = decay_coefficient
+        self.current_step = 0
+        self.current_value = self.initial_value
+        if decay_coefficient < 1 and final_value > initial_value:
+            raise ValueError("The final value should be lower than the initial value when the decay coefficient < 1")
+        if decay_coefficient > 1 and initial_value > final_value:
+            raise ValueError("The final value should be higher than the initial value when the decay coefficient > 1")
+
+    def step(self):
+        self.current_value *= self.decay_coefficient
+
+        # decreasing schedule
+        if self.final_value < self.initial_value:
+            self.current_value = np.clip(self.current_value, self.final_value, self.initial_value)
+        # increasing schedule
+        if self.final_value > self.initial_value:
+            self.current_value = np.clip(self.current_value, self.initial_value, self.final_value)
+
+        self.current_step += 1