coach v0.8.0

memories/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2017 Intel Corporation 
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from memories.differentiable_neural_dictionary import *
+from memories.episodic_experience_replay import *
+from memories.memory import *

memories/differentiable_neural_dictionary.py

@@ -0,0 +1,173 @@
+#
+# Copyright (c) 2017 Intel Corporation 
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import numpy as np
+from annoy import AnnoyIndex
+
+
+class AnnoyDictionary:
+    def __init__(self, dict_size, key_width, new_value_shift_coefficient=0.1, batch_size=100, key_error_threshold=0.01):
+        self.max_size = dict_size
+        self.curr_size = 0
+        self.new_value_shift_coefficient = new_value_shift_coefficient
+
+        self.index = AnnoyIndex(key_width, metric='euclidean')
+        self.index.set_seed(1)
+
+        self.embeddings = np.zeros((dict_size, key_width))
+        self.values = np.zeros(dict_size)
+
+        self.lru_timestamps = np.zeros(dict_size)
+        self.current_timestamp = 0.0
+
+        # keys that are in this distance will be considered as the same key
+        self.key_error_threshold = key_error_threshold
+
+        self.initial_update_size = batch_size
+        self.min_update_size = self.initial_update_size
+        self.key_dimension = key_width
+        self.value_dimension = 1
+        self._reset_buffer()
+
+        self.built_capacity = 0
+
+    def add(self, keys, values):
+        # Adds new embeddings and values to the dictionary
+        indices = []
+        indices_to_remove = []
+        for i in range(keys.shape[0]):
+            index = self._lookup_key_index(keys[i])
+            if index:
+                # update existing value
+                self.values[index] += self.new_value_shift_coefficient * (values[i] - self.values[index])
+                self.lru_timestamps[index] = self.current_timestamp
+                indices_to_remove.append(i)
+            else:
+                # add new
+                if self.curr_size >= self.max_size:
+                    # find the LRU entry
+                    index = np.argmin(self.lru_timestamps)
+                else:
+                    index = self.curr_size
+                    self.curr_size += 1
+                self.lru_timestamps[index] = self.current_timestamp
+                indices.append(index)
+
+        for i in reversed(indices_to_remove):
+            keys = np.delete(keys, i, 0)
+            values = np.delete(values, i, 0)
+
+        self.buffered_keys = np.vstack((self.buffered_keys, keys))
+        self.buffered_values = np.vstack((self.buffered_values, values))
+        self.buffered_indices = self.buffered_indices + indices
+
+        if len(self.buffered_indices) >= self.min_update_size:
+            self.min_update_size = max(self.initial_update_size, int(self.curr_size * 0.02))
+            self._rebuild_index()
+
+        self.current_timestamp += 1
+
+    # Returns the stored embeddings and values of the closest embeddings
+    def query(self, keys, k):
+        _, indices = self._get_k_nearest_neighbors_indices(keys, k)
+
+        embeddings = []
+        values = []
+        for ind in indices:
+            self.lru_timestamps[ind] = self.current_timestamp
+            embeddings.append(self.embeddings[ind])
+            values.append(self.values[ind])
+
+        self.current_timestamp += 1
+
+        return embeddings, values
+
+    def has_enough_entries(self, k):
+        return self.curr_size > k and (self.built_capacity > k)
+
+    def _get_k_nearest_neighbors_indices(self, keys, k):
+        distances = []
+        indices = []
+        for key in keys:
+            index, distance = self.index.get_nns_by_vector(key, k, include_distances=True)
+            distances.append(distance)
+            indices.append(index)
+        return distances, indices
+
+    def _rebuild_index(self):
+        self.index.unbuild()
+        self.embeddings[self.buffered_indices] = self.buffered_keys
+        self.values[self.buffered_indices] = np.squeeze(self.buffered_values)
+        for idx, key in zip(self.buffered_indices, self.buffered_keys):
+            self.index.add_item(idx, key)
+
+        self._reset_buffer()
+
+        self.index.build(50)
+        self.built_capacity = self.curr_size
+
+    def _reset_buffer(self):
+        self.buffered_keys = np.zeros((0, self.key_dimension))
+        self.buffered_values = np.zeros((0, self.value_dimension))
+        self.buffered_indices = []
+
+    def _lookup_key_index(self, key):
+        distance, index = self._get_k_nearest_neighbors_indices([key], 1)
+        if distance != [[]] and distance[0][0] <= self.key_error_threshold:
+            return index
+        return None
+
+
+class QDND:
+    def __init__(self, dict_size, key_width, num_actions, new_value_shift_coefficient=0.1, key_error_threshold=0.01):
+        self.num_actions = num_actions
+        self.dicts = []
+
+        # create a dict for each action
+        for a in range(num_actions):
+            new_dict = AnnoyDictionary(dict_size, key_width, new_value_shift_coefficient, key_error_threshold=key_error_threshold)
+            self.dicts.append(new_dict)
+
+    def add(self, embeddings, actions, values):
+        # add a new set of embeddings and values to each of the underlining dictionaries
+        embeddings = np.array(embeddings)
+        actions = np.array(actions)
+        values = np.array(values)
+        for a in range(self.num_actions):
+            idx = np.where(actions == a)
+            curr_action_embeddings = embeddings[idx]
+            curr_action_values = np.expand_dims(values[idx], -1)
+
+            self.dicts[a].add(curr_action_embeddings, curr_action_values)
+        return True
+
+    def query(self, embeddings, actions, k):
+        # query for nearest neighbors to the given embeddings
+        dnd_embeddings = []
+        dnd_values = []
+        for i, action in enumerate(actions):
+            embedding, value = self.dicts[action].query([embeddings[i]], k)
+            dnd_embeddings.append(embedding[0])
+            dnd_values.append(value[0])
+
+        return dnd_embeddings, dnd_values
+
+    def has_enough_entries(self, k):
+        # check if each of the action dictionaries has at least k entries
+        for a in range(self.num_actions):
+            if not self.dicts[a].has_enough_entries(k):
+                return False
+        return True

memories/episodic_experience_replay.py

@@ -0,0 +1,161 @@
+#
+# Copyright (c) 2017 Intel Corporation 
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from memories.memory import *
+import threading
+
+
+class EpisodicExperienceReplay(Memory):
+    def __init__(self, tuning_parameters):
+        """
+        :param tuning_parameters: A Preset class instance with all the running paramaters
+        :type tuning_parameters: Preset
+        """
+        Memory.__init__(self, tuning_parameters)
+        self.tp = tuning_parameters
+        self.max_size_in_episodes = tuning_parameters.agent.num_episodes_in_experience_replay
+        self.max_size_in_transitions = tuning_parameters.agent.num_transitions_in_experience_replay
+        self.discount = tuning_parameters.agent.discount
+        self.buffer = [Episode()]  # list of episodes
+        self.transitions = []
+        self._length = 1
+        self._num_transitions = 0
+        self._num_transitions_in_complete_episodes = 0
+        self.return_is_bootstrapped = tuning_parameters.agent.bootstrap_total_return_from_old_policy
+
+    def length(self):
+        """ Get the number of episodes in the ER (even if they are not complete) """
+        if self._length is not 0 and self.buffer[-1].is_empty():
+            return self._length - 1
+        return self._length
+
+    def num_complete_episodes(self):
+        """ Get the number of complete episodes in ER """
+        return self._length - 1
+
+    def num_transitions(self):
+        return self._num_transitions
+
+    def num_transitions_in_complete_episodes(self):
+        return self._num_transitions_in_complete_episodes
+
+    def sample_episode(self):
+        episode_idx = np.random.randint(self.num_complete_episodes())
+        return self.buffer[episode_idx]
+
+    def sample_n_episodes(self, n):
+        num_n_episodes = (self.num_complete_episodes()) / n
+        assert num_n_episodes > 0, \
+            'Tried sampling {} episodes when only {} completed episodes are available in the memory' \
+                .format(n, self.num_complete_episodes())
+        start_episode_idx = np.random.randint(num_n_episodes) * n
+        return self.buffer[start_episode_idx:(start_episode_idx + n)]
+
+    def sample_last_n_episodes(self, n):
+        num_n_episodes = (self.num_complete_episodes()) / n
+        assert num_n_episodes > 0, \
+            'Tried sampling {} episodes when only {} completed episodes are available in the memory' \
+                .format(n, self.num_complete_episodes())
+        start_episode_idx = -n
+        return self.buffer[start_episode_idx:(start_episode_idx + n)]
+
+    def sample(self, size):
+        assert self.num_transitions_in_complete_episodes() > size, \
+            'There are not enough transitions in the replay buffer'
+        batch = []
+        transitions_idx = np.random.randint(self.num_transitions_in_complete_episodes(), size=size)
+        for i in transitions_idx:
+            batch.append(self.transitions[i])
+
+        return batch
+
+    def enforce_length(self):
+        # clean up if necessary
+        if self.max_size_in_transitions is not None:
+            while self.max_size_in_transitions != 0 and self.num_transitions() > self.max_size_in_transitions:
+                self.remove_episode(0)
+        else:
+            while self.length() > self.max_size_in_episodes:
+                self.remove_episode(0)
+
+    def store(self, transition):
+        if len(self.buffer) == 0:
+            self.buffer.append(Episode())
+        last_episode = self.buffer[-1]
+        last_episode.insert(transition)
+        self.transitions.append(transition)
+        self._num_transitions += 1
+        if transition.game_over:
+            self._num_transitions_in_complete_episodes += last_episode.length()
+            self._length += 1
+            self.buffer[-1].update_returns(self.discount, is_bootstrapped=self.return_is_bootstrapped,
+                                           n_step_return=self.tp.agent.n_step)
+            self.buffer[-1].update_measurements_targets(self.tp.agent.num_predicted_steps_ahead)
+            # self.buffer[-1].update_actions_probabilities() # used for off-policy policy optimization
+            self.buffer.append(Episode())
+
+        self.enforce_length()
+
+    def insert_full_episode(self, episode):
+        # Do not add a new episode if the last one is not closed yet
+        if self.buffer[-1].get_last_transition().done != True:
+            return False
+
+        episode.update_returns(self.discount)
+        episode.update_measurements_targets(self.tp.agent.num_predicted_steps_ahead)
+        self.buffer.append(episode)
+        self.transitions += episode.transitions
+        self._length += 1
+        self._num_transitions += episode.length()
+
+        self.enforce_length()
+
+        return True
+
+    def get_episode(self, episode_index):
+        if self.length() == 0:
+            return None
+        episode = self.buffer[episode_index]
+        return episode
+
+    def remove_episode(self, episode_index):
+        if len(self.buffer) > episode_index:
+            episode_length = self.buffer[episode_index].length()
+            self._length -= 1
+            self._num_transitions -= episode_length
+            self._num_transitions_in_complete_episodes -= episode_length
+            del self.transitions[:episode_length]
+            del self.buffer[episode_index]
+
+    # for API compatibility
+    def get(self, index):
+        return self.get_episode(index)
+
+    def update_last_transition_info(self, info):
+        episode = self.buffer[-1]
+        if episode.length() == 0:
+            if len(self.buffer) < 2:
+                return
+            episode = self.buffer[-2]
+        for key, val in info.items():
+            episode.transitions[-1].info[key] = val
+
+    def clean(self):
+        self.transitions = []
+        self.buffer = [Episode()]
+        self._length = 1
+        self._num_transitions = 0
+        self._num_transitions_in_complete_episodes = 0

memories/memory.py

@@ -0,0 +1,135 @@
+#
+# Copyright (c) 2017 Intel Corporation 
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import numpy as np
+import copy
+from configurations import *
+
+
+class Memory:
+    def __init__(self, tuning_parameters):
+        """
+        :param tuning_parameters: A Preset class instance with all the running paramaters
+        :type tuning_parameters: Preset
+        """
+        pass
+
+    def store(self, obj):
+        pass
+
+    def get(self, index):
+        pass
+
+    def length(self):
+        pass
+
+    def sample(self, size):
+        pass
+
+    def clean(self):
+        pass
+
+
+class Episode:
+    def __init__(self):
+        self.transitions = []
+        # a num_transitions x num_transitions table with the n step return in the n'th row
+        self.returns_table = None
+        self._length = 0
+
+    def insert(self, transition):
+        self.transitions.append(transition)
+        self._length += 1
+
+    def is_empty(self):
+        return self.length() == 0
+
+    def length(self):
+        return self._length
+
+    def get_transition(self, transition_idx):
+        return self.transitions[transition_idx]
+
+    def get_last_transition(self):
+        return self.get_transition(-1)
+
+    def get_first_transition(self):
+        return self.get_transition(0)
+
+    def update_returns(self, discount, is_bootstrapped=False, n_step_return=-1):
+        if n_step_return == -1 or n_step_return > self.length():
+            n_step_return = self.length()
+        rewards = np.array([t.reward for t in self.transitions])
+        total_return = rewards.copy()
+        current_discount = discount
+        for i in range(1, n_step_return):
+            total_return += current_discount * np.pad(rewards[i:], (0, i), 'constant', constant_values=0)
+            current_discount *= discount
+
+        if is_bootstrapped:
+            bootstraps = np.array([np.squeeze(t.info['action_value']) for t in self.transitions[n_step_return:]])
+            total_return += current_discount * np.pad(bootstraps, (0, n_step_return), 'constant', constant_values=0)
+
+        for transition_idx in range(self.length()):
+            self.transitions[transition_idx].total_return = total_return[transition_idx]
+
+    def update_measurements_targets(self, num_steps):
+        if 'measurements' not in self.transitions[0].state:
+            return
+        measurements_size = self.transitions[0].state['measurements'].shape[-1]
+        total_return = sum([transition.reward for transition in self.transitions])
+        for transition_idx, transition in enumerate(self.transitions):
+            transition.info['future_measurements'] = np.zeros((num_steps, measurements_size))
+            for step in range(num_steps):
+                offset_idx = transition_idx + 2 ** step
+                if offset_idx >= self.length():
+                    offset_idx = -1
+                transition.info['future_measurements'][step] = self.transitions[offset_idx].next_state['measurements'] - \
+                                                               transition.state['measurements']
+            transition.info['total_episode_return'] = total_return
+
+    def update_actions_probabilities(self):
+        probability_product = 1
+        for transition_idx, transition in enumerate(self.transitions):
+            if 'action_probabilities' in transition.info.keys():
+                probability_product *= transition.info['action_probabilities']
+        for transition_idx, transition in enumerate(self.transitions):
+            transition.info['probability_product'] = probability_product
+
+    def get_returns_table(self):
+        return self.returns_table
+
+    def get_returns(self):
+        return [t.total_return for t in self.transitions]
+
+    def to_batch(self):
+        batch = []
+        for i in range(self.length()):
+            batch.append(self.get_transition(i))
+        return batch
+
+
+class Transition:
+    def __init__(self, state, action, reward, next_state, game_over):
+        self.state = copy.deepcopy(state)
+        self.state['observation'] = np.array(self.state['observation'], copy=False)
+        self.action = action
+        self.reward = reward
+        self.total_return = None
+        self.next_state = copy.deepcopy(next_state)
+        self.next_state['observation'] = np.array(self.next_state['observation'], copy=False)
+        self.game_over = game_over
+        self.info = {}