bug fix - preventing crashes when the probability of one of the actions is 0 in the policy head

agents/actor_critic_agent.py

@@ -121,7 +121,7 @@ class ActorCriticAgent(PolicyOptimizationAgent):
             else:
                 action = np.argmax(action_probabilities)
             action_info = {"action_probability": action_probabilities[action], "state_value": state_value}
-            self.entropy.add_sample(-np.sum(action_probabilities * np.log(action_probabilities)))
+            self.entropy.add_sample(-np.sum(action_probabilities * np.log(action_probabilities + eps)))
         else:
             # CONTINUOUS
             state_value, action_values_mean, action_values_std = self.main_network.online_network.predict(observation)

agents/policy_gradients_agent.py

@@ -73,7 +73,7 @@ class PolicyGradientsAgent(PolicyOptimizationAgent):
             else:
                 action = np.argmax(action_values)
             action_value = {"action_probability": action_values[action]}
-            self.entropy.add_sample(-np.sum(action_values * np.log(action_values)))
+            self.entropy.add_sample(-np.sum(action_values * np.log(action_values + eps)))
         else:
             # CONTINUOUS
             result = self.main_network.online_network.predict(observation)

architectures/tensorflow_components/heads.py

@@ -177,7 +177,8 @@ class PolicyHead(Head):
             self.policy_mean = tf.nn.softmax(policy_values, name="policy")
 
             # define the distributions for the policy and the old policy
-            self.policy_distribution = tf.contrib.distributions.Categorical(probs=self.policy_mean)
+            # (the + eps is to prevent probability 0 which will cause the log later on to be -inf)
+            self.policy_distribution = tf.contrib.distributions.Categorical(probs=(self.policy_mean + eps))
             self.output = self.policy_mean
         else:
             # mean

utils.py

@@ -23,6 +23,7 @@ from subprocess import call, Popen
 
 killed_processes = []
 
+eps = np.finfo(np.float32).eps
 
 class Enum(object):
     def __init__(self):