FIXED: clipping for numerical stability in SoftmaxLayer was not equivalent to that of PyBrain.

Author: bayerj

src/cpp/structure/modules/softmax.cpp

@@ -18,13 +18,17 @@ SoftmaxLayer::_forward()
     double* output_p = output()[timestep()];
     for(int i = 0; i < _insize; i++)
     {
-        double item = exp(input_p[i]);
-        item = item < -500 ? -500 : item;
-        item = item > 500 ? 500 : item;
+        // Clip of input argument if its to extreme to avoid NaNs and inf as a
+        // result of exp().
+        double inpt;
+        inpt = input_p[i] < -500 ? -500 : input_p[i];
+        inpt = inpt > 500 ? 500 : inpt;
+        double item = exp(inpt);
+
         sum += item;
         output_p[i] = item;
     }
-    for(int i = 0; i < _outsize; i++)
+    for(int i = 0; i < _insize; i++)
     {
         output_p[i] /= sum;
     }
@@ -38,4 +42,4 @@ SoftmaxLayer::_backward()
     void* sourcebuffer_p = (void*) outerror()[timestep() - 1];
     void* sinkbuffer_p = (void*) inerror()[timestep() - 1];
     memcpy(sinkbuffer_p, sourcebuffer_p, size);
-}
+}

src/cpp/tests/test_structure.cpp

@@ -417,25 +417,31 @@ TEST(TestModules, TanhLayer) {
 
 
 TEST(TestModules, SoftmaxLayer) {
-    SoftmaxLayer* layer_p = new SoftmaxLayer(2);
+    SoftmaxLayer* layer_p = new SoftmaxLayer(3);
 
     double* input_p = new double[2];
-    input_p[0] = 2.;
-    input_p[1] = 4.;
+    input_p[0] = 4.6296992222786457;
+    input_p[1] = -0.36272901550781184;
+    input_p[2] = 15.440919648395607;
 
     layer_p->add_to_input(input_p);
 
-    ASSERT_DOUBLE_EQ(2, layer_p->input()[0][0])
+    ASSERT_DOUBLE_EQ(input_p[0], layer_p->input()[0][0])
         << "add_to_input not working.";
-    ASSERT_DOUBLE_EQ(4, layer_p->input()[0][1])
+    ASSERT_DOUBLE_EQ(input_p[1], layer_p->input()[0][1])
+        << "add_to_input not working.";
+    ASSERT_DOUBLE_EQ(input_p[2], layer_p->input()[0][2])
         << "add_to_input not working.";
 
     layer_p->forward();
 
-    ASSERT_DOUBLE_EQ(0.11920292202211756, layer_p->output()[0][0])
+    ASSERT_DOUBLE_EQ(2.0171481969464377e-05, layer_p->output()[0][0])
         << "Forward pass incorrect.";
 
-    ASSERT_DOUBLE_EQ(0.88079707797788243, layer_p->output()[0][1])
+    ASSERT_DOUBLE_EQ(1.3694739368803625e-07, layer_p->output()[0][1])
+        << "Forward pass incorrect.";
+    
+    ASSERT_DOUBLE_EQ(0.99997969157063693, layer_p->output()[0][2])
         << "Forward pass incorrect.";
 
     double* outerror_p = new double[2];