NCE unit test + torch.Timer

Author: nicholas-leonard

NCEModule.lua

@@ -255,4 +255,57 @@ function NCEModule:clearState()
    self.gradInput[2]:set()
 end
 
--- TODO : speedup unigram sampling using frequency bins...
+-- NOT IN USE : the following is experimental and not currently in use.
+-- ref.: https://hips.seas.harvard.edu/blog/2013/03/03/the-alias-method-efficient-sampling-with-many-discrete-outcomes/
+function NCEModule:aliasDraw(J, q)
+   local K  = J:nElement()
+
+   -- Draw from the overall uniform mixture.
+   local kk = math.random(1,K)
+
+   -- Draw from the binary mixture, either keeping the
+   -- small one, or choosing the associated larger one.
+   if math.random() < q[kk] then
+      return kk
+   else
+      return J[kk]
+   end
+end
+
+function NCEModule:aliasSetup(probs)
+   assert(probs:dim() == 1)
+   local K = probs:nElement()
+   local q = probs.new(K)
+   local J = torch.LongTensor(K):zero()
+
+   -- Sort the data into the outcomes with probabilities
+   -- that are larger and smaller than 1/K.
+   local smaller, larger = {}, {}
+   for kk = 1,K do
+      local prob = probs[kk]
+      q[kk] = K*prob
+      if q[kk] < 1 then
+         table.insert(smaller, kk)
+      else
+         table.insert(larger, kk)
+      end
+   end
+   
+   -- Loop though and create little binary mixtures that
+   -- appropriately allocate the larger outcomes over the
+   -- overall uniform mixture.
+   while #smaller > 0 and #larger > 0 do
+      local small = table.remove(smaller)
+      local large = table.remove(larger)
+
+      J[small] = large
+      q[large] = q[large] - (1.0 - q[small])
+
+      if q[large] < 1.0 then
+         table.insert(smaller,large)
+      else
+         table.insert(larger,large)
+      end
+   end
+   return J, q
+end

Sequential.lua

@@ -5,10 +5,10 @@ function Sequential:profile()
    function Sequential:updateOutput(input)
       local currentOutput = input
       for i=1,#self.modules do
-         local start = sys.clock()
+         local start = torch.Timer()
          currentOutput = self.modules[i]:updateOutput(currentOutput)
          if cutorch then cutorch.synchronize() end
-         print(torch.type(self.modules[i])..' updateOutput: '..sys.clock() - start.." s")
+         print(torch.type(self.modules[i])..' updateOutput: '..start:time().real.." s")
       end
       self.output = currentOutput
       return currentOutput
@@ -19,16 +19,16 @@ function Sequential:profile()
       local currentModule = self.modules[#self.modules]
       for i=#self.modules-1,1,-1 do
          local previousModule = self.modules[i]
-         local start = sys.clock()
+         local start = torch.Timer()
          currentGradOutput = currentModule:updateGradInput(previousModule.output, currentGradOutput)
          if cutorch then cutorch.synchronize() end
-         print(torch.type(currentModule)..' updateGradInput: '..sys.clock() - start.." s")
+         print(torch.type(currentModule)..' updateGradInput: '..start:time().real.." s")
          currentModule = previousModule
       end
-      local start = sys.clock()
+      local start = torch.Timer()
       currentGradOutput = currentModule:updateGradInput(input, currentGradOutput)
       if cutorch then cutorch.synchronize() end
-      print(torch.type(currentModule)..' updateGradInput: '..sys.clock() - start.." s")
+      print(torch.type(currentModule)..' updateGradInput: '..start:time().real.." s")
       self.gradInput = currentGradOutput
       return currentGradOutput
    end
@@ -40,18 +40,18 @@ function Sequential:profile()
       local currentModule = self.modules[#self.modules]
       for i=#self.modules-1,1,-1 do
          local previousModule = self.modules[i]
-         local start = sys.clock()
+         local start = torch.Timer()
          currentModule:accGradParameters(previousModule.output, currentGradOutput, scale)
          if cutorch then cutorch.synchronize() end
-         print(torch.type(currentModule)..' accGradParameters: '..sys.clock() - start.." s")
+         print(torch.type(currentModule)..' accGradParameters: '..start:time().real.." s")
          currentGradOutput = currentModule.gradInput
          currentModule = previousModule
       end
 
-      local start = sys.clock()
+      local start = torch.Timer()
       currentModule:accGradParameters(input, currentGradOutput, scale)
       if cutorch then cutorch.synchronize() end
-      print(torch.type(currentModule)..' accGradParameters: '..sys.clock() - start.." s")
+      print(torch.type(currentModule)..' accGradParameters: '..start:time().real.." s")
    end
 
    function Sequential:backward(input, gradOutput, scale)
@@ -60,17 +60,17 @@ function Sequential:profile()
       local currentModule = self.modules[#self.modules]
       for i=#self.modules-1,1,-1 do
          local previousModule = self.modules[i]
-         local start = sys.clock()
+         local start = torch.Timer()
          currentGradOutput = currentModule:backward(previousModule.output, currentGradOutput, scale)
          if cutorch then cutorch.synchronize() end
-         print(torch.type(currentModule)..' backward: '..sys.clock() - start.." s")
+         print(torch.type(currentModule)..' backward: '..start:time().real.." s")
          currentModule.gradInput = currentGradOutput
          currentModule = previousModule
       end
-      local start = sys.clock()
+      local start = torch.Timer()
       currentGradOutput = currentModule:backward(input, currentGradOutput, scale)
       if cutorch then cutorch.synchronize() end
-      print(torch.type(currentModule)..' backward: '..sys.clock() - start.." s")
+      print(torch.type(currentModule)..' backward: '..start:time().real.." s")
       self.gradInput = currentGradOutput
       return currentGradOutput
    end
@@ -80,18 +80,18 @@ function Sequential:profile()
       local currentModule = self.modules[#self.modules]
       for i=#self.modules-1,1,-1 do
          local previousModule = self.modules[i]
-         local start = sys.clock()
+         local start = torch.Timer()
          currentModule:accUpdateGradParameters(previousModule.output, currentGradOutput, lr)
          if cutorch then cutorch.synchronize() end
-         print(torch.type(currentModule)..' accUpdateGradParameters: '..sys.clock() - start.." s")
+         print(torch.type(currentModule)..' accUpdateGradParameters: '..start:time().real.." s")
          currentGradOutput = currentModule.gradInput
          currentModule = previousModule
       end
 
-      local start = sys.clock()
+      local start = torch.Timer()
       currentModule:accUpdateGradParameters(input, currentGradOutput, lr)
       if cutorch then cutorch.synchronize() end
-      print(torch.type(currentModule)..' accUpdateGradParameters: '..sys.clock() - start.." s")
+      print(torch.type(currentModule)..' accUpdateGradParameters: '..start:time().real.." s")
    end
 
    parent.profile(self)

test/test.lua

@@ -282,36 +282,37 @@ function dpnntest.Module_type()
       mytester:assertTensorEq(gradParams[i], gradParams2[i], 0.00001, " gradParams err "..i)
    end
 
+   local input = torch.randn(3,32,32)
+   local cnn = nn.Sequential()
+   cnn:add(nn.SpatialConvolution(3,8,5,5))
+   cnn:add(nn.ReLU())
+   cnn:add(nn.SpatialAveragePooling(2,2,2,2))
+   cnn:add(nn.SpatialConvolution(8,12,5,5))
+   cnn:add(nn.ReLU())
+   cnn:add(nn.SpatialAveragePooling(2,2,2,2))
+   local outsize = cnn:outside{1,3,32,32}
+   cnn:add(nn.Collapse(3))
+   cnn:add(nn.Linear(outsize[2]*outsize[3]*outsize[4],20))
+   cnn:add(nn.ReLU())
+   cnn:add(nn.Linear(20,10))
+   local output = cnn:forward(input):clone()
+   local gradOutput = output:clone()
+   local gradInput = cnn:backward(input, gradOutput):clone()
+   cnn:float()
+   local input3 = input:float()
+   local output3 = cnn:forward(input3):clone()
+   local gradOutput3 = output3:clone()
+   local gradInput3 = cnn:backward(input3, gradOutput3):clone()
+   local o1, o2 = output3:float(), output:float()
+   mytester:assertTensorEq(o1, o2, 0.000001)
+   mytester:assertTensorEq(gradInput3:float(), gradInput:float(), 0.00001, "type float bwd err") 
    if pcall(function() require 'cunn' end) then
-      local input = torch.randn(3,32,32)
-      local cnn = nn.Sequential()
-      cnn:add(nn.SpatialConvolutionMM(3,8,5,5))
-      cnn:add(nn.ReLU())
-      cnn:add(nn.SpatialAveragePooling(2,2,2,2))
-      cnn:add(nn.SpatialConvolutionMM(8,12,5,5))
-      cnn:add(nn.ReLU())
-      cnn:add(nn.SpatialAveragePooling(2,2,2,2))
-      local outsize = cnn:outside{1,3,32,32}
-      cnn:add(nn.Collapse(3))
-      cnn:add(nn.Linear(outsize[2]*outsize[3]*outsize[4],20))
-      cnn:add(nn.ReLU())
-      cnn:add(nn.Linear(20,10))
-      local output = cnn:forward(input):clone()
-      local gradOutput = output:clone()
-      local gradInput = cnn:backward(input, gradOutput):clone()
-      cnn:float()
-      local input3 = input:float()
-      local output3 = cnn:forward(input3):clone()
-      local gradOutput3 = output3:clone()
-      local gradInput3 = cnn:backward(input3, gradOutput3):clone()
-      mytester:assertTensorEq(output3:float(), output:float(), 0.000001, "type float fwd err")
-      mytester:assertTensorEq(gradInput3:float(), gradInput:float(), 0.00001, "type float bwd err") 
       cnn:cuda()
       local input2 = input3:cuda()
       local gradOutput2 = gradOutput3:cuda()
       local output2 = cnn:forward(input2)
       local gradInput2 = cnn:backward(input2, gradOutput2)
-      mytester:assertTensorEq(output2:float(), output3, 0.000001, "type cuda fwd err")
+      mytester:assertTensorEq(output2:float(), output3, 0.000001)
       mytester:assertTensorEq(gradInput2:float(), gradInput3, 0.00001, "type cuda bwd err") 
    end
 end