Add a few more command line options

New options:
  --gradint-clipping (alters GC threshold)
  --learn-recurrent (converts RNN to ESN)
  --learn-embeddings (could be useful for fine tuning)

Author: Anton Bakhtin

AUTHORS

@@ -1,4 +1,4 @@
 The following authors have created the source code of "Faster RNNLM"
 published and distributed by YANDEX LLC as the owner:
 Anton Bakhtin <[email protected]>
-Ilia Edrenkin <[email protected]>
+Ilya Edrenkin <[email protected]>

README.md

@@ -17,7 +17,7 @@ Run `./build.sh` to download Eigen library and build faster-rnnlm.
 
 To train a simple model with GRU hidden unit and Noise Contrastive Estimation, use the following command:
 
-   `./rnnlm -rnnlm model_name -train train.txt -valid validation.txt -hidden 128 -hidden-type gru -nce 20 -alpha 0.01 -rmsprop 0.9`
+   `./rnnlm -rnnlm model_name -train train.txt -valid validation.txt -hidden 128 -hidden-type gru -nce 20 -alpha 0.01`
 
 Files train.txt and test.txt must contain one sentence per line. All distinct words that are found in the training file will be used for the nnet vocab, their counts will determine Huffman tree structure and remain fixed for this nnet. If you prefer using limited vocabulary (say, top 1 million words) you should map all other words to <unk> or another token of your choice. Limited vocabulary is usually a good idea if it helps you to have enough training examples for each word.
 
@@ -138,6 +138,12 @@ Optimization options
   --rmsprop <float>
     RMSprop coefficient; rmsprop=1 disables rmsprop and rmsprop=0 equivalent to RMS
     (default: 1)
+  --gradient-clipping <float>
+    Clip updates above the value (default: 1)
+  --learn-recurrent (0 | 1)
+    Learn hidden layer weights (default: 1)
+  --learn-embeddings (0 | 1)
+    Learn embedding weights (default: 1)
   --alpha <float>
     Learning rate for recurrent and embedding weights (default: 0.1)
   --maxent-alpha <float>

faster-rnnlm/Makefile

@@ -4,6 +4,7 @@ CC = g++
 CFLAGS = -Wall -march=native -funroll-loops -g -D__STDC_FORMAT_MACROS
 CFLAGS += -I../ -DEIGEN_DONT_PARALLELIZE # for Eigen
 CFLAGS += $(shell $(CC) -dumpversion | awk '{if(NR==1 && $$1>="4.6") print "-Ofast -Wno-unused-result"; else print "-O3";}')
+NVCC_CFLAGS = -O3 -march=native -funroll-loops
 LDFLAGS = -lm -lrt
 ifeq ($(NOTHREAD), 1)
 	CFLAGS += -DNOTHREAD
@@ -51,7 +52,7 @@ nnet.o : nnet.cc nnet.h maxent.h settings.h hierarchical_softmax.h words.h recur
 	$(CC) $< -c -o $@ $(CFLAGS)
 
 cuda_softmax.o : cuda_softmax.cu cuda_softmax.h settings.h
-	nvcc $< -c -Xcompiler "-Ofast -march=native -funroll-loops" -o $@
+	nvcc $< -c -Xcompiler "$(NVCC_CFLAGS)" -o $@
 
 
 clean:

faster-rnnlm/cuda_softmax.cu

@@ -26,7 +26,6 @@ void AssertCudaSuccessLast(const char* message) {
   return AssertCudaSuccess(cudaGetLastError(), message);
 }
 
-extern "C"
 void InitCudaStorage(CudaStorage* cust, size_t layer_size, size_t vocab_size, size_t maxent_hash_size, Real lnz) {
   cust->layer_size = layer_size;
   cust->vocab_size = vocab_size;
@@ -88,7 +87,6 @@ void InitCudaStorage(CudaStorage* cust, size_t layer_size, size_t vocab_size, si
   }
 }
 
-extern "C"
 void FreeCudaStorage(CudaStorage* cust) {
   cudaFree(cust->sm_embedding);
   if (cust->maxent_hash_size != 0) {
@@ -115,7 +113,6 @@ void FreeCudaStorage(CudaStorage* cust) {
   delete cust->inner;
 }
 
-extern "C"
 void UploadNetWeights(CudaStorage* cust, const Real* sm_embedding_cpu, const Real* maxent_cpu) {
   cudaMemcpy(cust->sm_embedding, sm_embedding_cpu, cust->layer_size * cust->vocab_size * sizeof(Real), cudaMemcpyHostToDevice);
   cudaMemcpy(cust->maxent, maxent_cpu, cust->maxent_hash_size * sizeof(Real), cudaMemcpyHostToDevice);
@@ -174,7 +171,6 @@ void CublasMultiply_A_BT(cublasHandle_t* handle, float beta, int rows_a, int row
             dev_c, rows_b);
 }
 
-extern "C"
 void CalculateSoftMax(
     CudaStorage* cust, const Real* hidden_layers,
     const uint64_t* maxent_indices_all, const int* maxent_indices_count_all,

faster-rnnlm/cuda_softmax.h

@@ -33,16 +33,12 @@ struct CudaStorage {
   CudaStorageInner* inner;
 };
 
-extern "C"
 void InitCudaStorage(CudaStorage* cust, size_t layer_size, size_t vocab_size, size_t maxent_hash_size, Real lnz);
 
-extern "C"
 void FreeCudaStorage(CudaStorage* cust);
 
-extern "C"
 void UploadNetWeights(CudaStorage* cust, const Real* sm_embedding_cpu, const Real* maxent);
 
-extern "C"
 void CalculateSoftMax(CudaStorage* cust, const Real* hidden_layers, const uint64_t* maxent_indices_all, const int* maxent_indices_count_all, size_t sentence_length, const WordIndex* sen, Real* logprobs);
 
 #endif  // FASTER_RNNLM_CUDA_SOFTMAX_H_

faster-rnnlm/hierarchical_softmax.cc

@@ -346,7 +346,7 @@ inline void PropagateNodeForwardByDepth(
 inline void PropagateNodeBackward(
     HSTree* hs, WordIndex target_word, int depth,
     const uint64_t* feature_hashes, int maxent_order,
-    Real lrate, Real maxent_lrate, Real l2reg, Real maxent_l2reg,
+    Real lrate, Real maxent_lrate, Real l2reg, Real maxent_l2reg, Real gradient_clipping,
     const double* state,
     const Real* hidden,
     Real* hidden_grad, MaxEnt* maxent
@@ -357,33 +357,32 @@ inline void PropagateNodeBackward(
   Real branch_gradient[ARITY - 1];
   for (int branch = 0; branch < ARITY - 1; ++branch) {
     const int match = (branch == selected_branch);
-    // gradient of softmax
-    // g = branch_softmax_prob[selected_branch] * (match - branch_softmax_prob[branch]);
     // gradient of logsoftmax
-    // g /= branch_softmax_prob[selected_branch];
-    Real g = (match - (Real) state[branch]);
-    // gradient clipping
-    g = Clip(g, GRAD_CLIPPING);
-    branch_gradient[branch] = g;
+    branch_gradient[branch] = (match - static_cast<Real>(state[branch]));
   }
 
   for (int branch = 0; branch < ARITY - 1; ++branch) {
-    Real g = branch_gradient[branch];
+    Real grad = branch_gradient[branch];
     int child_offset = hs->tree_->GetChildOffsetByDepth(target_word, depth, branch);
+    Real* sm_embedding = hs->weights_.row(child_offset).data();
 
     // Propagate errors output -> hidden
     for (int i = 0; i < hs->layer_size; ++i) {
-      hidden_grad[i] += g * hs->weights_(child_offset, i);
+      hidden_grad[i] += grad * sm_embedding[i];
     }
 
     // Learn weights hidden -> output
     for (int i = 0; i < hs->layer_size; ++i) {
-      Real update = g * lrate * hidden[i] - l2reg * hs->weights_(child_offset, i);
-      hs->weights_(child_offset, i) += Clip(update, GRAD_CLIPPING);
+      Real update = grad * hidden[i];
+      sm_embedding[i] *= (1 - l2reg);
+      sm_embedding[i] += lrate * Clip(update, gradient_clipping);
     }
+
+    // update maxent weights
+    Real maxent_grad = Clip(grad, gradient_clipping);
     for (int order = 0; order < maxent_order; ++order) {
       uint64_t maxent_index = feature_hashes[order] + child_offset;
-      maxent->UpdateValue(maxent_index, maxent_lrate, g, maxent_l2reg);
+      maxent->UpdateValue(maxent_index, maxent_lrate, maxent_grad, maxent_l2reg);
     }
   }
 }
@@ -392,7 +391,7 @@ inline void PropagateNodeBackward(
 Real HSTree::PropagateForwardAndBackward(
     bool calculate_probability, WordIndex target_word,
     const uint64_t* feature_hashes, int maxent_order,
-    Real lrate, Real maxent_lrate, Real l2reg, Real maxent_l2reg,
+    Real lrate, Real maxent_lrate, Real l2reg, Real maxent_l2reg, Real gradient_clipping,
     const Real* hidden,
     Real* hidden_grad, MaxEnt* maxent
     ) {
@@ -406,7 +405,7 @@ Real HSTree::PropagateForwardAndBackward(
 
     PropagateNodeBackward(
         this, target_word, depth, feature_hashes, maxent_order,
-        lrate, maxent_lrate, l2reg, maxent_l2reg,
+        lrate, maxent_lrate, l2reg, maxent_l2reg, gradient_clipping,
         softmax_state,
         hidden, hidden_grad, maxent);
 

faster-rnnlm/hierarchical_softmax.h

@@ -50,7 +50,7 @@ class HSTree {
   Real PropagateForwardAndBackward(
       bool calculate_probability, WordIndex target_word,
       const uint64_t* feature_hashes, int maxent_order,
-      Real lrate, Real maxent_lrate, Real l2reg, Real maxent_l2reg,
+      Real lrate, Real maxent_lrate, Real l2reg, Real maxent_l2reg, Real gradient_clipping,
       const Real* hidden,
       Real* hidden_grad, MaxEnt* maxent);
 

faster-rnnlm/nce.cc

@@ -92,7 +92,7 @@ void NCE::Updater::PropagateForwardAndBackward(
     const Ref<const RowVector> hidden, WordIndex target_word,
     const uint64_t* maxent_indices, size_t maxent_size,
     const NoiseSample& sample, Real lrate, Real l2reg,
-    Real maxent_lrate, Real maxent_l2reg,
+    Real maxent_lrate, Real maxent_l2reg, Real gradient_clipping,
     Ref<RowVector> hidden_grad, MaxEnt* maxent) {
 
   Real ln_sample_size = log(sample.size);
@@ -116,14 +116,15 @@ void NCE::Updater::PropagateForwardAndBackward(
 
     // update softmax weights
     embedding_grad_.noalias() = grad * hidden;
-    ClipMatrix(embedding_grad_);
+    ClipMatrix(embedding_grad_, gradient_clipping);
     nce_->sm_embedding_.row(word) *= (1 - l2reg);
     nce_->sm_embedding_.row(word) += embedding_grad_ * lrate;
 
-    // update softmax weights
+    // update maxent weights
+    Real maxent_grad = Clip(grad, gradient_clipping);
     for (size_t i = 0; i < maxent_size; i++) {
       uint64_t maxent_index = get_maxent_index(maxent_indices[i], nce_->maxent_hash_size_, word);
-      maxent->UpdateValue(maxent_index, maxent_lrate, grad, maxent_l2reg);
+      maxent->UpdateValue(maxent_index, maxent_lrate, maxent_grad, maxent_l2reg);
     }
   }
 }

faster-rnnlm/nce.h

@@ -41,7 +41,7 @@ class NCE {
           const Ref<const RowVector> hidden, WordIndex target_word,
           const uint64_t* maxent_indices, size_t maxent_size,
           const NoiseSample& sample, Real lrate, Real l2reg,
-          Real maxent_lrate, Real maxent_l2reg,
+          Real maxent_lrate, Real maxent_l2reg, Real gradient_clipping,
           Ref<RowVector> hidden_grad, MaxEnt* maxent);
 
      private: