more on LstmRnn model

Author: Lilian Weng

.gitignore

@@ -1,8 +1,9 @@
 *.*~
 *.pyc
 *.ipynb
-_data/*.tsv
-_data/*.csv
-_logs/*/
-_models/*/
+data/*.tsv
+data/*.csv
+logs/*
+models/*
+checkpoint/*
 .idea/

_logs/.placeholder

@@ -1,9 +1,9 @@
 class RNNConfig():
-    input_size=1
-    num_steps=30
-    lstm_size=128
-    num_layers=1
-    keep_prob=0.8
+    input_size = 1
+    num_steps = 30
+    lstm_size = 128
+    num_layers = 1
+    keep_prob = 0.8
 
     batch_size = 200
     init_learning_rate = 0.05
@@ -21,9 +21,10 @@ def __str__(self):
     def __repr__(self):
         return str(self.to_dict())
 
+
 DEFAULT_CONFIG = RNNConfig()
 print "Default configuration:", DEFAULT_CONFIG.to_dict()
 
-DATA_DIR = "_data"
-LOG_DIR = "_logs"
-MODEL_DIR = "_models"
+DATA_DIR = "data"
+LOG_DIR = "logs"
+MODEL_DIR = "models"

_models/.placeholder

@@ -12,13 +12,14 @@
 class StockDataSet(object):
     def __init__(self,
                  stock_sym,
-                 config=DEFAULT_CONFIG,
+                 input_size=1,
+                 num_steps=30,
                  test_ratio=0.1,
                  normalized=True,
                  close_price_only=True):
         self.stock_sym = stock_sym
-        self.input_size = config.input_size
-        self.num_steps = config.num_steps
+        self.input_size = input_size
+        self.num_steps = num_steps
         self.test_ratio = test_ratio
         self.close_price_only = close_price_only
         self.normalized = normalized
@@ -34,6 +35,10 @@ def __init__(self,
 
         self.train_X, self.train_y, self.test_X, self.test_y = self._prepare_data(self.raw_seq)
 
+    def info(self):
+        return "StockDataSet [%s] train: %d test: %d" % (
+            self.stock_sym, len(self.train_X), len(self.test_y))
+
     def _prepare_data(self, seq):
         # split into items of input_size
         seq = [np.array(seq[i * self.input_size: (i + 1) * self.input_size])

config.py

@@ -4,6 +4,7 @@
 import tensorflow as tf
 import tensorflow.contrib.slim as slim
 
+from data_model import StockDataSet
 from model import LstmRNN
 
 flags = tf.app.flags
@@ -32,15 +33,8 @@ def show_all_variables():
 def main(_):
     pp.pprint(flags.FLAGS.__flags)
 
-    if FLAGS.input_width is None:
-        FLAGS.input_width = FLAGS.input_height
-    if FLAGS.output_width is None:
-        FLAGS.output_width = FLAGS.output_height
-
     if not os.path.exists(FLAGS.checkpoint_dir):
         os.makedirs(FLAGS.checkpoint_dir)
-    if not os.path.exists(FLAGS.sample_dir):
-        os.makedirs(FLAGS.sample_dir)
 
     # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
     run_config = tf.ConfigProto()
@@ -59,8 +53,17 @@ def main(_):
 
         show_all_variables()
 
+        stock_data = StockDataSet(
+            "GOOG",
+            input_size=FLAGS.input_size,
+            num_steps=FLAGS.num_steps,
+            test_ratio=0.1,
+            close_price_only=True
+        )
+        print stock_data.info()
+
         if FLAGS.train:
-            rnn_model.train(FLAGS)
+            rnn_model.train(stock_data, FLAGS)
         else:
             if not rnn_model.load()[0]:
                 raise Exception("[!] Train a model first, then run test mode")

_data/.placeholder renamed to data/.placeholder

@@ -79,7 +79,7 @@ def _create_one_cell():
             self.pred_summ = tf.summary.histogram("pred", self.pred)
 
         # self.loss = -tf.reduce_sum(targets * tf.log(tf.clip_by_value(prediction, 1e-10, 1.0)))
-        self.loss = tf.reduce_mean(tf.square(self.pred - self.inputs), name="loss_mse")
+        self.loss = tf.reduce_mean(tf.square(self.pred - self.targets), name="loss_mse")
         self.optim = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, name="adam_optim")
 
         self.loss_sum = tf.summary.scalar("loss_mse", self.loss)
@@ -100,9 +100,12 @@ def train(self, dataset, config):
         self.writer = tf.summary.FileWriter(os.path.join("./logs", self.model_name))
         self.writer.add_graph(self.sess.graph)
 
-        step = 1
+        num_batches = int(len(dataset.train_X)) // config.batch_size
+        global_step = 1
+
+        for epoch in xrange(config.max_epoch):
+            epoch_step = 1
 
-        for epoch in xrange(config.epoch):
             learning_rate = config.init_learning_rate * (
                 config.learning_rate_decay ** max(float(epoch + 1 - config.init_epoch), 0.0)
             )
@@ -122,17 +125,19 @@ def train(self, dataset, config):
                     self.learning_rate: learning_rate,
                 }
                 train_loss, _ = self.sess.run([self.loss, self.optim], train_data_feed)
-                step += 1
+                global_step += 1
+                epoch_step += 1
 
-                if np.mod(epoch, 10) == 0:
+                if np.mod(epoch, 20) == 0:
                     test_loss, _pred, _merged_sum = self.sess.run(
                         [self.loss, self.pred, self.merged_sum], test_data_feed)
                     assert len(_pred) == len(dataset.test_y)
-                    print "Epoch %d [%f]:" % (epoch, learning_rate), test_loss
-                    self.writer.add_summary(_merged_sum, global_step=epoch)
+                    print "Epoch %d [%d/%d][learning rate: %f]: %.6f" % (
+                        epoch, epoch_step, num_batches, learning_rate, test_loss)
+                    self.writer.add_summary(_merged_sum, global_step=global_step)
 
-                if np.mod(step, 100) == 2:
-                    self.save(self.checkpoint_dir, step)
+                if np.mod(global_step, 500) == 2:
+                    self.save(global_step)
 
         print "Final Results:"
         final_pred, final_loss = self.sess.run([self.pred, self.loss], test_data_feed)
@@ -169,3 +174,6 @@ def load(self):
         else:
             print(" [*] Failed to find a checkpoint")
             return False, 0
+
+    def plot_samples(self):
+        pass

data_wrapper.py renamed to data_model.py

@@ -4,16 +4,16 @@
 """
 import json
 import os
-import random
 import tensorflow as tf
 
 from build_graph import build_lstm_graph_with_config
 from config import DEFAULT_CONFIG, MODEL_DIR
-from data_wrapper import StockDataSet
+from data_model import StockDataSet
 
 
-def load_data(stock_name, config=DEFAULT_CONFIG):
-    stock_dataset = StockDataSet(stock_name, config, test_ratio=0.1, close_price_only=True)
+def load_data(stock_name, input_size, num_steps):
+    stock_dataset = StockDataSet(stock_name, input_size=input_size, num_steps=num_steps,
+                                 test_ratio=0.1, close_price_only=True)
     print "Train data size:", len(stock_dataset.train_X)
     print "Test data size:", len(stock_dataset.test_X)
     return stock_dataset
@@ -34,7 +34,7 @@ def train_lstm_graph(stock_name, lstm_graph, config=DEFAULT_CONFIG):
     stock_name (str)
     lstm_graph (tf.Graph)
     """
-    stock_dataset = load_data(stock_name, config=config)
+    stock_data = load_data(stock_name, input_size=config.input_size, num_steps=config.num_steps)
 
     final_prediction = []
     final_loss = None
@@ -61,8 +61,8 @@ def train_lstm_graph(stock_name, lstm_graph, config=DEFAULT_CONFIG):
         learning_rate = graph.get_tensor_by_name('learning_rate:0')
 
         test_data_feed = {
-            inputs: stock_dataset.test_X,
-            targets: stock_dataset.test_y,
+            inputs: stock_data.test_X,
+            targets: stock_data.test_y,
             learning_rate: 0.0
         }
 
@@ -73,7 +73,7 @@ def train_lstm_graph(stock_name, lstm_graph, config=DEFAULT_CONFIG):
         for epoch_step in range(config.max_epoch):
             current_lr = learning_rates_to_use[epoch_step]
 
-            for batch_X, batch_y in stock_dataset.generate_one_epoch(config.batch_size):
+            for batch_X, batch_y in stock_data.generate_one_epoch(config.batch_size):
                 train_data_feed = {
                     inputs: batch_X,
                     targets: batch_y,
@@ -83,13 +83,13 @@ def train_lstm_graph(stock_name, lstm_graph, config=DEFAULT_CONFIG):
 
             if epoch_step % 10 == 0:
                 test_loss, _pred, _summary = sess.run([loss, prediction, merged_summary], test_data_feed)
-                assert len(_pred) == len(stock_dataset.test_y)
+                assert len(_pred) == len(stock_data.test_y)
                 print "Epoch %d [%f]:" % (epoch_step, current_lr), test_loss
                 if epoch_step % 50 == 0:
                     print "Predictions:", [(
-                        map(lambda x: round(x, 4), _pred[-j]),
-                        map(lambda x: round(x, 4), stock_dataset.test_y[-j])
-                    ) for j in range(5)]
+                                               map(lambda x: round(x, 4), _pred[-j]),
+                                               map(lambda x: round(x, 4), stock_data.test_y[-j])
+                                           ) for j in range(5)]
 
             writer.add_summary(_summary, global_step=epoch_step)