Add basic API

Author: schedutron

.gitignore

@@ -11,3 +11,4 @@ images/*
 .ipynb_checkpoints/
 tmp_data/
 .DS_Store
+api_log/

api.py

@@ -0,0 +1,22 @@
+import pickle
+import subprocess
+
+from flask import Flask, request, jsonify
+from flask_restful import Resource, Api
+from json import dumps
+
+app = Flask(__name__)
+api = Api(app)
+
+class Predict(Resource):
+    def get(self, sym):
+        command = 'python2 main.py stock_symbol=%s --write' % sym
+        subprocess.call(command.split())
+        with open('api_log/'+sym+'.pkl', 'rb') as f:
+            prediction = str(pickle.load(f))
+            return jsonify(prediction)
+
+api.add_resource(Predict, '/predict/<sym>') # Route_3
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0', port='5002')

check.png

@@ -1,7 +1,9 @@
 import os
 import pandas as pd
+import pickle
 import pprint
 
+import matplotlib.pyplot as plt
 import tensorflow as tf
 import tensorflow.contrib.slim as slim
 
@@ -24,6 +26,7 @@
 flags.DEFINE_string("stock_symbol", None, "Target stock symbol [None]")
 flags.DEFINE_integer("sample_size", 4, "Number of stocks to plot during training. [4]")
 flags.DEFINE_boolean("train", False, "True for training, False for testing [False]")
+flags.DEFINE_boolean("write", False, "True for writing contents to the file of the same name")
 
 FLAGS = flags.FLAGS
 
@@ -104,6 +107,14 @@ def main(_):
         if FLAGS.train:
             rnn_model.train(stock_data_list, FLAGS)
         else:
+            test_prediction, test_loss = rnn_model.predict(stock_data_list, 50, FLAGS)
+            if FLAGS.write:
+                with open('api_log/'+FLAGS.stock_symbol+".pkl", 'wb') as f:
+                    pickle.dump(test_prediction, f)
+            
+            #rnn_model.plot_samples(test_prediction, test_prediction, 'check.png', 'GOOG')
+            #plt.show(block=True)
+            print '-'*33
             if not rnn_model.load()[0]:
                 raise Exception("[!] Train a model first, then run test mode")
 

scripts/config.py renamed to config.py

@@ -13,6 +13,8 @@
 
 from tensorflow.contrib.tensorboard.plugins import projector
 
+from restore_model import prediction_by_trained_graph
+
 
 class LstmRNN(object):
     def __init__(self, sess, stock_count,
@@ -161,7 +163,7 @@ def train(self, dataset_list, config):
         merged_test_X = []
         merged_test_y = []
         merged_test_labels = []
-
+        
         for label_, d_ in enumerate(dataset_list):
             merged_test_X += list(d_.test_X)
             merged_test_y += list(d_.test_y)
@@ -181,7 +183,8 @@ def train(self, dataset_list, config):
             self.targets: merged_test_y,
             self.symbols: merged_test_labels,
         }
-
+        print 'merged_test_X', merged_test_X
+        print 'merged_test_y', merged_test_y
         global_step = 0
 
         num_batches = sum(len(d_.train_X) for d_ in dataset_list) // config.batch_size
@@ -196,7 +199,7 @@ def train(self, dataset_list, config):
                 i for i, sym_label in enumerate(merged_test_labels)
                 if sym_label[0] == l])
             sample_indices[sym] = target_indices
-        print sample_indices
+        
 
         print "Start training for stocks:", [d.stock_sym for d in dataset_list]
         for epoch in xrange(config.max_epoch):
@@ -222,7 +225,7 @@ def train(self, dataset_list, config):
 
                     if np.mod(global_step, len(dataset_list) * 100 / config.input_size) == 1:
                         test_loss, test_pred = self.sess.run([self.loss, self.pred], test_data_feed)
-
+                        
                         print "Step:%d [Epoch:%d] [Learning rate: %.6f] train_loss:%.6f test_loss:%.6f" % (
                             global_step, epoch, learning_rate, train_loss, test_loss)
 
@@ -238,7 +241,7 @@ def train(self, dataset_list, config):
                         self.save(global_step)
 
         final_pred, final_loss = self.sess.run([self.pred, self.loss], test_data_feed)
-
+       
         # Save the final model
         self.save(global_step)
         return final_pred
@@ -275,6 +278,8 @@ def save(self, step):
             global_step=step
         )
 
+        print os.path.join(self.model_logs_dir, model_name)
+
     def load(self):
         print(" [*] Reading checkpoints...")
         ckpt = tf.train.get_checkpoint_state(self.model_logs_dir)
@@ -292,7 +297,7 @@ def load(self):
     def plot_samples(self, preds, targets, figname, stock_sym=None):
         def _flatten(seq):
             return [x for y in seq for x in y]
-
+        
         truths = _flatten(targets)[-200:]
 
         preds = _flatten(preds)[-200:]
@@ -337,4 +342,44 @@ def _flatten(seq):
             plt.title(stock_sym + " | Last %d days in test" % len(truths))
 
         plt.savefig(figname.split('.')[0]+'_normalized.png', format='png', bbox_inches='tight', transparent=True)
-        plt.close()
+        plt.close()
+    
+
+    def predict(self, dataset_list, max_epoch, config):
+        merged_test_X, merged_test_y, merged_test_labels = [], [], []
+        for label_, d_ in enumerate(dataset_list):
+            merged_test_X += list(d_.test_X)
+            merged_test_y += list(d_.test_y)
+            merged_test_labels += [[label_]] * len(d_.test_X)
+
+        test_X = np.array(merged_test_X)
+        test_y = np.array(merged_test_y)
+    
+        status, counter = self.load()
+        if status:
+            graph = tf.get_default_graph()
+            test_data_feed = {
+                self.learning_rate: 0.0,
+                self.inputs: test_X,
+                self.targets: test_y
+            }
+            #prediction = graph.get_tensor_by_name('output_layer/add:0')
+            #loss = graph.get_tensor_by_name('train/loss_mse:0')
+
+            # Select samples for plotting.
+            sample_labels = range(min(config.sample_size, len(dataset_list)))
+            sample_indices = {}
+            for l in sample_labels:
+                sym = dataset_list[l].stock_sym
+                target_indices = np.array([
+                    i for i, sym_label in enumerate(merged_test_labels)
+                    if sym_label[0] == l])
+                sample_indices[sym] = target_indices
+            
+            
+            test_prediction, test_loss = self.sess.run([self.pred, self.loss], test_data_feed)
+
+            for sample_sym, indices in sample_indices.iteritems():
+                test_pred = test_prediction[indices]
+
+        return test_pred, test_loss

main.py

@@ -5,3 +5,5 @@ scikit-learn==0.16.1
 scipy==0.19.1
 tensorflow==1.2.1
 urllib3==1.8
+flask
+flask_restful