02 - SPARK - Data Prep - TFIDF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload Review Data using AzureML\n",
    "\n",
    "Create a batch file and execute:\n",
    "    \n",
    "```\n",
    "cd \"C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\AzCopy\"\n",
    "AzCopy /Source:C:\\_ilia_share\\amazon_prod_reviews_clean\\raw /Dest:https://ikcentralusstore.blob.core.windows.net/amazonrev /DestKey:dLR5lH2QN/ejGmyD61nQoh7Cc2DW8jIKhR5n5uvGu8+H3Qem4J0XzWG1/7XtBxmVlWr+y/GNRlwX4Km5YU68sg== /Pattern:\"aggressive_dedup.json\"\n",
    "pause\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Review Data (from Blob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating SparkContext as 'sc'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>38</td><td>application_1469453428769_0016</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://hn1-ikclus.ftd4jbtqjxzuhd0uvhsmx0be3e.gx.internal.cloudapp.net:8088/proxy/application_1469453428769_0016/\">Link</a></td><td><a target=\"_blank\" href=\"http://10.0.0.8:30060/node/containerlogs/container_e04_1469453428769_0016_01_000001/spark\">Link</a></td><td>✔</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating HiveContext as 'sqlContext'\n",
      "SparkContext and HiveContext created. Executing user code ...\n"
     ]
    }
   ],
   "source": [
    "# paths\n",
    "blob = \"wasb://amazonrev@ikcentralusstore.blob.core.windows.net\"\n",
    "json_dta = blob + \"/aggressive_dedup.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pyspark.sql.dataframe.DataFrame'>\n",
      "+----------+-------+-------+--------------------+-----------+--------------------+---------------+--------------------+--------------+\n",
      "|      asin|helpful|overall|          reviewText| reviewTime|          reviewerID|   reviewerName|             summary|unixReviewTime|\n",
      "+----------+-------+-------+--------------------+-----------+--------------------+---------------+--------------------+--------------+\n",
      "|B003UYU16G| [0, 0]|    5.0|It is and does ex...|11 21, 2012|A00000262KYZUE4J5...| Steven N Elich|Does what it's su...|    1353456000|\n",
      "|B005FYPK9C| [0, 0]|    5.0|I was sketchy at ...| 01 8, 2013|A000008615DZQRRI9...|      mj waldon|           great buy|    1357603200|\n",
      "|B000VEBG9Y| [0, 0]|    3.0|Very mobile produ...|03 24, 2014|A00000922W28P2OCH...|Gabriel Merrill|Great product but...|    1395619200|\n",
      "|B001EJMS6K| [0, 0]|    4.0|Easy to use a mob...|03 24, 2014|A00000922W28P2OCH...|Gabriel Merrill|Great inexpensive...|    1395619200|\n",
      "|B003XJCNVO| [0, 0]|    4.0|Love this feeder....|03 24, 2014|A00000922W28P2OCH...|Gabriel Merrill|Great feeder. Wou...|    1395619200|\n",
      "+----------+-------+-------+--------------------+-----------+--------------------+---------------+--------------------+--------------+\n",
      "only showing top 5 rows"
     ]
    }
   ],
   "source": [
    "# load data\n",
    "jsonFile = sqlContext.read.json(json_dta)\n",
    "jsonFile.registerTempTable(\"reviews\")\n",
    "\n",
    "print(type(jsonFile)) #  <class 'pyspark.sql.dataframe.DataFrame'>\n",
    "jsonFile.show(5)\n",
    "\n",
    "# Note: also load the IMDB data at some point\n",
    "# ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Examine some of the reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%sql \n",
    "SELECT overall, reviewText\n",
    "FROM reviews\n",
    "LIMIT 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%sql \n",
    "SELECT overall, COUNT(overall) as freq\n",
    "FROM reviews\n",
    "GROUP BY overall\n",
    "ORDER by -freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------+\n",
      "|label|   count|\n",
      "+-----+--------+\n",
      "|  mid| 7039272|\n",
      "|  low|10963811|\n",
      "| high|64453794|\n",
      "+-----+--------+"
     ]
    }
   ],
   "source": [
    "# Create a dataframe of our reviews\n",
    "# To analyse class imbalance\n",
    "reviews =  sqlContext.sql(\"SELECT \" + \n",
    "                          \"CASE WHEN overall < 3 THEN 'low' \" +\n",
    "                          \"WHEN overall > 3 THEN 'high' ELSE 'mid' END as label, \" + \n",
    "                          \"reviewText as sentences \" + \n",
    "                          \"FROM reviews\")\n",
    "\n",
    "tally = reviews.groupBy(\"label\").count()\n",
    "tally.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. It is and does exactly what the description said it would be and would do. Couldn't be happier with it.\n",
      "2. I was sketchy at first about these but once you wear them for a couple hours they break in they fit good on my board an have little wear from skating in them. They are a little heavy but won't get eaten up as bad by your grip tape like poser dc shoes.\n",
      "3. Very mobile product. Efficient. Easy to use; however product needs a varmint guard. Critters are able to gorge themselves without a guard.\n",
      "4. Easy to use a mobile. If you're taller than 4ft, be ready to tuck your legs behind you as you hang and pull.\n",
      "5. Love this feeder. Heavy duty & capacity. Best feature is the large varmint guard. Definitely use a small lock or securing device on the battery housing latch. I gave 4 stars because several bolts were missing. Check contents b4 beginning.\n",
      "6. Solid, stable mount. Holds iPhone with phone protector well. I have not however used the dash mount part of this product (only windshield).\n",
      "7. I bought this pepper because I wanted a lot of cayenne powder I mean a lot. I drink shots of this powder daily and I do like it but I'm not sure if it's just me but it does not seem to strong I sweat for a minute but I feel like it could be stronger I even touched my eyes to see if it would hurt still not feeling much pain. Im one in 7 billion so do forget my review since I'm more hardcore.\n",
      "8. Beautiful photos/film with wonderful music.  Giriodi lets you know where you are with on screen notes.  It helps to keep your heart in Colorado when you can't be there.\n",
      "9. My idea of Colorado is &#34;Mountains&#34;.  Colorado Landscapes seemed to focus on the flatlands and foothills.  It gave you no idea where you were  - it could have used a narrator or notes on-screen.\n",
      "10. No matter what we did the bills just kept jamming in the machine. The bill counts were not consistent. We returned the product because it did not work correctly.\n",
      "11. i do not suggest buying this product i thought it seemed nice and everything but it did not work when i removed it from the packaging. for this reason i do not trust the shipping methods that the seller uses.\n",
      "12. Useless  - ALL you need is a Good Diet , and 1 hour cardio everyday , These kind of thigs are useless\n",
      "13. This book is really great for those that have an artistic soul and looking for better life or business. I highly recommend it.\n",
      "14. It is not a sticker, it is a Chritsmas story  by itself, full of details, and cover a big  space\n",
      "15. LOve the size and the details, and its very colorful. It looks really nice  and  catch your attention  when you pass by\n",
      "16. Its very colorful and the image once  you  put all de pieces in order is pretty  cute. The window looks terrific\n",
      "17. The condition of the book was exactly as described. There were minimal if any damages, and certainly nothing that would hinder the use of the book.\n",
      "18. Only negative. Thing I can say is when my fish swim near the sides of the tank at the top they aren't illuminated by the light from the center. The blue night light is beautiful. The led light gives a healthy glow during the day, and overall the Hood looks and works great\n",
      "19. Bought this for my daughter and she loves it!  It is also easy to attach the wheels; it's ready to go in under 5 minutes.\n",
      "20. This book is a great eye opener for small to mid- size business owners. The marketing and SEO tips are very up to date and accurate. This information is vital to running a successful business. I learned more about marketing and how to make your page simulated for the customers you want to reel in. Also the process of \"Big Fish Results\" helps me understand how I can \"reel in the catch\" instead of just trying an idea and hoping it will work. With this information I've managed to set up better marketing tools that actually work!!!Great work! What a book!"
     ]
    }
   ],
   "source": [
    "# Let's look at some reviews to see how clean they are\n",
    "# there seems to be lots of html formatting\n",
    "for c,r in enumerate(reviews.take(20)):\n",
    "    print(\"%d. %s\" % (c+1,r['sentences']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Some very basic cleaning\n",
    "from pyspark.sql.functions import UserDefinedFunction, col\n",
    "from pyspark.sql.types import StringType, BooleanType \n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def cleanerHTML(line):\n",
    "    # html formatting\n",
    "    html_clean = BeautifulSoup(line, \"lxml\").get_text().lower()\n",
    "    # pad punctuation\n",
    "    punctuation = \"\\\"'.,?!:;(){}[]/\"\n",
    "    for char in punctuation:\n",
    "        html_clean = html_clean.replace(char, ' ' + char + ' ')\n",
    "    # remove any double spaces, line-breaks, etc.\n",
    "    return \" \".join(html_clean.split())\n",
    "\n",
    "cleaner = UserDefinedFunction(cleanerHTML, StringType())\n",
    "cleanedReviews = reviews.select(reviews.label, \n",
    "                                cleaner(reviews.sentences).alias('sentences'))\n",
    "\n",
    "def longEnough(line):\n",
    "    return len(line) > 10\n",
    "\n",
    "minlength = UserDefinedFunction(longEnough, BooleanType())\n",
    "cleanedReviews = cleanedReviews.where(minlength(col('sentences')))\n",
    "\n",
    "# Lose around 13,000 bad reviews\n",
    "#82,456,877\n",
    "#82,443,303"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. it is and does exactly what the description said it would be and would do . couldn ' t be happier with it .\n",
      "2. i was sketchy at first about these but once you wear them for a couple hours they break in they fit good on my board an have little wear from skating in them . they are a little heavy but won ' t get eaten up as bad by your grip tape like poser dc shoes .\n",
      "3. very mobile product . efficient . easy to use ; however product needs a varmint guard . critters are able to gorge themselves without a guard .\n",
      "4. easy to use a mobile . if you ' re taller than 4ft , be ready to tuck your legs behind you as you hang and pull .\n",
      "5. love this feeder . heavy duty & capacity . best feature is the large varmint guard . definitely use a small lock or securing device on the battery housing latch . i gave 4 stars because several bolts were missing . check contents b4 beginning .\n",
      "6. solid , stable mount . holds iphone with phone protector well . i have not however used the dash mount part of this product ( only windshield ) .\n",
      "7. i bought this pepper because i wanted a lot of cayenne powder i mean a lot . i drink shots of this powder daily and i do like it but i ' m not sure if it ' s just me but it does not seem to strong i sweat for a minute but i feel like it could be stronger i even touched my eyes to see if it would hurt still not feeling much pain . im one in 7 billion so do forget my review since i ' m more hardcore .\n",
      "8. beautiful photos / film with wonderful music . giriodi lets you know where you are with on screen notes . it helps to keep your heart in colorado when you can ' t be there .\n",
      "9. my idea of colorado is \" mountains \" . colorado landscapes seemed to focus on the flatlands and foothills . it gave you no idea where you were - it could have used a narrator or notes on-screen .\n",
      "10. no matter what we did the bills just kept jamming in the machine . the bill counts were not consistent . we returned the product because it did not work correctly .\n",
      "11. i do not suggest buying this product i thought it seemed nice and everything but it did not work when i removed it from the packaging . for this reason i do not trust the shipping methods that the seller uses .\n",
      "12. useless - all you need is a good diet , and 1 hour cardio everyday , these kind of thigs are useless\n",
      "13. this book is really great for those that have an artistic soul and looking for better life or business . i highly recommend it .\n",
      "14. it is not a sticker , it is a chritsmas story by itself , full of details , and cover a big space\n",
      "15. love the size and the details , and its very colorful . it looks really nice and catch your attention when you pass by\n",
      "16. its very colorful and the image once you put all de pieces in order is pretty cute . the window looks terrific\n",
      "17. the condition of the book was exactly as described . there were minimal if any damages , and certainly nothing that would hinder the use of the book .\n",
      "18. only negative . thing i can say is when my fish swim near the sides of the tank at the top they aren ' t illuminated by the light from the center . the blue night light is beautiful . the led light gives a healthy glow during the day , and overall the hood looks and works great\n",
      "19. bought this for my daughter and she loves it ! it is also easy to attach the wheels ; it ' s ready to go in under 5 minutes .\n",
      "20. this book is a great eye opener for small to mid- size business owners . the marketing and seo tips are very up to date and accurate . this information is vital to running a successful business . i learned more about marketing and how to make your page simulated for the customers you want to reel in . also the process of \" big fish results \" helps me understand how i can \" reel in the catch \" instead of just trying an idea and hoping it will work . with this information i ' ve managed to set up better marketing tools that actually work ! ! ! great work ! what a book !"
     ]
    }
   ],
   "source": [
    "# A bit cleaner ...\n",
    "for c,r in enumerate(cleanedReviews.take(20)):\n",
    "    print(\"%d. %s\" % (c+1,r['sentences']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+\n",
      "|label|           sentences|\n",
      "+-----+--------------------+\n",
      "| high|it is and does ex...|\n",
      "| high|i was sketchy at ...|\n",
      "|  mid|very mobile produ...|\n",
      "| high|easy to use a mob...|\n",
      "| high|love this feeder ...|\n",
      "| high|solid , stable mo...|\n",
      "| high|i bought this pep...|\n",
      "| high|beautiful photos ...|\n",
      "|  low|my idea of colora...|\n",
      "|  low|no matter what we...|\n",
      "|  low|i do not suggest ...|\n",
      "|  low|useless - all you...|\n",
      "| high|this book is real...|\n",
      "| high|it is not a stick...|\n",
      "| high|love the size and...|\n",
      "| high|its very colorful...|\n",
      "| high|the condition of ...|\n",
      "| high|only negative . t...|\n",
      "| high|bought this for m...|\n",
      "| high|this book is a gr...|\n",
      "+-----+--------------------+\n",
      "only showing top 20 rows"
     ]
    }
   ],
   "source": [
    "cleanedReviews.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Equalise classes \n",
    "neg_rev = cleanedReviews.filter(\"label = 'low'\")\n",
    "pos_rev = cleanedReviews.filter(\"label = 'high'\").limit(neg_rev.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10961702"
     ]
    }
   ],
   "source": [
    "pos_rev.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21923404"
     ]
    }
   ],
   "source": [
    "# Save data\n",
    "allData = pos_rev.unionAll(neg_rev)\n",
    "print(allData.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#allDataLoc = blob + \"/cleaned_equal_classes.json\"\n",
    "#allData.write.format(\"json\").mode(\"overwrite\").save(allDataLoc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#allDataLoc = blob + \"/cleaned_equal_classes.json\"\n",
    "#allData = sqlContext.read.json(allDataLoc)\n",
    "\n",
    "#data_count = allData.count()\n",
    "#print(data_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#sub_sample = 1000000\n",
    "#sub_sample_ratio = float(sub_sample)/float(data_count)\n",
    "\n",
    "#print(sub_sample_ratio)\n",
    "#print(type(allData))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# sub_sample -> sample(boolean withReplacement, double fraction, long seed)\n",
    "#allData = allData.sample(False, sub_sample_ratio, 12345)\n",
    "\n",
    "# split intro training and test (50%, 50%)\n",
    "trainingData, testData = allData.randomSplit([0.5, 0.5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+\n",
      "|label|           sentences|\n",
      "+-----+--------------------+\n",
      "| high|! ! ! ! ! ! ! ! !...|\n",
      "| high|! ! ! ! ! ! ! ! !...|\n",
      "| high|! ! ! ! ! ! ! ! v...|\n",
      "| high|! ! ! ! ! ! this ...|\n",
      "| high|! ! ! ! ! ! very ...|\n",
      "| high|! ! ! ! ! wow . ....|\n",
      "| high|! ! ! ! ## @@ ? ?...|\n",
      "| high|! ! ! ! aaliyah 4...|\n",
      "| high|! ! ! ! this game...|\n",
      "| high|! ! ! ! this is t...|\n",
      "| high|! ! ! ! to god al...|\n",
      "| high|! ! ! ' s most re...|\n",
      "| high|! ! ! a question ...|\n",
      "| high|! ! ! fan-freakin...|\n",
      "| high|! ! ! if you are ...|\n",
      "| high|! ! ! masterpiece...|\n",
      "| high|! ! ! spelt healt...|\n",
      "| high|! ! ! spoiler ale...|\n",
      "| high|! ! ! the best ac...|\n",
      "| high|! ! ! this is an ...|\n",
      "+-----+--------------------+\n",
      "only showing top 20 rows"
     ]
    }
   ],
   "source": [
    "trainingData.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+\n",
      "|label|           sentences|\n",
      "+-----+--------------------+\n",
      "| high|! ! ! ! ! ! ! ! !...|\n",
      "| high|! ! ! ! ! ! ! ! !...|\n",
      "| high|! ! ! ! ! ! ! ! !...|\n",
      "| high|! ! ! ! ! ! ! ! !...|\n",
      "| high|! ! ! ! ! ! ! ! !...|\n",
      "| high|! ! ! ! ! ! ! ! !...|\n",
      "| high|! ! ! ! ! ! ! *no...|\n",
      "| high|! ! ! ! ! ! ! lov...|\n",
      "| high|! ! ! ! ! ! gets ...|\n",
      "| high|! ! ! ! ! ! this ...|\n",
      "| high|! ! ! ! ! ! very ...|\n",
      "| high|! ! ! ! ! fabulou...|\n",
      "| high|! ! ! ! ! wwwooow...|\n",
      "| high|! ! ! ! great fun...|\n",
      "| high|! ! ! ! great pro...|\n",
      "| high|! ! ! ! like it b...|\n",
      "| high|! ! ! ! spoilers ...|\n",
      "| high|! ! ! ! this came...|\n",
      "| high|! ! ! ! this is t...|\n",
      "| high|! ! ! a question ...|\n",
      "+-----+--------------------+\n",
      "only showing top 20 rows"
     ]
    }
   ],
   "source": [
    "testData.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Row(label=u'high', sentences=u'! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! the emotions were so strong , if the characters laughed , you laughed , if they smiled , you smiled , if they cried , you cried . . . some moments were just so heartbreaking . . .', sentiment_idx=1.0, words=[u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'the', u'emotions', u'were', u'so', u'strong', u',', u'if', u'the', u'characters', u'laughed', u',', u'you', u'laughed', u',', u'if', u'they', u'smiled', u',', u'you', u'smiled', u',', u'if', u'they', u'cried', u',', u'you', u'cried', u'.', u'.', u'.', u'some', u'moments', u'were', u'just', u'so', u'heartbreaking', u'.', u'.', u'.'], 2gram=[u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! the', u'the emotions', u'emotions were', u'were so', u'so strong', u'strong ,', u', if', u'if the', u'the characters', u'characters laughed', u'laughed ,', u', you', u'you laughed', u'laughed ,', u', if', u'if they', u'they smiled', u'smiled ,', u', you', u'you smiled', u'smiled ,', u', if', u'if they', u'they cried', u'cried ,', u', you', u'you cried', u'cried .', u'. .', u'. .', u'. some', u'some moments', u'moments were', u'were just', u'just so', u'so heartbreaking', u'heartbreaking .', u'. .', u'. .'], 3gram=[u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! the', u'! the emotions', u'the emotions were', u'emotions were so', u'were so strong', u'so strong ,', u'strong , if', u', if the', u'if the characters', u'the characters laughed', u'characters laughed ,', u'laughed , you', u', you laughed', u'you laughed ,', u'laughed , if', u', if they', u'if they smiled', u'they smiled ,', u'smiled , you', u', you smiled', u'you smiled ,', u'smiled , if', u', if they', u'if they cried', u'they cried ,', u'cried , you', u', you cried', u'you cried .', u'cried . .', u'. . .', u'. . some', u'. some moments', u'some moments were', u'moments were just', u'were just so', u'just so heartbreaking', u'so heartbreaking .', u'heartbreaking . .', u'. . .'], ngrams=[u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'!', u'the', u'emotions', u'were', u'so', u'strong', u',', u'if', u'the', u'characters', u'laughed', u',', u'you', u'laughed', u',', u'if', u'they', u'smiled', u',', u'you', u'smiled', u',', u'if', u'they', u'cried', u',', u'you', u'cried', u'.', u'.', u'.', u'some', u'moments', u'were', u'just', u'so', u'heartbreaking', u'.', u'.', u'.', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! !', u'! the', u'the emotions', u'emotions were', u'were so', u'so strong', u'strong ,', u', if', u'if the', u'the characters', u'characters laughed', u'laughed ,', u', you', u'you laughed', u'laughed ,', u', if', u'if they', u'they smiled', u'smiled ,', u', you', u'you smiled', u'smiled ,', u', if', u'if they', u'they cried', u'cried ,', u', you', u'you cried', u'cried .', u'. .', u'. .', u'. some', u'some moments', u'moments were', u'were just', u'just so', u'so heartbreaking', u'heartbreaking .', u'. .', u'. .', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! !', u'! ! the', u'! the emotions', u'the emotions were', u'emotions were so', u'were so strong', u'so strong ,', u'strong , if', u', if the', u'if the characters', u'the characters laughed', u'characters laughed ,', u'laughed , you', u', you laughed', u'you laughed ,', u'laughed , if', u', if they', u'if they smiled', u'they smiled ,', u'smiled , you', u', you smiled', u'you smiled ,', u'smiled , if', u', if they', u'if they cried', u'they cried ,', u'cried , you', u', you cried', u'you cried .', u'cried . .', u'. . .', u'. . some', u'. some moments', u'some moments were', u'moments were just', u'were just so', u'just so heartbreaking', u'so heartbreaking .', u'heartbreaking . .', u'. . .'], rawFeatures=SparseVector(40000, {33: 166.0, 44: 6.0, 46: 6.0, 522: 2.0, 568: 1.0, 814: 1.0, 992: 2.0, 1172: 1.0, 1873: 1.0, 1959: 1.0, 2823: 1.0, 3357: 3.0, 3676: 2.0, 3785: 1.0, 3819: 1.0, 3932: 2.0, 5244: 4.0, 5825: 2.0, 6184: 2.0, 6571: 1.0, 7539: 1.0, 8144: 1.0, 8313: 1.0, 8457: 1.0, 8889: 2.0, 10638: 1.0, 10773: 1.0, 10936: 1.0, 11019: 2.0, 13543: 1.0, 14330: 1.0, 15228: 1.0, 16116: 1.0, 16193: 1.0, 16829: 1.0, 17173: 1.0, 17504: 1.0, 18340: 1.0, 19066: 1.0, 19379: 1.0, 19499: 1.0, 19863: 1.0, 20153: 1.0, 20529: 1.0, 20662: 1.0, 21700: 1.0, 21847: 2.0, 22243: 164.0, 22649: 1.0, 23166: 1.0, 23910: 1.0, 24234: 1.0, 24306: 1.0, 24465: 1.0, 24603: 1.0, 24860: 1.0, 24900: 1.0, 24913: 3.0, 26438: 1.0, 27308: 2.0, 28075: 3.0, 28217: 1.0, 28579: 1.0, 29157: 1.0, 29734: 1.0, 30149: 1.0, 30151: 1.0, 30983: 1.0, 31283: 1.0, 31354: 1.0, 32389: 1.0, 32683: 1.0, 32738: 165.0, 33240: 1.0, 33495: 1.0, 33964: 1.0, 34737: 1.0, 34801: 2.0, 35753: 1.0, 37279: 1.0, 38242: 1.0, 38952: 2.0, 39123: 1.0, 39181: 1.0, 39707: 1.0, 39786: 1.0, 39839: 3.0}), logRawFeatures=SparseVector(40000, {33: 6.112, 44: 2.7918, 46: 2.7918, 522: 1.6931, 568: 1.0, 814: 1.0, 992: 1.6931, 1172: 1.0, 1873: 1.0, 1959: 1.0, 2823: 1.0, 3357: 2.0986, 3676: 1.6931, 3785: 1.0, 3819: 1.0, 3932: 1.6931, 5244: 2.3863, 5825: 1.6931, 6184: 1.6931, 6571: 1.0, 7539: 1.0, 8144: 1.0, 8313: 1.0, 8457: 1.0, 8889: 1.6931, 10638: 1.0, 10773: 1.0, 10936: 1.0, 11019: 1.6931, 13543: 1.0, 14330: 1.0, 15228: 1.0, 16116: 1.0, 16193: 1.0, 16829: 1.0, 17173: 1.0, 17504: 1.0, 18340: 1.0, 19066: 1.0, 19379: 1.0, 19499: 1.0, 19863: 1.0, 20153: 1.0, 20529: 1.0, 20662: 1.0, 21700: 1.0, 21847: 1.6931, 22243: 6.0999, 22649: 1.0, 23166: 1.0, 23910: 1.0, 24234: 1.0, 24306: 1.0, 24465: 1.0, 24603: 1.0, 24860: 1.0, 24900: 1.0, 24913: 2.0986, 26438: 1.0, 27308: 1.6931, 28075: 2.0986, 28217: 1.0, 28579: 1.0, 29157: 1.0, 29734: 1.0, 30149: 1.0, 30151: 1.0, 30983: 1.0, 31283: 1.0, 31354: 1.0, 32389: 1.0, 32683: 1.0, 32738: 6.1059, 33240: 1.0, 33495: 1.0, 33964: 1.0, 34737: 1.0, 34801: 1.6931, 35753: 1.0, 37279: 1.0, 38242: 1.0, 38952: 1.6931, 39123: 1.0, 39181: 1.0, 39707: 1.0, 39786: 1.0, 39839: 2.0986}), features=SparseVector(40000, {33: 8.24, 44: 1.1997, 46: 0.1343, 522: 3.5487, 568: 4.6855, 814: 3.906, 992: 8.8416, 1172: 5.2232, 1873: 3.9716, 1959: 5.1352, 2823: 5.207, 3357: 3.2373, 3676: 2.2171, 3785: 6.0592, 3819: 4.8901, 3932: 9.7297, 5244: 4.5835, 5825: 3.6954, 6184: 9.6061, 6571: 5.2093, 7539: 5.1471, 8144: 4.9177, 8313: 3.7189, 8457: 5.6332, 8889: 8.8448, 10638: 4.0877, 10773: 6.0492, 10936: 5.697, 11019: 7.2218, 13543: 5.2372, 14330: 5.8596, 15228: 5.6854, 16116: 2.0848, 16193: 4.0623, 16829: 5.3007, 17173: 5.4154, 17504: 5.3061, 18340: 5.5552, 19066: 5.5898, 19379: 5.7772, 19499: 5.666, 19863: 3.9599, 20153: 4.7814, 20529: 5.7475, 20662: 5.7335, 21700: 5.0679, 21847: 9.0377, 22243: 20.2713, 22649: 5.5965, 23166: 5.6355, 23910: 5.8252, 24234: 3.0764, 24306: 3.8814, 24465: 5.003, 24603: 4.798, 24860: 5.2047, 24900: 5.8589, 24913: 7.8126, 26438: 5.7778, 27308: 9.3861, 28075: 6.5385, 28217: 5.3612, 28579: 4.7758, 29157: 6.0374, 29734: 4.2008, 30149: 3.918, 30151: 5.0085, 30983: 4.8197, 31283: 4.5912, 31354: 6.0971, 32389: 5.4899, 32683: 5.7596, 32738: 17.0504, 33240: 5.9304, 33495: 5.0549, 33964: 1.5513, 34737: 5.6167, 34801: 0.3262, 35753: 6.0355, 37279: 5.3329, 38242: 5.2813, 38952: 2.5961, 39123: 5.5658, 39181: 5.5073, 39707: 3.6607, 39786: 5.4735, 39839: 2.4922}))"
     ]
    }
   ],
   "source": [
    "\"\"\" Pipeline for feature selection and classification\n",
    "Using:\n",
    "\n",
    "https://spark.apache.org/docs/1.5.2/ml-features.html\n",
    "https://spark.apache.org/docs/1.6.1/api/python/pyspark.sql.html\n",
    "http://spark.apache.org/docs/latest/api/python/pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionModel\n",
    "http://nlp.stanford.edu/IR-book/html/htmledition/document-and-query-weighting-schemes-1.html#sec:querydocweighting\n",
    "\n",
    "Attempting to replicate: \n",
    "\n",
    "class sklearn.feature_extraction.text.TfidfVectorizer(input='content', encoding='utf-8',\n",
    "decode_error='strict', strip_accents=None, lowercase=True, preprocessor=None, \n",
    "tokenizer=None, analyzer='word', stop_words=None, token_pattern='(?u)\\b\\w\\w+\\b', \n",
    "ngram_range=(1, 3), max_df=1.0, min_df=1, max_features=40000, vocabulary=None, \n",
    "binary=False, dtype=<class 'numpy.int64'>, norm='l2', use_idf=True, \n",
    "smooth_idf=True, sublinear_tf=True)\n",
    "\n",
    "I think only sublinear_tf and ngram_range need to be modified\n",
    "\n",
    "(https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/feature_extraction/text.py)\n",
    "if self.sublinear_tf:\n",
    "    np.log(X.data, X.data)\n",
    "    X.data += 1\n",
    "            \n",
    "\"\"\"\n",
    "\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import HashingTF, IDF, Tokenizer, NGram, StringIndexer\n",
    "from pyspark.sql.functions import col, udf\n",
    "from itertools import chain\n",
    "from pyspark.sql.types import ArrayType, StringType\n",
    "from pyspark.mllib.linalg import Vectors, VectorUDT\n",
    "import numpy as np\n",
    "\n",
    "numfeat = 40000\n",
    "\n",
    "########################\n",
    "# 1. Feature-extraction\n",
    "########################\n",
    "\n",
    "def concat(type):\n",
    "    \"\"\" UDF to concatenate lists across columns to create\n",
    "    an n-gram range. To reproduce ngram_range=(1,3) from sklearn\n",
    "    \"\"\"\n",
    "    def concat_(*args):\n",
    "        return list(chain(*args))\n",
    "    return udf(concat_, ArrayType(type))    \n",
    "\n",
    "# UDF to combine n-grams into one column\n",
    "concat_string_arrays = concat(StringType())\n",
    "\n",
    "# UDF to apply sub-linear scaling on sparse vectors tf\n",
    "vector_udf = udf(lambda sv: Vectors.sparse(sv.size,\n",
    "                                           dict(zip(sv.indices, np.log(sv.values) + 1))),\n",
    "                 VectorUDT())\n",
    "\n",
    "indexer = StringIndexer(inputCol=\"label\", outputCol=\"sentiment_idx\")\n",
    "tokenizer = Tokenizer(inputCol=\"sentences\", outputCol=\"words\")\n",
    "biGram = NGram(inputCol = \"words\", n=2, outputCol = \"2gram\")\n",
    "triGram = NGram(inputCol = \"words\", n=3, outputCol = \"3gram\")\n",
    "hashingtf  = HashingTF(inputCol=\"ngrams\", outputCol=\"rawFeatures\", numFeatures=numfeat)\n",
    "idf = IDF(inputCol=\"logRawFeatures\", outputCol=\"features\")\n",
    "\n",
    "#######\n",
    "# Train\n",
    "#######\n",
    "indexerModel = indexer.fit(trainingData)\n",
    "trainingDataIx = indexerModel.transform(trainingData)\n",
    "tokenized_train = tokenizer.transform(trainingDataIx)\n",
    "\n",
    "biGram_train = biGram.transform(tokenized_train)\n",
    "triGram_train = triGram.transform(biGram_train)\n",
    "ngrammed_train = triGram_train.withColumn(\"ngrams\", concat_string_arrays(\n",
    "        col(\"words\"),\n",
    "        col(\"2gram\"),\n",
    "        col(\"3gram\")))\n",
    "hashed_train = hashingtf.transform(ngrammed_train)\n",
    "sublintf_train = hashed_train.withColumn('logRawFeatures', vector_udf(\n",
    "        hashed_train.rawFeatures))\n",
    "\n",
    "idfModel = idf.fit(sublintf_train)\n",
    "idf_train = idfModel.transform(sublintf_train)\n",
    "idf_train.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tfidf variants:\n",
    "\n",
    "![alt text](http://nlp.stanford.edu/IR-book/html/htmledition/img462.png \"TFs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"\\n###########################\\n# Example: Apply sub-linear\\n###########################\\nfrom pyspark.mllib.linalg import Vectors, VectorUDT\\nimport numpy as np\\n\\ntesty = hashed_train.first()['rawFeatures']\\n\\nprint(type(testy))\\nprint(type(testy.values))\\nprint(testy)\\n\\nvector_udf = udf(lambda sv: Vectors.sparse(sv.size, dict(zip(sv.indices, np.log(sv.values) + 10))), VectorUDT())\\nsublintf_train = hashed_train.withColumn('sublintf', vector_udf(hashed_train.rawFeatures))\\n\\ntesty2 = sublintf_train.first()['sublintf']\\nprint(testy2)\\n\""
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "###########################\n",
    "# Example: Apply sub-linear\n",
    "###########################\n",
    "from pyspark.mllib.linalg import Vectors, VectorUDT\n",
    "import numpy as np\n",
    "\n",
    "testy = hashed_train.first()['rawFeatures']\n",
    "\n",
    "print(type(testy))\n",
    "print(type(testy.values))\n",
    "print(testy)\n",
    "\n",
    "vector_udf = udf(lambda sv: Vectors.sparse(sv.size, dict(zip(sv.indices, np.log(sv.values) + 10))), VectorUDT())\n",
    "sublintf_train = hashed_train.withColumn('sublintf', vector_udf(hashed_train.rawFeatures))\n",
    "\n",
    "testy2 = sublintf_train.first()['sublintf']\n",
    "print(testy2)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "######\n",
    "# Test\n",
    "######\n",
    "testDataIx = indexerModel.transform(testData)\n",
    "tokenized_test = tokenizer.transform(testDataIx)\n",
    "\n",
    "biGram_test = biGram.transform(tokenized_test)\n",
    "triGram_test = triGram.transform(biGram_test)\n",
    "ngrammed_test = triGram_test.withColumn(\"ngrams\", concat_string_arrays(\n",
    "        col(\"words\"),\n",
    "        col(\"2gram\"),\n",
    "        col(\"3gram\")))\n",
    "hashed_test = hashingtf.transform(ngrammed_test)\n",
    "sublintf_test = hashed_test.withColumn('logRawFeatures', vector_udf(\n",
    "        hashed_test.rawFeatures))\n",
    "idf_test = idfModel.transform(sublintf_test)\n",
    "\n",
    "idf_test.first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import LogisticRegression, GBTClassifier\n",
    "\n",
    "# 2A. Classifier (Logistic Regression)\n",
    "classi = LogisticRegression(labelCol=\"sentiment_idx\", featuresCol=\"features\")\n",
    "tfidfModel = classi.fit(idf_train)\n",
    "pred = tfidfModel.transform(idf_test)\n",
    "\n",
    "# 3. Examine\n",
    "numSuccesses = pred.where(\"\"\"(prediction = sentiment_idx)\"\"\").count()\n",
    "numInspections = numSuccesses + pred.where(\"\"\"(prediction != sentiment_idx)\"\"\").count()\n",
    "acc = (float(numSuccesses) / float(numInspections)) * 100\n",
    "print(\"%.2f success rate\" % acc)\n",
    "\n",
    "\"\"\"\n",
    "# Sub-sample: 1 mill total\n",
    "Standard: 76.77 success rate\n",
    "With ngrams(1,3): 88.17 success rate\n",
    "With ngrams + sublineartf: 88.32 success rate\n",
    "# Full data: 89.05 success rate\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 3. Evaluation\n",
    "pred.select(col('prediction'),col('sentiment_idx')).show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python"
   },
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}