basic_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Establish Spark Context for remaining tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pyspark\n",
    "sc = pyspark.SparkContext('local[*]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[431, 579, 864, 836, 926]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd = sc.parallelize(range(1000))\n",
    "rdd.takeSample(False, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Test processing of text file, based on quick start\n",
    "textFile = sc.textFile(\"README.md\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use a Spark action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lines with spark: 2\n"
     ]
    }
   ],
   "source": [
    "linesWithSpark = textFile.filter(lambda line: \"Spark\" in line)\n",
    "print(\"Lines with spark: %s\" % linesWithSpark.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mostWordsInLine = textFile.map(lambda line: len(line.split())).reduce(lambda a, b: a if (a > b) else b)\n",
    "mostWordsInLine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get count of distinct words in the README.md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('guide', 1),\n",
       " ('Install', 1),\n",
       " ('Once', 1),\n",
       " ('name', 1),\n",
       " ('--name', 1),\n",
       " ('Docker:', 1),\n",
       " ('learning', 1),\n",
       " ('ready-made', 1),\n",
       " ('2.', 1),\n",
       " ('if', 2),\n",
       " ('but', 1),\n",
       " ('IP', 1),\n",
       " (\"You'll\", 1),\n",
       " ('the', 11),\n",
       " ('form', 1),\n",
       " ('it', 1),\n",
       " ('install', 2),\n",
       " ('[basic_experiments](https://rawgit.com/rsmith72/spark-experiments/master/basic_experiments.html)',\n",
       "  1),\n",
       " ('commands.', 1),\n",
       " ('\\\\*nix', 1),\n",
       " ('<name', 1),\n",
       " ('as', 1),\n",
       " ('executed.', 1),\n",
       " ('you', 3),\n",
       " ('running', 1),\n",
       " ('using:', 1),\n",
       " ('HTML', 1),\n",
       " ('container', 2),\n",
       " ('easier', 1),\n",
       " ('have', 1),\n",
       " ('and', 2),\n",
       " ('assumes', 1),\n",
       " ('some', 1),\n",
       " ('$SPARK_EXPERIMENTS:/home/jovyan/work', 1),\n",
       " ('-v', 1),\n",
       " ('when', 1),\n",
       " ('SPARK_EXPERIMENTS=`pwd`', 1),\n",
       " ('on', 1),\n",
       " ('8888:8888', 1),\n",
       " ('These', 1),\n",
       " ('http://127.0.0.1:8888', 1),\n",
       " ('in', 1),\n",
       " ('Open', 1),\n",
       " ('know', 1),\n",
       " ('up,', 1),\n",
       " ('yourself,', 1),\n",
       " ('See', 1),\n",
       " ('may', 1),\n",
       " ('set', 1),\n",
       " ('project', 1),\n",
       " ('pyspark-notebook', 1),\n",
       " ('https://github.com/jupyter/docker-stacks/tree/master/pyspark-notebook', 1),\n",
       " ('```', 6),\n",
       " ('stop', 1),\n",
       " ('Jupyter', 2),\n",
       " ('above', 4),\n",
       " ('address', 1),\n",
       " ('remove', 1),\n",
       " ('need', 1),\n",
       " ('docker-machine', 1),\n",
       " ('machine>', 1),\n",
       " ('The', 1),\n",
       " ('3.5', 1),\n",
       " ('are', 3),\n",
       " ('To', 1),\n",
       " ('Spark', 2),\n",
       " ('Exported', 1),\n",
       " ('implemented', 1),\n",
       " ('your', 1),\n",
       " (\"ls'\", 1),\n",
       " ('/', 2),\n",
       " ('URL', 1),\n",
       " ('that', 1),\n",
       " ('#', 3),\n",
       " ('Python', 1),\n",
       " ('to', 4),\n",
       " ('Spark.', 1),\n",
       " ('on:', 1),\n",
       " ('of', 5),\n",
       " ('go', 1),\n",
       " ('notebooks:', 1),\n",
       " ('MacOS', 1),\n",
       " ('work,', 1),\n",
       " ('IPython', 1),\n",
       " ('Docker', 5),\n",
       " ('this', 1),\n",
       " ('a', 1),\n",
       " ('them', 1),\n",
       " ('image:', 2),\n",
       " ('ip', 1),\n",
       " ('be', 1),\n",
       " ('dependencies', 2),\n",
       " ('Create', 1),\n",
       " ('docker', 3),\n",
       " ('run:', 1),\n",
       " ('spark-experiments', 3),\n",
       " ('way', 1),\n",
       " ('jupyter/pyspark-notebook', 1),\n",
       " ('an', 1),\n",
       " ('done,', 1),\n",
       " ('comes', 1),\n",
       " ('Apache', 2),\n",
       " ('2.X', 1),\n",
       " ('I', 1),\n",
       " ('notebooks.', 1),\n",
       " ('pyspark', 1),\n",
       " ('Following', 1),\n",
       " ('is', 1),\n",
       " ('3.', 1),\n",
       " ('This', 2),\n",
       " ('export', 1),\n",
       " ('https://docs.docker.com/engine/installation/', 1),\n",
       " ('use', 2),\n",
       " ('If', 1),\n",
       " ('OS', 1),\n",
       " ('machine:', 1),\n",
       " ('contains', 1),\n",
       " ('Installation', 1),\n",
       " (\"'docker-machine\", 1),\n",
       " ('>', 4),\n",
       " ('You', 1),\n",
       " ('exports', 1),\n",
       " ('1.', 1),\n",
       " ('-p', 1),\n",
       " (\"you're\", 2),\n",
       " ('based', 2),\n",
       " ('rm', 1),\n",
       " (\"doesn't\", 1),\n",
       " ('run', 3),\n",
       " ('-d', 1),\n",
       " ('experiments', 3),\n",
       " ('part', 1),\n",
       " ('*', 6),\n",
       " (\"don't\", 2),\n",
       " ('&&', 1),\n",
       " ('4.', 1),\n",
       " ('can', 2),\n",
       " ('using', 1),\n",
       " ('notebooks', 2),\n",
       " ('already', 1)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCounts = textFile.flatMap(lambda line: line.split()).map(lambda word: (word, 1)).reduceByKey(lambda a, b: a+b)\n",
    "wordCounts.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "163066"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csv.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}