Spark Introduction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MapReduce as a programming model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 3, 4, 2, 7]\n",
      "[1, 9, 16, 4, 49]\n",
      "79\n",
      "49\n"
     ]
    }
   ],
   "source": [
    "def fun(x):\n",
    "    return x**2\n",
    "\n",
    "def add(x,y):\n",
    "    return x+y\n",
    "\n",
    "def maxi(x,y):\n",
    "    return max(x,y)\n",
    "\n",
    "\n",
    "L = [1,3,4,2,7]\n",
    "\n",
    "S = map(fun, L)\n",
    "\n",
    "print(L)\n",
    "print(S)\n",
    "\n",
    "print reduce(add, S)\n",
    "print reduce(maxi, S)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spark\n",
    "\n",
    "RDD : Resilient Distributed Dataset\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load a text file on the HDFS as a RDD\n",
    "* Display number of lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "132647"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "#textFile = sc.textFile(\"data/books-eng/hamlet.txt\")\n",
    "textFile = sc.textFile(\"data/books-eng\")\n",
    "textFile.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the first line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'ACT I'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find number of lines where a word appears"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40486"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word = \"the\"\n",
    "textFile.filter(lambda line: word in line).count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get a random sample from the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u\"The triple pillar of the world transform'd\""
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.sample(withReplacement=False, fraction=0.05).first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find the line with most words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "122"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.map(lambda line: len(line.split())).reduce(lambda a, b: a if (a > b) else b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Counting words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'rear.', 2),\n",
       " (u'yellow', 50),\n",
       " (u'grenadier,', 2),\n",
       " (u'men?...', 2),\n",
       " (u'approchez,', 2),\n",
       " (u'mentioning.', 2),\n",
       " (u'Certains', 2),\n",
       " (u'crossbar', 2),\n",
       " (u'vanterie,', 2),\n",
       " (u'CORRIDOR', 42),\n",
       " (u'circonstances.', 2),\n",
       " (u'wood:', 4),\n",
       " (u'Ottoman.', 2),\n",
       " (u'writing.', 12),\n",
       " (u'feigned.', 2),\n",
       " (u'seven,', 4),\n",
       " (u'her!', 36),\n",
       " (u'WOOD,', 2),\n",
       " (u'Napoleon--that', 2),\n",
       " (u'atom.', 2),\n",
       " (u'chin,', 10),\n",
       " (u'colloquies', 2),\n",
       " (u'humbugged', 4),\n",
       " (u'apartments?\"', 2),\n",
       " (u'rencontrerons', 2),\n",
       " (u'so...quiet.', 2),\n",
       " (u'kid!', 6),\n",
       " (u'momentary;', 2),\n",
       " (u'understood.', 16),\n",
       " (u'Poltava', 2),\n",
       " (u'admiral', 8),\n",
       " (u'kid.', 22),\n",
       " (u'insinuation', 2),\n",
       " (u'toyshop,', 2),\n",
       " (u'fervent', 2),\n",
       " (u\"X-wing's\", 2),\n",
       " (u'hers', 38),\n",
       " (u'undivulged', 2),\n",
       " (u'idiocy--Gossner', 2),\n",
       " (u'shriek', 8),\n",
       " (u'momie', 2),\n",
       " (u'sooner;', 2),\n",
       " (u'Stop,', 8),\n",
       " (u'sooner.', 4),\n",
       " (u'pressed,', 4),\n",
       " (u'\"shelter\"', 4),\n",
       " (u'remnant', 4),\n",
       " (u\"CAESAR's\", 26),\n",
       " (u'Indeed,', 30),\n",
       " (u'MALCOLM', 82),\n",
       " (u'cents', 2),\n",
       " (u'fingertips.', 2),\n",
       " (u'symphony', 2),\n",
       " (u'waist,', 8),\n",
       " (u'dry,', 6),\n",
       " (u'Stepanovich', 2),\n",
       " (u'relax', 4),\n",
       " (u'Worship.', 2),\n",
       " (u'mirth;', 6),\n",
       " (u'feeders?', 2),\n",
       " (u'shakos.', 2),\n",
       " (u'degrees;', 2),\n",
       " (u'burneth', 4),\n",
       " (u\"Macbeth's\", 8),\n",
       " (u'POPILIUS', 4),\n",
       " (u'boomed,', 2),\n",
       " (u'stooped,', 2),\n",
       " (u'packthread', 2),\n",
       " (u'Heath.]', 2),\n",
       " (u'world--is', 2),\n",
       " (u'ungraceful', 2),\n",
       " (u'ASCII\"', 8),\n",
       " (u'adjutants-general', 2),\n",
       " (u'expressed.', 6),\n",
       " (u'Announce', 2),\n",
       " (u'organizer', 2),\n",
       " (u'Volumnius.', 4),\n",
       " (u'gusts', 4),\n",
       " (u'reproached,', 2),\n",
       " (u'pas.]', 2),\n",
       " (u'd\\xe9mon!', 4),\n",
       " (u\"unfather'd'\", 2),\n",
       " (u'misfortune!', 4),\n",
       " (u'misfortune,', 6),\n",
       " (u'Malchus', 2),\n",
       " (u'post-haste', 2),\n",
       " (u'Hyrcanian', 2),\n",
       " (u'guerri\\xe8re!', 2),\n",
       " (u'approbation.', 2),\n",
       " (u'concurrence', 4),\n",
       " (u'jurerais', 2),\n",
       " (u'administered.', 2),\n",
       " (u\"Caesar's,\", 2),\n",
       " (u'wickedly', 2),\n",
       " (u'died.', 16),\n",
       " (u'SIX:', 2),\n",
       " (u'danger,\"', 2),\n",
       " (u'revenue', 4),\n",
       " (u'dump,', 2),\n",
       " (u'd\\xe9sormais', 4),\n",
       " (u'copilot,', 8),\n",
       " (u'Hie', 20),\n",
       " (u'stockings.', 4),\n",
       " (u'fil', 2),\n",
       " (u'amour!--et', 2),\n",
       " (u'FALKINBURG', 2),\n",
       " (u'Sojourn', 2),\n",
       " (u'throned', 4),\n",
       " (u'interrupt', 14),\n",
       " (u'good?\"', 2),\n",
       " (u\"''Tis\", 2),\n",
       " (u'senator,', 6),\n",
       " (u'noble,', 18),\n",
       " (u'truant.', 2),\n",
       " (u'Wostovs', 2),\n",
       " (u'fancies.', 2),\n",
       " (u'dumps', 2),\n",
       " (u'Emperor!\"', 12),\n",
       " (u'dark-faced', 4),\n",
       " (u'unreasonable,', 2),\n",
       " (u'bowls', 6),\n",
       " (u'Accountant', 4),\n",
       " (u'commodities.', 2),\n",
       " (u'self,', 16),\n",
       " (u'sauv\\xe9es.', 2),\n",
       " (u'aller,', 2),\n",
       " (u'go!\"', 24),\n",
       " (u'dug,', 6),\n",
       " (u'hobby-horse,', 2),\n",
       " (u'sweeter,', 2),\n",
       " (u'musician:', 2),\n",
       " (u'temper)', 4),\n",
       " (u'fugitives.', 2),\n",
       " (u'good-bye...I', 2),\n",
       " (u'Krasnaya', 4),\n",
       " (u'noticed,\"', 2),\n",
       " (u'woe,', 18),\n",
       " (u'snake,', 4),\n",
       " (u'satisfactorily.', 2),\n",
       " (u'speedy', 10),\n",
       " (u\"women's.\", 2),\n",
       " (u'Duc', 12),\n",
       " (u'lunge,', 2),\n",
       " (u'likes.', 2),\n",
       " (u'\"Anna', 4),\n",
       " (u'filching', 2),\n",
       " (u'hay:', 2),\n",
       " (u'easy?', 4),\n",
       " (u'nation.', 8),\n",
       " (u'blossoms', 2),\n",
       " (u'was?', 4),\n",
       " (u\"pester'd\", 2),\n",
       " (u'maladie,', 2),\n",
       " (u'task', 34),\n",
       " (u\"quoiqu'elle\", 2),\n",
       " (u'DANIELL,', 2),\n",
       " (u'CASSIO.--Ardente', 2),\n",
       " (u'leg,', 24),\n",
       " (u'(everything', 2),\n",
       " (u'old;', 2),\n",
       " (u'spoken?', 2),\n",
       " (u\"please--I've\", 2),\n",
       " (u'seminary', 2),\n",
       " (u'really.\"', 2),\n",
       " (u'menteuse,', 2),\n",
       " (u'well,', 200),\n",
       " (u'hacking,', 2),\n",
       " (u'heads:', 4),\n",
       " (u'fuyait', 2),\n",
       " (u'Asp', 2),\n",
       " (u'matin?', 2),\n",
       " (u'KOUTOUZOV:', 2),\n",
       " (u'Dessalles.', 8),\n",
       " (u'soaked', 4),\n",
       " (u'fois.', 2),\n",
       " (u'willing!', 2),\n",
       " (u'What!', 4),\n",
       " (u'tort.', 8),\n",
       " (u'drunk:', 4),\n",
       " (u'fugitive:', 2),\n",
       " (u'motivator.', 2),\n",
       " (u'feverishly,', 2),\n",
       " (u'place?\"', 4),\n",
       " (u'half-insane', 2),\n",
       " (u'believes,', 2),\n",
       " (u'wench,', 4),\n",
       " (u'long.', 44),\n",
       " (u'difficulties,', 2),\n",
       " (u'[Drum', 8),\n",
       " (u'est;', 2),\n",
       " (u'CASSIO.--Celle', 2),\n",
       " (u'affair:', 4),\n",
       " (u'professor', 4),\n",
       " (u'door?', 4),\n",
       " (u'prim', 2),\n",
       " (u'momentum,', 2),\n",
       " (u'prix', 8),\n",
       " (u'shaking,', 4),\n",
       " (u'tempted', 6),\n",
       " (u'sewn.', 2),\n",
       " (u'apace', 2),\n",
       " (u'r\\xe9v\\xe8le.', 2),\n",
       " (u'ceased.', 12),\n",
       " (u'maids,', 18),\n",
       " (u'Immediately,', 6),\n",
       " (u'mend', 20),\n",
       " (u'lies?', 2),\n",
       " (u'spheres,', 4),\n",
       " (u'stammered', 6),\n",
       " (u'came;--with', 2),\n",
       " (u'home!...', 2),\n",
       " (u'salve', 4),\n",
       " (u'post-horses;', 2),\n",
       " (u'produced,', 2),\n",
       " (u'patch', 14),\n",
       " (u'crossway.', 2),\n",
       " (u'avenged', 2),\n",
       " (u\"leader's\", 2),\n",
       " (u'momentan\\xe9ment', 2),\n",
       " (u'Montague!', 4),\n",
       " (u'blew', 12),\n",
       " (u'watchword--shaft,', 2),\n",
       " (u'fais', 16),\n",
       " (u'guerre', 12),\n",
       " (u'Montague,', 14),\n",
       " (u'Montague;', 6),\n",
       " (u'estate\"', 2),\n",
       " (u'stamps', 4),\n",
       " (u'curves,', 2),\n",
       " (u'trigger,', 4),\n",
       " (u\"Siward's\", 2),\n",
       " (u'half-built', 2),\n",
       " (u'skull,', 8),\n",
       " (u'nature', 160),\n",
       " (u'Perchance!', 2),\n",
       " (u'roofless,', 2),\n",
       " (u'pad.', 2),\n",
       " (u'accident', 12),\n",
       " (u'lot?', 2),\n",
       " (u'Wednesday,', 2),\n",
       " (u\"qu'au\", 4),\n",
       " (u'thought,\"', 4),\n",
       " (u'propagate,', 2),\n",
       " (u'ring,', 14),\n",
       " (u'to-morrow:', 8),\n",
       " (u'quests', 2),\n",
       " (u'widower?\"', 2),\n",
       " (u'(sighing)', 2),\n",
       " (u'doubt.', 8),\n",
       " (u'Colonel.', 2),\n",
       " (u'acquainted.\"', 2),\n",
       " (u'choix.', 4),\n",
       " (u'philanthropy,', 4),\n",
       " (u'startled', 28),\n",
       " (u'ports.', 4),\n",
       " (u'held;', 2),\n",
       " (u'say?\"', 20),\n",
       " (u'form--the', 2),\n",
       " (u'barque', 10),\n",
       " (u'LXXI.', 2),\n",
       " (u'shock.', 4),\n",
       " (u'rapidity.', 4),\n",
       " (u'crupper', 4),\n",
       " (u'flooding', 4),\n",
       " (u'congratulation', 2),\n",
       " (u'equator', 2),\n",
       " (u'angle--we', 2),\n",
       " (u'said;', 16),\n",
       " (u'LON', 2),\n",
       " (u'wonder-working', 6),\n",
       " (u'levied', 2),\n",
       " (u'it--take', 2),\n",
       " (u\"Ten's\", 4),\n",
       " (u'Y-wing.', 2),\n",
       " (u'passed,', 16),\n",
       " (u\"'death;'\", 2),\n",
       " (u'increased.', 4),\n",
       " (u'Jawas.', 2),\n",
       " (u\"Plutus'\", 2),\n",
       " (u'Contrary', 8),\n",
       " (u'Natasha.\"', 2),\n",
       " (u'eBook,', 4),\n",
       " (u'thing?', 20),\n",
       " (u\"Ilagin's,\", 2),\n",
       " (u'told,\"', 2),\n",
       " (u'Fett.', 4),\n",
       " (u'juridical', 4),\n",
       " (u'malefactor.', 2),\n",
       " (u'monkeys,', 2),\n",
       " (u'astir.', 2),\n",
       " (u'addresses.', 4),\n",
       " (u'cheerly,', 2),\n",
       " (u'bellow', 2),\n",
       " (u'bush.', 2),\n",
       " (u'desertion,', 2),\n",
       " (u'own--there', 2),\n",
       " (u'CX.', 2),\n",
       " (u\"'Everyone\", 2),\n",
       " (u'Ligne', 2),\n",
       " (u'Surely', 8),\n",
       " (u'quiet-like.', 2),\n",
       " (u\"one,'\", 2),\n",
       " (u'sabretaches,', 2),\n",
       " (u'vehicles.', 6),\n",
       " (u\"t'a\", 4),\n",
       " (u'samovar--and', 2),\n",
       " (u'sleep,', 64),\n",
       " (u'opinion)', 2),\n",
       " (u'fournit', 2),\n",
       " (u'Peut-\\xeatre', 2),\n",
       " (u'spectacle,', 2),\n",
       " (u'begin', 188),\n",
       " (u'left!', 6),\n",
       " (u'desired--to', 2),\n",
       " (u'stealthily', 10),\n",
       " (u'demerits,', 2),\n",
       " (u'Macduff:', 4),\n",
       " (u'Stoop,', 4),\n",
       " (u'(switching', 2),\n",
       " (u'Mautern', 2),\n",
       " (u'Citizens,', 4),\n",
       " (u'Decius', 10),\n",
       " (u'foh', 2),\n",
       " (u'chemin,', 2),\n",
       " (u'forever.', 16),\n",
       " (u'unfrozen', 2),\n",
       " (u'JAGO.--Eh', 14),\n",
       " (u'serpent.', 2),\n",
       " (u'ruffle,', 4),\n",
       " (u'twain.', 6),\n",
       " (u'slightly', 134),\n",
       " (u'Use', 26),\n",
       " (u'statements', 6),\n",
       " (u'Scotland', 12),\n",
       " (u'administer', 4),\n",
       " (u'gripe,', 2),\n",
       " (u'parlait', 2),\n",
       " (u'wrap', 4),\n",
       " (u'meats,', 2),\n",
       " (u'gauzy', 2),\n",
       " (u'Count,', 38),\n",
       " (u'lay,', 10),\n",
       " (u'avais', 6),\n",
       " (u'width', 2),\n",
       " (u'Count;', 2),\n",
       " (u\"boy's\", 14),\n",
       " (u'said?\"', 2),\n",
       " (u'traitors,\"', 2),\n",
       " (u'irregularities.', 2),\n",
       " (u'spurting', 2),\n",
       " (u'straggly', 2),\n",
       " (u'tigers', 2),\n",
       " (u'brute,', 4),\n",
       " (u'Latin,', 2),\n",
       " (u'rippling', 2),\n",
       " (u'full-fortuned', 2),\n",
       " (u'serf,', 6),\n",
       " (u'panels', 6),\n",
       " (u'PANEL.', 12),\n",
       " (u'Sit', 14),\n",
       " (u'dispos\\xe9,', 2),\n",
       " (u'proves', 20),\n",
       " (u'exist', 28),\n",
       " (u'Brundusium', 2),\n",
       " (u'bien:', 4),\n",
       " (u'disentangled', 4),\n",
       " (u'Track', 2),\n",
       " (u'minds,', 10),\n",
       " (u'offerings,', 2),\n",
       " (u'crowns', 4),\n",
       " (u'flood', 12),\n",
       " (u'vanished,', 4),\n",
       " (u'LOUIS', 2),\n",
       " (u'STARDESTROYER.', 2),\n",
       " (u'transportee', 2),\n",
       " (u\"servants'\", 10),\n",
       " (u'smelling', 6),\n",
       " (u'action.)', 2),\n",
       " (u'crown?', 2),\n",
       " (u'invitations,', 2),\n",
       " (u'DUC.--Brave', 2),\n",
       " (u\"'A\", 14),\n",
       " (u'Empire', 34),\n",
       " (u'crown!', 2),\n",
       " (u'whoever', 14),\n",
       " (u'voice--\"now', 2),\n",
       " (u'l\\'Empereur!\"', 18),\n",
       " (u\"l'impossibilit\\xe9\", 2),\n",
       " (u'JAGO.--Quelle', 2),\n",
       " (u'coincidence!', 2),\n",
       " (u'bribes', 2),\n",
       " (u'happy.\"', 8),\n",
       " (u'rousing', 4),\n",
       " (u'coincidence.', 2),\n",
       " (u'\"First,', 4),\n",
       " (u'drinking?\"', 2),\n",
       " (u'buddy,', 4),\n",
       " (u'apt;', 2),\n",
       " (u'reining', 4),\n",
       " (u'howling,', 2),\n",
       " (u'poor.', 8),\n",
       " (u'all-disgraced', 2),\n",
       " (u\"occupation's\", 2),\n",
       " (u'unworthy.', 2),\n",
       " (u'hindered', 16),\n",
       " (u'M.', 34),\n",
       " (u'thrumming', 2),\n",
       " (u'aims,', 6),\n",
       " (u'emm\\xe8ne', 2),\n",
       " (u'kindliness', 4),\n",
       " (u'SMITH,', 10),\n",
       " (u\"So,'\", 2),\n",
       " (u\"JAGO.--Qu'y\", 2),\n",
       " (u'tiresome,', 2),\n",
       " (u'saw,', 14),\n",
       " (u'celebrates', 2),\n",
       " (u'jury', 2),\n",
       " (u'concepts', 2),\n",
       " (u'plucking', 2),\n",
       " (u'soup\\xe7onne,', 2),\n",
       " (u'trail.', 8),\n",
       " (u'alone', 376),\n",
       " (u'muskets', 24),\n",
       " (u'elle-m\\xeame._]', 2),\n",
       " (u'roan,', 2),\n",
       " (u'Dack', 8),\n",
       " (u\"seem'd,\", 2),\n",
       " (u'WEDGE', 18),\n",
       " (u'predictions', 4),\n",
       " (u'vigour', 2),\n",
       " (u'villainous!', 2),\n",
       " (u'He-e-elp!', 2),\n",
       " (u'scratched', 8),\n",
       " (u'soumis', 4),\n",
       " (u'August,\"', 2),\n",
       " (u'irrit\\xe9', 4),\n",
       " (u'inflamed', 6),\n",
       " (u'values', 6),\n",
       " (u'peppered,', 4),\n",
       " (u\"doll's,\", 2),\n",
       " (u'vraisemblable', 2),\n",
       " (u\"'scaped.\", 2),\n",
       " (u'candles?...', 2),\n",
       " (u'bridal.', 2),\n",
       " (u'scraggly', 2),\n",
       " (u'unyoke.', 2),\n",
       " (u\"n'attendent\", 2),\n",
       " (u\"s'\", 2),\n",
       " (u'snaps', 18),\n",
       " (u'hide.', 2),\n",
       " (u\"l'objet\", 2),\n",
       " (u'don\\'t.\"', 2),\n",
       " (u'cut-throats:', 2),\n",
       " (u'mangy', 2),\n",
       " (u\"qu'un[3].\", 2),\n",
       " (u'biographical', 2),\n",
       " (u'Duncan.', 2),\n",
       " (u'doom.', 8),\n",
       " (u'foundations.', 2),\n",
       " (u'runner,', 2),\n",
       " (u'noyer', 4),\n",
       " (u'distempering', 2),\n",
       " (u'privation', 8),\n",
       " (u'ROYAL', 2),\n",
       " (u'WATSON', 4),\n",
       " (u'cigars,', 2),\n",
       " (u'solitary', 20),\n",
       " (u\"d'ob\\xe9issance,\", 2),\n",
       " (u'unobtrusively,', 2),\n",
       " (u'troubler', 4),\n",
       " (u'tort', 4),\n",
       " (u'dwarfs', 2),\n",
       " (u'Hiding', 2),\n",
       " (u'modestly', 8),\n",
       " (u'haut,', 2),\n",
       " (u'PIER', 2),\n",
       " (u'CUMMING', 2),\n",
       " (u'pendre', 2),\n",
       " (u'greatcoats', 8),\n",
       " (u'fully.', 6),\n",
       " (u'steers:', 2),\n",
       " (u'trouvez-vous?', 6),\n",
       " (u'crushing', 16),\n",
       " (u'gloomy-looking', 2),\n",
       " (u'overhead,', 4),\n",
       " (u'domes,', 2),\n",
       " (u\"j'engage\", 2),\n",
       " (u'citadelle,', 2),\n",
       " (u'flutes', 2),\n",
       " (u'Deluxe', 2),\n",
       " (u'Owen,', 4),\n",
       " (u'astray', 4),\n",
       " (u'JESUS', 2),\n",
       " (u'PORTMAN', 2),\n",
       " (u'exploding', 14),\n",
       " (u\"l'ann\\xe9e\", 4),\n",
       " (u'Charmian', 2),\n",
       " (u'seventh', 22),\n",
       " (u'possibility', 112),\n",
       " (u'flurried', 4),\n",
       " (u'patronage', 10),\n",
       " (u'please!\"', 10),\n",
       " (u'beggar-maid!', 2),\n",
       " (u'logis.', 4),\n",
       " (u'guardhouse', 4),\n",
       " (u'\"For', 46),\n",
       " (u'coins.', 2),\n",
       " (u'thyself:', 2),\n",
       " (u'seventy,', 2),\n",
       " (u'\"Poor', 4),\n",
       " (u\"'Ah,\", 6),\n",
       " (u'movements--as', 2),\n",
       " (u'chere', 8),\n",
       " (u'afford.', 2),\n",
       " (u'shawl', 34),\n",
       " (u'Alexas,', 8),\n",
       " (u'emprunt\\xe9', 4),\n",
       " (u'authority.', 6),\n",
       " (u'appreciation.', 4),\n",
       " (u'cavalier', 2),\n",
       " (u'enriched,', 2),\n",
       " (u'friend...', 6),\n",
       " (u'messagers', 6),\n",
       " (u'consolation?\"', 2),\n",
       " (u'uninterrupted', 6),\n",
       " (u'graveness.', 2),\n",
       " (u'illness.', 10),\n",
       " (u'horse,\"', 8),\n",
       " (u'perdre', 14),\n",
       " (u\"sail!'\", 6),\n",
       " (u'bad,', 48),\n",
       " (u\"teacher's\", 2),\n",
       " (u'clock....', 2),\n",
       " (u'Drop', 4),\n",
       " (u\"fell'st\", 2),\n",
       " (u'horridly', 4),\n",
       " (u'nice.', 4),\n",
       " (u'armrest', 2),\n",
       " (u'City.', 8),\n",
       " (u'booty,', 2),\n",
       " (u'blockhead', 2),\n",
       " (u'downward.', 6),\n",
       " (u'Switzers?', 2),\n",
       " (u'Halfway', 8),\n",
       " (u'Juliet,', 34),\n",
       " (u'died--this', 2),\n",
       " (u\"Clamour'd\", 2),\n",
       " (u'Juliet!', 6),\n",
       " (u'Praskovya', 2),\n",
       " (u'cornered.', 2),\n",
       " (u\"l'aurai\", 2),\n",
       " (u'Gwiska--my', 2),\n",
       " (u'serais', 2),\n",
       " (u'hindrance,', 2),\n",
       " (u\"gunner's\", 2),\n",
       " (u'diff\\xe9rentes', 2),\n",
       " (u'lanterns.', 4),\n",
       " (u'to...\"', 8),\n",
       " (u'uppermost', 2),\n",
       " (u\"fee'd.\", 2),\n",
       " (u'tethered,', 2),\n",
       " (u'character,\"', 2),\n",
       " (u'come;', 46),\n",
       " (u'usual.', 30),\n",
       " (u'woos,', 2),\n",
       " (u'keys.', 2),\n",
       " (u'r\\xe9els,', 4),\n",
       " (u'monde', 38),\n",
       " (u'usual;', 2),\n",
       " (u'factotum', 2),\n",
       " (u'DESD\\xc9MONA.--Seulement', 2),\n",
       " (u'business.\"', 4),\n",
       " (u'hinges', 4),\n",
       " (u'recurred', 14),\n",
       " (u'carrion', 2),\n",
       " (u'bombs', 2),\n",
       " (u'attaqu\\xe9', 2),\n",
       " (u'foreseen,', 4),\n",
       " (u'\"Duport!', 2),\n",
       " (u'happy,\"', 2),\n",
       " (u'That', 1682),\n",
       " (u'concentrate.', 4),\n",
       " (u'moldy', 2),\n",
       " (u'trader,\\'\"', 2),\n",
       " (u'estrangement,', 2),\n",
       " (u'true...', 6),\n",
       " (u'consid\\xe8re', 2),\n",
       " (u'impression.', 10),\n",
       " (u'pures,', 2),\n",
       " (u'spartan', 2),\n",
       " (u'responds,', 2),\n",
       " (u'spark', 18),\n",
       " (u'conversed', 4),\n",
       " (u'moon,', 42),\n",
       " (u'George--I', 2),\n",
       " (u'gluttoning', 2),\n",
       " (u'unlimbered', 8),\n",
       " (u's', 10),\n",
       " (u'd\\xe9r\\xe9gl\\xe9s.', 2),\n",
       " (u'gouverneur.', 4),\n",
       " (u'affectionate,', 4),\n",
       " (u'surviendront', 2),\n",
       " (u'twin', 8),\n",
       " (u'moans', 14),\n",
       " (u'hardiesse;', 2),\n",
       " (u'doctor-like', 2),\n",
       " (u'teddy', 2),\n",
       " (u'virtuous:', 2),\n",
       " (u'agility,', 2),\n",
       " (u'variance.', 2),\n",
       " (u'combined', 18),\n",
       " (u'observe', 36),\n",
       " (u'Melyukova,', 2),\n",
       " (u'bewildered,', 2),\n",
       " (u'close-clinging', 2),\n",
       " (u'pastime.', 4),\n",
       " (u'wantonness;', 2),\n",
       " (u'throttle,', 4),\n",
       " (u'militiamen,', 4),\n",
       " (u'_chantant_.', 2),\n",
       " (u'situation', 46),\n",
       " (u'written,', 12),\n",
       " (u'alone--with', 2),\n",
       " (u'gaderffii', 2),\n",
       " (u'ingrats.', 2),\n",
       " (u'fame', 22),\n",
       " (u'hidden,', 2),\n",
       " (u'Iago:', 10),\n",
       " (u'inns,', 2),\n",
       " (u'preparatory', 4),\n",
       " (u'coexisting', 2),\n",
       " (u'cost.', 2),\n",
       " (u'thief.', 4),\n",
       " (u'cost;', 2),\n",
       " (u'play!\"', 2),\n",
       " (u'associations,', 2),\n",
       " (u'lois.', 2),\n",
       " (u'way,', 172),\n",
       " (u'births', 2),\n",
       " (u'IRA', 2),\n",
       " (u'pourquoi?', 8),\n",
       " (u'benefactor,\"', 6),\n",
       " (u\"token'd\", 2),\n",
       " (u'jest--that', 2),\n",
       " (u'madness:', 6),\n",
       " (u'CAMPSITE', 4),\n",
       " (u'slime', 8),\n",
       " (u'judiciously', 2),\n",
       " (u'followed?', 2),\n",
       " (u'fateful', 2),\n",
       " (u'different.)', 2),\n",
       " (u'nuisance,\"', 2),\n",
       " (u'Forminsk.', 2),\n",
       " (u'muskets;', 2),\n",
       " (u'flames...', 2),\n",
       " (u'operation.', 2),\n",
       " (u'loudly.', 8),\n",
       " (u'plumes', 18),\n",
       " (u'well--\"I', 2),\n",
       " (u'alarms', 4),\n",
       " (u'unerring', 2),\n",
       " (u'devrais', 4),\n",
       " (u'savant,', 2),\n",
       " (u'_part_.--Il', 2),\n",
       " (u'rompu', 2),\n",
       " (u'snare', 2),\n",
       " (u'don\\'t,\"', 4),\n",
       " (u\"aide-de-camp's\", 2),\n",
       " (u'hereabout:', 4),\n",
       " (u'Yourself', 8),\n",
       " (u'triviality--these', 2),\n",
       " (u'seating', 6),\n",
       " (u\"couldn't\", 42),\n",
       " (u'putrefaction', 2),\n",
       " (u'divine', 60),\n",
       " (u'Philippe,', 2),\n",
       " (u'packet?\"', 2),\n",
       " (u'copse,', 4),\n",
       " (u'attacks,', 6),\n",
       " (u'Gertrude;', 2),\n",
       " (u'BULLOCH', 2),\n",
       " (u'countenances', 2),\n",
       " (u'red-roofed', 2),\n",
       " (u'SQUARE', 4),\n",
       " (u'power', 440),\n",
       " (u\"doctor's\", 32),\n",
       " (u'think:', 2),\n",
       " (u'alternately,', 2),\n",
       " (u'CASSIO.--Par', 4),\n",
       " (u'monitors.', 2),\n",
       " (u'Seneca', 2),\n",
       " (u'cakes;', 2),\n",
       " (u'approaching,', 2),\n",
       " (u'itself--how', 2),\n",
       " (u'fr\\xe9mis', 2),\n",
       " (u'horses--we', 2),\n",
       " (u\"'Stroy'd\", 2),\n",
       " (u'CREATURE', 42),\n",
       " (u'electricity.', 2),\n",
       " (u'tap-dances', 2),\n",
       " (u'gibes,', 2),\n",
       " (u'mourn,', 2),\n",
       " (u'her', 9084),\n",
       " (u'Walk,', 2),\n",
       " (u'bookcases', 2),\n",
       " (u'rag,', 4),\n",
       " (u'Zweck', 2),\n",
       " (u'birth-day;', 2),\n",
       " (u'Bartenstein.', 2),\n",
       " (u'another;', 14),\n",
       " (u'happen--despite', 2),\n",
       " (u'recommends', 4),\n",
       " (u'wits', 18),\n",
       " (u'another!', 4),\n",
       " (u\"they're\", 68),\n",
       " (u'honour;', 2),\n",
       " (u'capitulation?\"', 4),\n",
       " (u'c\\xf4tes.', 2),\n",
       " (u'impute', 4),\n",
       " (u'dirty', 48),\n",
       " (u'politically', 4),\n",
       " (u'dish.', 2),\n",
       " (u'pourriez-vous', 2),\n",
       " (u'excusing', 2),\n",
       " (u'au', 262),\n",
       " (u'creak', 14),\n",
       " (u\"e'en\", 24),\n",
       " (u'querulous', 8),\n",
       " (u'comprends-tu', 2),\n",
       " (u'disgraceful', 8),\n",
       " (u'elle.', 10),\n",
       " (u'discussed.', 4),\n",
       " (u'damnable', 2),\n",
       " (u'accent.', 6),\n",
       " (u'hennissent', 2),\n",
       " (u'contr\\xe9e.', 2),\n",
       " (u'EFFECTS,', 4),\n",
       " (u'original', 28),\n",
       " (u'faintly)', 2),\n",
       " (u\"to't,\", 6),\n",
       " (u'(it', 30),\n",
       " (u'consider', 130),\n",
       " (u'one....', 4),\n",
       " (u'(in', 70),\n",
       " (u'\"Robber!...', 2),\n",
       " (u'JENSON', 2),\n",
       " (u'duty!', 2),\n",
       " (u'\"Dearest,', 2),\n",
       " (u'detention', 12),\n",
       " (u'\"Europe', 2),\n",
       " (u'conscience,', 6),\n",
       " (u'beard.', 10),\n",
       " (u'emancipation.', 2),\n",
       " (u'stouter!...\"', 2),\n",
       " (u'marvellous', 6),\n",
       " (u'broken.', 2),\n",
       " (u'Donalbain!', 2),\n",
       " (u'broken;', 2),\n",
       " (u'br\\xfblante,', 2),\n",
       " (u'sighs:', 2),\n",
       " (u'lying?', 2),\n",
       " (u'healed', 4),\n",
       " (u'starfleet,', 2),\n",
       " (u'Restrain', 2),\n",
       " (u'pass', 160),\n",
       " (u'installation.', 2),\n",
       " (u'caleches,', 2),\n",
       " (u'b\\xe9lier', 2),\n",
       " (u'roadway;', 2),\n",
       " (u'lifted,', 2),\n",
       " (u'fleur', 2),\n",
       " (u'chaade', 2),\n",
       " (u'Calamari).', 2),\n",
       " (u'\"Lucky', 2),\n",
       " (u'subtils,', 2),\n",
       " (u'spoken?\"', 2),\n",
       " (u'mieux,', 2),\n",
       " (u'carriage:', 2),\n",
       " (u'yourself:', 12),\n",
       " (u'cherish.', 2),\n",
       " (u'pick', 34),\n",
       " (u\"capital!'\", 2),\n",
       " (u'Alsatian', 4),\n",
       " (u'fat;', 2),\n",
       " (u'sovereigns;', 2),\n",
       " (u'God;', 4),\n",
       " (u\"link'd\", 2),\n",
       " (u'salute:', 2),\n",
       " (u'gentleman\\'s\"', 4),\n",
       " (u'pearl', 8),\n",
       " (u'goeth', 4),\n",
       " (u'6', 10),\n",
       " (u'social,', 2),\n",
       " (u'\\xe9pouvante', 2),\n",
       " (u'God!', 58),\n",
       " (u'Fili?', 2),\n",
       " (u'attach\\xe9', 4),\n",
       " (u\"s'embrassaient:\", 2),\n",
       " (u'pursuers', 4),\n",
       " (u'Makeev?\"', 4),\n",
       " (u'jupons,', 2),\n",
       " (u'identify,', 4),\n",
       " (u'scan', 6),\n",
       " (u'front.\"', 2),\n",
       " (u'autumn', 50),\n",
       " (u'bandaged.', 4),\n",
       " (u'Tydirium', 2),\n",
       " (u'plan,', 42),\n",
       " (u'moaning,', 4),\n",
       " (u'scoundrels', 6),\n",
       " (u\"Confessor's\", 2),\n",
       " (u'ext\\xe9rieurs', 2),\n",
       " (u'latch,', 2),\n",
       " (u'tr\\xe8s-vif', 2),\n",
       " (u'appealed', 6),\n",
       " (u'brown,', 2),\n",
       " (u'scoundrel!', 4),\n",
       " (u'Visions', 2),\n",
       " (u'threw,', 2),\n",
       " (u'Birnam.', 4),\n",
       " (u'Purple', 2),\n",
       " (u'Dispose', 2),\n",
       " (u'looks;', 8),\n",
       " (u'promotions.)', 2),\n",
       " (u'tyrant!', 2),\n",
       " (u'army;', 20),\n",
       " (u'looks,', 26),\n",
       " (u'tassel-gentle', 2),\n",
       " (u'esclave,', 2),\n",
       " (u'traitor...', 2),\n",
       " (u'ronflent;', 2),\n",
       " (u'occulted', 2),\n",
       " (u'measures,', 4),\n",
       " (u\"furbish'd\", 2),\n",
       " (u'ruining', 4),\n",
       " (u'Tsars,', 2),\n",
       " (u'accidental.', 2),\n",
       " (u'it)', 20),\n",
       " (u'passage;', 4),\n",
       " (u'great!', 2),\n",
       " (u'it:', 96),\n",
       " (u'mazuwka', 2),\n",
       " (u'd\\xe9ployer,', 4),\n",
       " (u'Entrera-t-elle?', 2),\n",
       " (u'great,', 34),\n",
       " (u'coarse,', 12),\n",
       " (u'glisserai', 2),\n",
       " (u'resolute', 52),\n",
       " (u'contiguous', 2),\n",
       " (u'disclaiming', 2),\n",
       " (u'adjutants--was', 2),\n",
       " (u'Field,\"', 4),\n",
       " (u'Pompeius', 6),\n",
       " (u'streaking', 4),\n",
       " (u'sentinelle,', 2),\n",
       " (u'exclamations,', 2),\n",
       " (u'owner,', 6),\n",
       " (u\"s'\\xe9vanouira\", 2),\n",
       " (u'Whom?', 2),\n",
       " (u'later...\"', 2),\n",
       " (u'Nine', 4),\n",
       " (u'reaction.', 10),\n",
       " (u'knapsacks,', 4),\n",
       " (u'pairs', 8),\n",
       " (u'expedient', 6),\n",
       " (u'unsatisfied', 2),\n",
       " (u'Arakcheev.', 8),\n",
       " (u'Hurry.', 2),\n",
       " (u'Cassius:', 4),\n",
       " (u'new-planted', 2),\n",
       " (u'theatrically', 2),\n",
       " (u'determine,', 2),\n",
       " (u'embattle', 2),\n",
       " (u'Rostov),', 2),\n",
       " (u\"Cyril's\", 2),\n",
       " (u'Descended', 2),\n",
       " (u'head;', 14),\n",
       " (u'vocal', 2),\n",
       " (u'saddled.', 4),\n",
       " (u'you--the', 2),\n",
       " (u'Cancel', 2),\n",
       " (u'head.', 248),\n",
       " (u'wafture', 2),\n",
       " (u'milky', 6),\n",
       " (u'CASSIO.--Non,', 2),\n",
       " (u'him\"--little', 2),\n",
       " (u'skirt--this', 2),\n",
       " (u'hypochondria,', 2),\n",
       " (u'mend!', 6),\n",
       " (u'caress', 2),\n",
       " (u'mend,', 2),\n",
       " (u'melting!', 2),\n",
       " (u'Club', 52),\n",
       " (u'trowel', 2),\n",
       " (u'also', 538),\n",
       " (u'con\\xe7u...', 2),\n",
       " (u\"WEDGE'S\", 34),\n",
       " (u'odor)', 2),\n",
       " (u'effecting', 4),\n",
       " (u'confluence', 4),\n",
       " (u'explain....', 2),\n",
       " (u'\"Karataev!\"', 2),\n",
       " (u'\"Raising', 2),\n",
       " (u'[HECATE', 2),\n",
       " (u'eau,', 2),\n",
       " (u'wording', 4),\n",
       " (u'unimportance', 6),\n",
       " (u'Han', 618),\n",
       " (u\"You're\", 106),\n",
       " (u'produire', 4),\n",
       " (u'fatal,', 4),\n",
       " (u'seated,', 2),\n",
       " (u'forcible', 4),\n",
       " (u'adventurer.', 4),\n",
       " (u'maddened--even', 2),\n",
       " (u'Mamonov', 2),\n",
       " (u'fatal!', 2),\n",
       " (u\"kiss'd,--the\", 2),\n",
       " (u'pinched,', 2),\n",
       " (u'onwards,', 2),\n",
       " (u'indifference', 24),\n",
       " (u\"won't;\", 2),\n",
       " (u'meetly.', 2),\n",
       " (u'charmingly', 2),\n",
       " (u\"won't.\", 4),\n",
       " (u\"d'instances:\", 2),\n",
       " (u'column.', 6),\n",
       " (u'pleasures', 24),\n",
       " (u'\"Sufferer,\"', 2),\n",
       " (u'banking', 6),\n",
       " (u'infid\\xe8le:', 4),\n",
       " (u'write.', 4),\n",
       " (u'importunez-la;', 2),\n",
       " (u'secondly,', 12),\n",
       " (u'insane', 12),\n",
       " (u'stronger.', 8),\n",
       " (u'see', 1794),\n",
       " (u'presenteth', 2),\n",
       " (u'readjusted', 6),\n",
       " (u'rite', 6),\n",
       " (u'available', 16),\n",
       " (u'politics.', 6),\n",
       " (u'bienveillante', 2),\n",
       " (u'occupent:', 2),\n",
       " (u\"bescreen'd\", 2),\n",
       " (u'visible,', 8),\n",
       " (u'scamp', 2),\n",
       " (u\"'Twould\", 2),\n",
       " (u\"'s,\", 2),\n",
       " (u'\"Having', 2),\n",
       " (u'anxious,', 4),\n",
       " (u'pleasures,', 4),\n",
       " (u'bell', 26),\n",
       " (u'acted', 50),\n",
       " (u'sacr\\xe9.', 2),\n",
       " (u'\"d\\'', 2),\n",
       " (u'Murderer', 64),\n",
       " (u'loaded.', 2),\n",
       " (u\"That'll\", 4),\n",
       " (u'eyebrows.', 10),\n",
       " (u'd\\xe9vouement', 2),\n",
       " (u'towns', 22),\n",
       " (u'eyes.)', 2),\n",
       " (u'becomings', 2),\n",
       " (u'reliable', 14),\n",
       " (u'pressez', 2),\n",
       " (u'Seleucus?', 2),\n",
       " (u'BERU:', 14),\n",
       " (u'error', 22),\n",
       " (u\"von't\", 2),\n",
       " (u'commun.', 2),\n",
       " (u'unproportioned', 2),\n",
       " (u'dragged.', 2),\n",
       " (u'elevates', 2),\n",
       " (u'circumstances--did', 2),\n",
       " (u'Stardestroyer', 12),\n",
       " (u'parole.\"', 2),\n",
       " (u'late--\"I', 2),\n",
       " (u'health!\"', 2),\n",
       " (u'\"Rubbish,', 2),\n",
       " (u'viewing', 2),\n",
       " (u'earth--to', 2),\n",
       " (u'devil...?\"', 2),\n",
       " (u'betises!\"', 2),\n",
       " (u'obey,\"', 2),\n",
       " (u'honor', 160),\n",
       " (u'concrete', 2),\n",
       " (u'Talks', 2),\n",
       " (u'CREATURES', 2),\n",
       " (u'inches;', 2),\n",
       " (u'drenched,', 2),\n",
       " (u\"O'\", 6),\n",
       " (u'\"Commander', 6),\n",
       " (u'railings.', 2),\n",
       " (u'ah-ing.', 2),\n",
       " (u'drizzled', 2),\n",
       " (u'location', 12),\n",
       " (u'PATIE-FRELENG', 2),\n",
       " (u'impose:', 2),\n",
       " (u'yield;', 2),\n",
       " ...]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCounts = textFile.flatMap(lambda line: line.split()).map(lambda word: (word, 2)).reduceByKey(lambda a, b: a+b)\n",
    "wordCounts.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the Cache\n",
    "Pull a data sets into a cluster-wide in-memory cache. \n",
    "\n",
    "Very useful when using iterative algorithms.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "notes/data/hamlet.txt MapPartitionsRDD[30] at textFile at NativeMethodAccessorImpl.java:-2"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find words that start with an 'H' or 'h'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'honour.', 11),\n",
       " (u'hallucination.', 1),\n",
       " (u'Han', 309),\n",
       " (u'happy!', 4),\n",
       " (u'happen--she', 1),\n",
       " (u'honour;', 1),\n",
       " (u'harmlessly', 3),\n",
       " (u'her--filled', 1),\n",
       " (u'habiliments', 1),\n",
       " (u'heath.', 1),\n",
       " (u'heiress,', 1),\n",
       " (u'heavens', 20),\n",
       " (u'herself,\"', 1),\n",
       " (u'history:', 5),\n",
       " (u'happen--despite', 1),\n",
       " (u'heave.', 2),\n",
       " (u'hyssop', 1),\n",
       " (u'happy.\"', 4),\n",
       " (u'hotly,', 2),\n",
       " (u'him.\"', 31),\n",
       " (u'hopeful', 3),\n",
       " (u'happiness...', 1),\n",
       " (u'him?\"', 31),\n",
       " (u'humming,', 1),\n",
       " (u'Hie', 10),\n",
       " (u'hail,', 9),\n",
       " (u'host;', 1),\n",
       " (u'humanity', 27),\n",
       " (u'horse._]', 1),\n",
       " (u'happened.', 17),\n",
       " (u'haut,', 1),\n",
       " (u'hostess.', 8),\n",
       " (u'hears.', 1),\n",
       " (u'hardships', 3),\n",
       " (u'harms.', 1),\n",
       " (u'homes', 9),\n",
       " (u'home!...', 1),\n",
       " (u'homeopaths,', 1),\n",
       " (u'hangman.', 1),\n",
       " (u'hesitated:', 2),\n",
       " (u'harlot', 1),\n",
       " (u\"hang'd\", 2),\n",
       " (u'house--that', 1),\n",
       " (u'hero!', 1),\n",
       " (u'howling,', 1),\n",
       " (u'hereabout:', 2),\n",
       " (u'Hurrah!', 3),\n",
       " (u'hinges', 2),\n",
       " (u'histoire.', 1),\n",
       " (u'Hercules', 2),\n",
       " (u'hereafter;', 2),\n",
       " (u'homme?', 3),\n",
       " (u'happily...\"', 1),\n",
       " (u'hands...\"', 1),\n",
       " (u'help:', 2),\n",
       " (u'hanged,\"', 1),\n",
       " (u'harder', 8),\n",
       " (u'her!', 18),\n",
       " (u'himself,\"', 4),\n",
       " (u'here--that', 1),\n",
       " (u'Hippolyte,', 8),\n",
       " (u'healed', 2),\n",
       " (u'hounds,', 10),\n",
       " (u'hut,', 14),\n",
       " (u'him--of', 1),\n",
       " (u'hasard', 1),\n",
       " (u'having,', 7),\n",
       " (u'Hyrcan', 1),\n",
       " (u'honors', 9),\n",
       " (u'homage:', 2),\n",
       " (u'humbugged', 2),\n",
       " (u'happy,\"', 1),\n",
       " (u'hide', 60),\n",
       " (u'horseback--raised', 1),\n",
       " (u'hallways', 1),\n",
       " (u'horribly', 1),\n",
       " (u'hydraulic', 3),\n",
       " (u'hardiesse;', 1),\n",
       " (u'HUT', 1),\n",
       " (u'Hyperdrive.', 1),\n",
       " (u'hobby-horse,', 1),\n",
       " (u'HANSON,', 1),\n",
       " (u'Hoth,', 1),\n",
       " (u'Hold', 23),\n",
       " (u'honours.', 1),\n",
       " (u'half-built', 1),\n",
       " (u'He', 2227),\n",
       " (u'hai!', 1),\n",
       " (u'honest;', 2),\n",
       " (u'harm_!]', 1),\n",
       " (u'heathen?', 1),\n",
       " (u\"Helena.'\", 1),\n",
       " (u'honor?', 1),\n",
       " (u'hand....', 2),\n",
       " (u'Hussars', 1),\n",
       " (u'hands--the', 1),\n",
       " (u'hers', 19),\n",
       " (u'himself!...', 1),\n",
       " (u'hussars.\"', 1),\n",
       " (u'horsemen,', 1),\n",
       " (u'hiding', 24),\n",
       " (u\"honour's\", 1),\n",
       " (u'hell-gate,', 1),\n",
       " (u'hot!\"', 2),\n",
       " (u'hilts,', 1),\n",
       " (u\"have't!\", 1),\n",
       " (u'hurrying:', 1),\n",
       " (u'Hanging', 3),\n",
       " (u'horses--we', 1),\n",
       " (u'her--or', 1),\n",
       " (u'honor', 80),\n",
       " (u'Hungarian?\"', 1),\n",
       " (u'heures,', 1),\n",
       " (u'heedful', 1),\n",
       " (u'himself....', 1),\n",
       " (u'have;', 4),\n",
       " (u'Hugh', 1),\n",
       " (u'him--an', 3),\n",
       " (u'him?...\"', 1),\n",
       " (u'Hurry.', 1),\n",
       " (u'hidden,', 1),\n",
       " (u'h\\xe2ter', 1),\n",
       " (u'housekeeping,', 1),\n",
       " (u'hoof', 1),\n",
       " (u'horse.', 31),\n",
       " (u\"Haven't\", 2),\n",
       " (u'him,\"', 38),\n",
       " (u'horse', 162),\n",
       " (u'Hendrikhovna.', 3),\n",
       " (u'her', 4542),\n",
       " (u'hay:', 1),\n",
       " (u'handkerchief:', 2),\n",
       " (u'heir--apparent', 1),\n",
       " (u'home?', 4),\n",
       " (u'hothouses.', 1),\n",
       " (u'her...', 4),\n",
       " (u'him--which', 1),\n",
       " (u'habiles', 1),\n",
       " (u'hindered', 8),\n",
       " (u'HARRELL', 1),\n",
       " (u'hundred', 134),\n",
       " (u'hand--a', 1),\n",
       " (u'hussies.', 1),\n",
       " (u'He-e-elp!', 1),\n",
       " (u'held;', 1),\n",
       " (u'hug', 3),\n",
       " (u'happiness!\"', 3),\n",
       " (u'husbandmen,', 1),\n",
       " (u'happy...\"', 1),\n",
       " (u'hodgepodge', 1),\n",
       " (u'him...\"', 2),\n",
       " (u'hut', 35),\n",
       " (u'hindrance,', 1),\n",
       " (u'holes.', 1),\n",
       " (u'he', 8765),\n",
       " (u'helping,', 1),\n",
       " (u'holiday,', 2),\n",
       " (u'Hurry', 3),\n",
       " (u'Highness?', 1),\n",
       " (u'history--whatever', 1),\n",
       " (u'hour', 90),\n",
       " (u'Happily,', 1),\n",
       " (u'harnessed,', 1),\n",
       " (u'head;', 7),\n",
       " (u'Hills;', 1),\n",
       " (u'head.', 124),\n",
       " (u'Hiding', 1),\n",
       " (u'harbour,', 1),\n",
       " (u'helplessly,', 2),\n",
       " (u'holiday?', 2),\n",
       " (u'hastily),', 1),\n",
       " (u'housekeeper,', 3),\n",
       " (u\"help!'\", 1),\n",
       " (u'huh!', 2),\n",
       " (u'haze.', 1),\n",
       " (u'Heath.]', 1),\n",
       " (u'heroics.', 1),\n",
       " (u'hypochondria,', 1),\n",
       " (u'hennissent', 1),\n",
       " (u'honey', 11),\n",
       " (u'hacking,', 1),\n",
       " (u'Horatio:', 2),\n",
       " (u'Howlings', 1),\n",
       " (u'hearted', 1),\n",
       " (u'heraldry,', 1),\n",
       " (u'heads:', 2),\n",
       " (u'hesitate', 1),\n",
       " (u'Hoists', 1),\n",
       " (u'happen?\"', 3),\n",
       " (u\"honour'd\", 8),\n",
       " (u'horse,\"', 4),\n",
       " (u'handkerchiefs;', 1),\n",
       " (u'hangar', 14),\n",
       " (u'hers,', 11),\n",
       " (u'hatch.', 6),\n",
       " (u'howls.', 4),\n",
       " (u'heads.\"', 1),\n",
       " (u'heals,', 1),\n",
       " (u'half-insane', 1),\n",
       " (u'hide.', 1),\n",
       " (u'health!\"', 1),\n",
       " (u'halted', 14),\n",
       " (u'hitched', 1),\n",
       " (u'hilts;', 1),\n",
       " (u'horridly', 2),\n",
       " (u'horn--played', 1),\n",
       " (u'hatred.', 1),\n",
       " (u'him\"--little', 1),\n",
       " (u'hold:', 1),\n",
       " (u'half-burned', 2),\n",
       " (u'housewifery,', 1),\n",
       " (u'hinders', 3),\n",
       " (u'h\\xe9ritage.', 1),\n",
       " (u'Helicopter', 1),\n",
       " (u'honeying', 1),\n",
       " (u'Herein', 2),\n",
       " (u'hums,', 1),\n",
       " (u'heir,', 2),\n",
       " (u'Ha-ha-ha!...\"', 1),\n",
       " (u'Hyrcanian', 1),\n",
       " (u'human.', 1),\n",
       " (u'Halfway', 4),\n",
       " (u'half-dozen', 2),\n",
       " (u'howl', 5),\n",
       " (u'hasarde', 1),\n",
       " (u'heat,', 6),\n",
       " (u'humor,', 6),\n",
       " (u'him--everything', 1),\n",
       " (u'Heights', 3),\n",
       " (u'hence!', 4),\n",
       " (u'hare,', 10),\n",
       " (u'here\"--and', 1),\n",
       " (u'HAVE', 2),\n",
       " (u'hack', 5),\n",
       " (u'horse!\"', 6),\n",
       " (u'health...\"', 1),\n",
       " (u'hence,', 13),\n",
       " (u'horror,', 9),\n",
       " (u'housemaid...', 1),\n",
       " (u'hands--those', 1),\n",
       " (u'hovel', 1),\n",
       " (u'house.)', 1),\n",
       " (u'heights.', 1),\n",
       " (u'hap', 4),\n",
       " (u'holiday', 6),\n",
       " (u'host:', 1),\n",
       " (u'hadst,', 1),\n",
       " (u'hover', 1),\n",
       " (u'hadst', 13),\n",
       " (u'HANGAR.', 1),\n",
       " (u'history.', 15),\n",
       " (u'HUM.', 1),\n",
       " (u'houses--but', 1),\n",
       " (u'history;', 1),\n",
       " (u'harrow', 1),\n",
       " (u'heureuse,', 1),\n",
       " (u'Home\"', 2),\n",
       " (u'hate.', 5),\n",
       " (u'hooking', 1),\n",
       " (u'hey!\"', 1),\n",
       " (u'h', 2),\n",
       " (u'hunger.', 2),\n",
       " (u'helter-skelter', 1),\n",
       " (u'handiest', 1),\n",
       " (u'habits', 14),\n",
       " (u'humility.', 1),\n",
       " (u'half-closed', 4),\n",
       " (u'help;', 2),\n",
       " (u'heartsick', 1),\n",
       " (u'headquarters.', 5),\n",
       " (u'HOMESTEAD.', 3),\n",
       " (u'HERD,', 1),\n",
       " (u'Hussar,', 1),\n",
       " (u'happy?', 2),\n",
       " (u'HEAD', 1),\n",
       " (u'hmm?', 2),\n",
       " (u'Hollandais', 2),\n",
       " (u'hair', 105),\n",
       " (u'helped....', 1),\n",
       " (u'heard,\"', 3),\n",
       " (u'has:', 1),\n",
       " (u'hurries', 12),\n",
       " (u'hives;', 1),\n",
       " (u'hinges,', 1),\n",
       " (u'Hurrah', 3),\n",
       " (u'humming', 7),\n",
       " (u\"horse's\", 15),\n",
       " (u'however,\"', 1),\n",
       " (u'him....', 7),\n",
       " (u'heaviness', 2),\n",
       " (u'ha!...', 2),\n",
       " (u'hearted;', 1),\n",
       " (u'Huttese', 18),\n",
       " (u'Heat', 2),\n",
       " (u'herself,', 58),\n",
       " (u'hospitality,', 3),\n",
       " (u'habits.', 1),\n",
       " (u'harvesting,\"', 1),\n",
       " (u'harp,', 1),\n",
       " (u'HARTLEY', 1),\n",
       " (u'harmonized', 1),\n",
       " (u'he!', 7),\n",
       " (u'habitude?', 1),\n",
       " (u'harlotry,', 1),\n",
       " (u'hastens', 2),\n",
       " (u'heart,\"', 4),\n",
       " (u'hymn.', 2),\n",
       " (u'hangar.', 10),\n",
       " (u'Hullo,', 1),\n",
       " (u'hooted', 1),\n",
       " (u'high-placed', 1),\n",
       " (u'Historical', 1),\n",
       " (u'Hot,', 1),\n",
       " (u'harsh:', 1),\n",
       " (u'Honor', 5),\n",
       " (u'hungry!\"', 1),\n",
       " (u'handiwork.', 3),\n",
       " (u'healths', 1),\n",
       " (u'hindmost,', 1),\n",
       " (u'HOSKER,', 1),\n",
       " (u'heavily.', 1),\n",
       " (u'Huttese.', 2),\n",
       " (u'happen?', 4),\n",
       " (u'haunts', 1),\n",
       " (u'hue?', 1),\n",
       " (u'him),', 2),\n",
       " (u'H.', 4),\n",
       " (u'hies', 1),\n",
       " (u'hussars', 64),\n",
       " (u'humorously', 1),\n",
       " (u'hardi', 2),\n",
       " (u'here.', 98),\n",
       " (u'home.\"', 1),\n",
       " (u'Halt!', 1),\n",
       " (u'have:', 4),\n",
       " (u'habitual', 17),\n",
       " (u'How?', 3),\n",
       " (u'heard.\"', 3),\n",
       " (u'HATCH', 1),\n",
       " (u'hooks:', 1),\n",
       " (u'Hail!', 3),\n",
       " (u'ham,', 1),\n",
       " (u'highness,', 1),\n",
       " (u'hapless', 3),\n",
       " (u'horsemen!\"', 1),\n",
       " (u'halt.', 2),\n",
       " (u'humour,', 4),\n",
       " (u'husband--as', 1),\n",
       " (u'Helene,', 28),\n",
       " (u'hellish', 3),\n",
       " (u'hovels', 1),\n",
       " (u'humour?', 1),\n",
       " (u'Herself', 2),\n",
       " (u\"hangman's\", 1),\n",
       " (u\"harlot's\", 1),\n",
       " (u'hitting,', 1),\n",
       " (u'heart.\"', 4),\n",
       " (u'holy', 60),\n",
       " (u'high-sighted', 1),\n",
       " (u'hunters.', 1),\n",
       " (u'hiding.', 2),\n",
       " (u'Helene.\"', 1),\n",
       " (u\"heap'd\", 3),\n",
       " (u'harpoon.', 4),\n",
       " (u'heap', 14),\n",
       " (u\"here's\", 29),\n",
       " (u'humiliate', 2),\n",
       " (u'hush', 3),\n",
       " (u'hire', 5),\n",
       " (u'headdress,', 1),\n",
       " (u'House', 1),\n",
       " (u'habile,', 1),\n",
       " (u'head:', 7),\n",
       " (u'handle.', 2),\n",
       " (u'harvested', 1),\n",
       " (u'Harpier', 1),\n",
       " (u'highwrought', 1),\n",
       " (u'historical', 48),\n",
       " (u'hatchway', 2),\n",
       " (u\"hate's\", 2),\n",
       " (u'h\\xe9ritier', 1),\n",
       " (u'historical-pastoral,', 1),\n",
       " (u'helm,', 1),\n",
       " (u'heirs,', 3),\n",
       " (u'huh?', 7),\n",
       " (u\"He's\", 65),\n",
       " (u'happened.\"', 3),\n",
       " (u'husband)', 1),\n",
       " (u'him....\"', 2),\n",
       " (u'horseflies', 1),\n",
       " (u'herein', 3),\n",
       " (u'happens,', 11),\n",
       " (u'here...now,', 1),\n",
       " (u'hours,', 15),\n",
       " (u'Hop', 1),\n",
       " (u'Horatio;', 1),\n",
       " (u'handicraft,', 1),\n",
       " (u'him--and', 5),\n",
       " (u'hideuse.', 1),\n",
       " (u'hark.', 1),\n",
       " (u'house,--But,', 1),\n",
       " (u\"housewives'\", 1),\n",
       " (u'hampered', 2),\n",
       " (u'hole.', 2),\n",
       " (u'horrible!--Le', 1),\n",
       " (u'holding', 139),\n",
       " (u'happening:', 1),\n",
       " (u'hell.', 7),\n",
       " (u'how,', 12),\n",
       " (u'higher', 55),\n",
       " (u\"harp'd\", 1),\n",
       " (u'her--knew', 1),\n",
       " (u'horror!', 2),\n",
       " (u'hugger-mugger', 1),\n",
       " (u\"hurlyburly's\", 1),\n",
       " (u'Ho,', 6),\n",
       " (u'hazel', 3),\n",
       " (u'had.\"', 1),\n",
       " (u'hand', 426),\n",
       " (u'hard-hearted', 1),\n",
       " (u'hateth', 1),\n",
       " (u'happening', 35),\n",
       " (u'harvest,', 4),\n",
       " (u'history,\"', 2),\n",
       " (u'hard--', 1),\n",
       " (u'helplessness.', 2),\n",
       " (u'half-ruined', 1),\n",
       " (u'healthy,', 3),\n",
       " (u'haunted', 1),\n",
       " (u'hesitated,', 2),\n",
       " (u\"honor--it's\", 1),\n",
       " (u'historians--those', 1),\n",
       " (u'hurrah,', 1),\n",
       " (u'honour,', 16),\n",
       " (u'hordes', 1),\n",
       " (u'highmost', 2),\n",
       " (u'hugging', 3),\n",
       " (u'hurlements', 1),\n",
       " (u'hindrances', 1),\n",
       " (u'Hercules,', 2),\n",
       " (u'high-born', 1),\n",
       " (u'hatchway.', 1),\n",
       " (u'heiress.', 1),\n",
       " (u'hate', 43),\n",
       " (u'HECATE]', 2),\n",
       " (u'Higher', 4),\n",
       " (u'hand...', 1),\n",
       " (u'Hover', 1),\n",
       " (u'humored,', 1),\n",
       " (u'Hobbie,', 2),\n",
       " (u'Hadst', 4),\n",
       " (u'had', 5562),\n",
       " (u'him!...', 2),\n",
       " (u'honesty:', 1),\n",
       " (u\"Herod's\", 1),\n",
       " (u'households,', 2),\n",
       " (u'hosts!', 1),\n",
       " (u'hussar,', 7),\n",
       " (u\"haven't\", 32),\n",
       " (u'hindmost', 2),\n",
       " (u'heaven,', 49),\n",
       " (u'hostess,', 4),\n",
       " (u'horn,', 3),\n",
       " (u'Ha!', 16),\n",
       " (u'Half', 16),\n",
       " (u\"Han's\", 28),\n",
       " (u'his--has', 1),\n",
       " (u'here...', 5),\n",
       " (u'here.\"', 9),\n",
       " (u'happy?\"', 1),\n",
       " (u'HUSE,', 1),\n",
       " (u'hales,', 1),\n",
       " (u'hieroglyph', 1),\n",
       " (u'holiness', 1),\n",
       " (u'harried', 1),\n",
       " (u'hunt', 17),\n",
       " (u'honourably.', 1),\n",
       " (u'heureusement', 1),\n",
       " (u'Hut,', 1),\n",
       " (u'honeyless.', 1),\n",
       " (u'hill.', 12),\n",
       " (u'hems.', 1),\n",
       " (u'hussars?\"', 2),\n",
       " (u'Hills--even', 1),\n",
       " (u'hysterics.', 1),\n",
       " (u'hussar!\"', 1),\n",
       " (u'hut;', 1),\n",
       " (u'humbug,', 1),\n",
       " (u'hoglike', 1),\n",
       " (u'Hollander--Drink,', 1),\n",
       " (u'holies', 1),\n",
       " (u'hair-breadth', 1),\n",
       " (u'hair?\"', 1),\n",
       " (u'hais', 3),\n",
       " (u'hors', 5),\n",
       " (u'hut.', 12),\n",
       " (u'hair,', 36),\n",
       " (u'has;', 2),\n",
       " (u'hautboys', 1),\n",
       " (u\"hadn't\", 7),\n",
       " (u'Haugwitz', 1),\n",
       " (u'heureux.--Et', 1),\n",
       " (u'huts.', 1),\n",
       " (u'holder,', 2),\n",
       " (u'human,', 4),\n",
       " (u'HAMLET;', 1),\n",
       " (u'hushed', 4),\n",
       " (u'hungry,', 7),\n",
       " (u'hearse,', 1),\n",
       " (u'Hoth.', 1),\n",
       " (u'honor!\"', 9),\n",
       " (u'hustle', 1),\n",
       " (u\"Hutt's\", 1),\n",
       " (u'HAMLET]', 11),\n",
       " (u'hum,', 1),\n",
       " (u'huntsmen', 10),\n",
       " (u'him--who', 1),\n",
       " (u'hips,', 1),\n",
       " (u'humanum', 1),\n",
       " (u'hung', 38),\n",
       " (u'honor...', 1),\n",
       " (u'humeur.', 1),\n",
       " (u'hopelessness', 3),\n",
       " (u\"hold!'\", 1),\n",
       " (u'hectic', 1),\n",
       " (u'Hochgeboren!', 1),\n",
       " (u'hast', 116),\n",
       " (u'heathen,', 1),\n",
       " (u\"Hoffman's\", 1),\n",
       " (u'hoops', 1),\n",
       " (u'harsh;', 2),\n",
       " (u'heroic', 15),\n",
       " (u'HENDERSON', 1),\n",
       " (u\"hair's\", 5),\n",
       " (u'haleines', 1),\n",
       " (u'hesitant', 1),\n",
       " (u'Hate', 2),\n",
       " (u'hinting', 1),\n",
       " (u'himself)', 9),\n",
       " (u'hurriedly.', 12),\n",
       " (u'Hors', 5),\n",
       " (u\"hack'd.\", 1),\n",
       " (u'her_', 1),\n",
       " (u'howbeit', 1),\n",
       " (u'hospital;', 1),\n",
       " (u'happiness,\\'\"', 1),\n",
       " (u'homeward', 1),\n",
       " (u'Huge', 2),\n",
       " (u'Helena).', 1),\n",
       " (u'honest;--so', 1),\n",
       " (u'hilly.', 1),\n",
       " (u\"honor--that's\", 1),\n",
       " (u'harpoons', 1),\n",
       " (u'historian', 13),\n",
       " (u'hope!', 1),\n",
       " (u'hardworking', 1),\n",
       " (u'ha-ve', 1),\n",
       " (u'HARDBURGER', 1),\n",
       " (u'hearing', 73),\n",
       " (u'hook', 4),\n",
       " (u'horse!', 2),\n",
       " (u'hyperdrive.', 4),\n",
       " (u'habitation', 1),\n",
       " (u'HOLLY', 1),\n",
       " (u'happiness--floated', 1),\n",
       " (u'hood', 7),\n",
       " (u'horse,', 73),\n",
       " (u'hot...', 1),\n",
       " (u'Hendrikhovna,', 4),\n",
       " (u'harmed,', 1),\n",
       " (u'Hang', 16),\n",
       " (u'hopelessly.', 2),\n",
       " (u'hurry...', 1),\n",
       " (u'happened,\"', 5),\n",
       " (u'ho!--are', 1),\n",
       " (u'HUNTER,', 1),\n",
       " (u'hist!', 1),\n",
       " (u'hand--he', 1),\n",
       " (u'home.', 38),\n",
       " (u'hand,\"', 3),\n",
       " (u'happen.\"', 2),\n",
       " (u'heaven,\\'\"', 1),\n",
       " (u'him--\"did', 1),\n",
       " (u'heal', 4),\n",
       " (u\"honour'd,\", 1),\n",
       " (u'hands?', 1),\n",
       " (u\"hear'st\", 4),\n",
       " (u'hopes', 27),\n",
       " (u'hands,', 71),\n",
       " (u'heels--we', 1),\n",
       " (u'honn\\xeate', 10),\n",
       " (u'hears,', 1),\n",
       " (u'HUMAN:', 4),\n",
       " (u\"humanity's\", 4),\n",
       " (u'happiness.\"', 3),\n",
       " (u'honor,\"', 8),\n",
       " (u'hounds.', 4),\n",
       " (u'Hamlet', 23),\n",
       " (u'ha\\xefr.', 1),\n",
       " (u'here,\"', 30),\n",
       " (u'he\"', 3),\n",
       " (u'horses,', 50),\n",
       " (u'head,', 117),\n",
       " (u'Hillo,', 2),\n",
       " (u'Heh!', 2),\n",
       " (u'hesitantly', 1),\n",
       " (u'hatch', 19),\n",
       " (u'hysterical', 1),\n",
       " (u'heels', 11),\n",
       " (u'holiday.', 1),\n",
       " (u'husband;', 3),\n",
       " (u'helpmeet', 1),\n",
       " (u'hussars--was', 1),\n",
       " (u'Hm...!\"', 1),\n",
       " (u'house....', 3),\n",
       " (u'hospitals,', 6),\n",
       " (u'harmony;', 1),\n",
       " (u'handkerchiefs.', 1),\n",
       " (u'himself--I', 1),\n",
       " (u'HORRIGAN,', 1),\n",
       " (u'him--but', 1),\n",
       " (u'haps,', 1),\n",
       " (u'hazel-nut', 1),\n",
       " (u'haphazard', 1),\n",
       " (u'health?\"', 2),\n",
       " (u'health...', 1),\n",
       " (u'historians)', 1),\n",
       " (u'hem!_', 1),\n",
       " (u'honourable.', 2),\n",
       " (u'hungerly,', 1),\n",
       " (u'Highness!\"', 1),\n",
       " (u'handsomer', 2),\n",
       " (u'hers.', 5),\n",
       " (u'Hold;', 1),\n",
       " (u'hinds?', 1),\n",
       " (u'hear?', 8),\n",
       " (u'horrid,', 1),\n",
       " (u'Herr\"', 1),\n",
       " (u'hammers,', 1),\n",
       " (u'HARRIS,', 1),\n",
       " (u'horrible!', 3),\n",
       " (u'hanged--the', 1),\n",
       " (u'hide?', 1),\n",
       " (u'heroic.', 1),\n",
       " (u'hatred,', 4),\n",
       " (u\"her.'\", 1),\n",
       " (u'heir;', 2),\n",
       " (u'hood!', 1),\n",
       " (u'Hutt.', 5),\n",
       " (u'haul', 2),\n",
       " (u'heir.', 2),\n",
       " (u'Hood', 1),\n",
       " (u'heure;', 1),\n",
       " (u'hindrance', 7),\n",
       " (u'habitera', 1),\n",
       " (u'him--those', 1),\n",
       " (u'humane,', 1),\n",
       " (u'Heloise?\"', 1),\n",
       " (u'HUGH', 1),\n",
       " (u'humor.', 2),\n",
       " (u'Hamlets', 1),\n",
       " (u'humili\\xe9,', 1),\n",
       " (u'held', 153),\n",
       " (u'Hill', 6),\n",
       " (u'hot,\"', 2),\n",
       " (u'hand?', 8),\n",
       " (u'hence.', 8),\n",
       " (u'Hospital,', 1),\n",
       " (u'hovers', 3),\n",
       " (u'hums', 1),\n",
       " (u'heels;', 1),\n",
       " (u'Hmm?', 2),\n",
       " (u'heard!\"', 1),\n",
       " (u'hole,', 1),\n",
       " (u\"ha't\", 1),\n",
       " (u'Historians', 1),\n",
       " (u'Hamburg', 1),\n",
       " (u'Hear', 18),\n",
       " (u'hail!', 3),\n",
       " (u'History', 11),\n",
       " (u'happiness--the', 1),\n",
       " (u'hates', 2),\n",
       " (u'hall.', 9),\n",
       " (u'Hamlet?', 4),\n",
       " (u'howl.', 4),\n",
       " (u'harmed', 1),\n",
       " (u'heap:', 1),\n",
       " (u'heart!\"', 2),\n",
       " (u'horrid,\"', 1),\n",
       " (u'hit....\"', 1),\n",
       " (u'Hamlet,', 25),\n",
       " (u'heaven:', 3),\n",
       " (u'Hellespont,', 1),\n",
       " (u'hotshot', 1),\n",
       " (u'hoofs.', 2),\n",
       " (u'Hamlet!', 6),\n",
       " (u'heeded.', 1),\n",
       " (u'HOOTKINS', 1),\n",
       " (u'HIRSH,', 1),\n",
       " (u'husband--God', 1),\n",
       " (u'hill:', 1),\n",
       " (u'HEADQUARTERS', 3),\n",
       " (u'humility;', 1),\n",
       " (u'herald', 5),\n",
       " (u'Held', 1),\n",
       " (u'hate?', 2),\n",
       " (u'heed', 12),\n",
       " (u'harp.', 2),\n",
       " (u'hawkers', 1),\n",
       " (u\"here--don't\", 1),\n",
       " (u'hugging,', 1),\n",
       " (u'hunchback', 1),\n",
       " (u'hounds:', 1),\n",
       " (u'honteuses,', 1),\n",
       " (u'havior,', 1),\n",
       " (u'Hosjeradek', 1),\n",
       " (u'hurried', 28),\n",
       " (u'hives.', 1),\n",
       " (u'harder,', 3),\n",
       " (u'him--whether', 1),\n",
       " (u'here!\"', 10),\n",
       " (u'holds,', 2),\n",
       " (u'horrors,', 1),\n",
       " (u'him--this', 1),\n",
       " (u'hoarfrost', 3),\n",
       " (u'hourly', 2),\n",
       " (u\"harm's\", 1),\n",
       " (u'heavy-headed', 1),\n",
       " (u'HURREN', 1),\n",
       " (u'haine,', 2),\n",
       " (u'health!', 2),\n",
       " (u'heart-inflaming', 1),\n",
       " (u'h\\xe9riss\\xe9es', 1),\n",
       " (u'Hills--and', 1),\n",
       " (u'heavenly', 21),\n",
       " (u'helps,', 1),\n",
       " (u'Happy', 4),\n",
       " (u'HERBERT,', 1),\n",
       " (u'health,', 21),\n",
       " (u'HOLMAN', 1),\n",
       " (u'heaps.', 1),\n",
       " (u'honor)', 1),\n",
       " (u'horreur', 1),\n",
       " (u'half-grown', 1),\n",
       " (u'Hostilities', 1),\n",
       " (u'here', 413),\n",
       " (u'Herald', 2),\n",
       " (u'habitual,', 1),\n",
       " (u'historians', 95),\n",
       " (u'hoarse', 17),\n",
       " (u'Hyperion', 1),\n",
       " (u'hundredweight', 1),\n",
       " (u'Hah.', 1),\n",
       " (u'hurriedly:', 2),\n",
       " (u'hurry?\"--Nicholas', 1),\n",
       " (u'Huttese,', 1),\n",
       " (u'honest', 61),\n",
       " (u'he:--O', 1),\n",
       " (u'him).', 1),\n",
       " (u\"hush'd\", 1),\n",
       " (u'hauls', 1),\n",
       " (u'horrified', 16),\n",
       " (u'horizon', 20),\n",
       " (u'herself.', 41),\n",
       " (u'hairs', 9),\n",
       " (u'happiness....\"', 1),\n",
       " (u'hatred', 18),\n",
       " (u'himself?', 7),\n",
       " (u'Hence,', 5),\n",
       " (u'his:', 3),\n",
       " (u'hint,', 1),\n",
       " (u\"her!'\", 1),\n",
       " (u'Ha-ha-ha!', 1),\n",
       " (u'hunching', 1),\n",
       " (u'Hist!', 1),\n",
       " (u'Hence!', 3),\n",
       " (u'how...', 2),\n",
       " (u'homme!\"', 1),\n",
       " (u'halt,', 1),\n",
       " (u'handkerchief?', 5),\n",
       " (u'had--if', 1),\n",
       " (u'highness?', 1),\n",
       " (u'hypothetical', 1),\n",
       " (u'him--dead--', 1),\n",
       " (u'hardtack', 1),\n",
       " (u'hit!\"', 1),\n",
       " (u\"hussars'\", 1),\n",
       " (u\"him.'\", 1),\n",
       " (u'hesitations,', 1),\n",
       " (u'hideous?\"', 1),\n",
       " (u'horseman', 3),\n",
       " (u'hard,', 15),\n",
       " (u'husband--who,', 1),\n",
       " (u'Help!', 5),\n",
       " (u'HANCOCK', 1),\n",
       " (u'hereditary', 1),\n",
       " (u'Honest', 5),\n",
       " (u'happiest,', 1),\n",
       " (u'hoist', 2),\n",
       " (u'ho', 1),\n",
       " (u'horn!\"', 1),\n",
       " (u'Hunts', 1),\n",
       " (u'helmet-encased', 1),\n",
       " (u'harnessing', 1),\n",
       " (u'handsome', 90),\n",
       " (u'handkerchiefs', 3),\n",
       " (u'haute_.--Je', 1),\n",
       " (u'Hollabrunn,', 1),\n",
       " (u'half-closed,', 2),\n",
       " (u'highroad.', 4),\n",
       " (u'hothouse;', 1),\n",
       " (u'Helena', 1),\n",
       " (u'heiresses', 4),\n",
       " (u'heard)', 1),\n",
       " (u'Husband?', 1),\n",
       " (u'house!\"', 2),\n",
       " (u'hero...', 1),\n",
       " (u'holiday:', 1),\n",
       " (u'hunters,', 1),\n",
       " (u'harmonies,', 1),\n",
       " (u'happily,', 2),\n",
       " (u'horrors', 10),\n",
       " (u'hanggliders', 1),\n",
       " (u'happens.', 3),\n",
       " (u'hearts.', 5),\n",
       " (u'hours.', 6),\n",
       " (u'husband:', 4),\n",
       " (u'her.)', 1),\n",
       " (u'herder!', 1),\n",
       " (u'him--as', 2),\n",
       " (u'hip,', 1),\n",
       " (u'heaven,\"', 2),\n",
       " (u'hardness,', 1),\n",
       " (u'Heaven!', 1),\n",
       " (u'hast:', 1),\n",
       " (u'honourable:', 1),\n",
       " (u'h\\xe9siter', 1),\n",
       " (u'him?...', 1),\n",
       " (u'heart:', 11),\n",
       " (u'how!', 1),\n",
       " (u'He\"--he', 1),\n",
       " (u'husbands;', 1),\n",
       " (u'happens', 43),\n",
       " (u'him--no', 1),\n",
       " (u'hell,', 16),\n",
       " (u'harvest.', 1),\n",
       " (u'Hello.', 2),\n",
       " (u'headquarters,', 3),\n",
       " (u'halter', 1),\n",
       " (u'him?', 50),\n",
       " (u'here,', 103),\n",
       " (u'however.', 1),\n",
       " (u'heart)', 1),\n",
       " (u'hell,\"', 1),\n",
       " (u'hit;', 1),\n",
       " (u'HOLD', 12),\n",
       " (u'headdress', 2),\n",
       " (u'hussar', 61),\n",
       " (u\"hold'st\", 1),\n",
       " (u'happened...\"', 1),\n",
       " (u'house...\"', 1),\n",
       " (u'helplessly', 6),\n",
       " (u'house-affairs', 1),\n",
       " (u'houseful', 1),\n",
       " (u'Hippolyte!\"', 1),\n",
       " (u'historic', 34),\n",
       " (u'haired', 1),\n",
       " (u'husky', 2),\n",
       " (u'hospital', 18),\n",
       " (u'http://pglaf.org/donate', 2),\n",
       " (u'heaven!...', 2),\n",
       " (u'Hide', 2),\n",
       " (u'help', 187),\n",
       " (u'huge,', 8),\n",
       " (u'heart--watching', 1),\n",
       " (u'households.', 1),\n",
       " (u'hath', 297),\n",
       " (u'him--yet', 1),\n",
       " (u'hits.', 1),\n",
       " (u'hoodwink.', 1),\n",
       " (u'hustles', 1),\n",
       " (u'haven.', 1),\n",
       " (u'half-witted', 1),\n",
       " (u'Her', 129),\n",
       " (u'hindquarters,', 1),\n",
       " (u'haste?', 2),\n",
       " (u'her,\"', 12),\n",
       " (u'hall', 36),\n",
       " (u'HAN:', 278),\n",
       " (u'Hers', 1),\n",
       " (u'hammers', 1),\n",
       " (u'has', 880),\n",
       " (u'his', 9665),\n",
       " (u'heaven.', 8),\n",
       " (u'Henri', 1),\n",
       " (u'horologe', 1),\n",
       " (u'him--though', 1),\n",
       " (u'heaven;', 7),\n",
       " (u'himself--not', 1),\n",
       " (u'hook,', 1),\n",
       " (u'happiest', 5),\n",
       " (u'him', 3434),\n",
       " (u'Help', 16),\n",
       " (u'hours!', 1),\n",
       " (u'had!\"', 1),\n",
       " (u'helped!', 1),\n",
       " (u'hereafter', 3),\n",
       " (u'him--would', 1),\n",
       " (u'healthsome', 1),\n",
       " (u'hill;', 2),\n",
       " (u'hive', 13),\n",
       " (u\"ha'\", 6),\n",
       " (u'Hath', 61),\n",
       " (u'hoar,', 2),\n",
       " (u'hosts', 1),\n",
       " (u\"hoof's\", 1),\n",
       " (u'Here!\"', 1),\n",
       " (u'hill,', 22),\n",
       " (u'hologram)', 1),\n",
       " (u'hems,', 1),\n",
       " (u'HAROLD', 1),\n",
       " (u'Harsh', 1),\n",
       " (u'house', 260),\n",
       " (u'hare', 18),\n",
       " (u'hurriedly', 52),\n",
       " (u'hair.', 23),\n",
       " (u'hour;', 1),\n",
       " (u'helpless', 14),\n",
       " (u'horns,', 1),\n",
       " (u'helplessness', 1),\n",
       " (u'holder.', 2),\n",
       " (u\"Hyperion's\", 1),\n",
       " (u'hie,', 2),\n",
       " (u'hussar.', 7),\n",
       " (u'hilt', 3),\n",
       " (u'hallways.', 1),\n",
       " (u'handwriting.', 2),\n",
       " (u'heartily,', 1),\n",
       " (u'hollering', 2),\n",
       " (u'honors.', 1),\n",
       " (u'hoped,', 1),\n",
       " (u'him\"', 4),\n",
       " (u'Highnessness?', 1),\n",
       " (u\"Here's\", 25),\n",
       " (u'half-contemptuous', 1),\n",
       " (u'horrid.\"', 1),\n",
       " (u'honor;', 1),\n",
       " (u'hundredth', 3),\n",
       " (u'humours!', 1),\n",
       " (u'Hecuba', 1),\n",
       " (u'herd', 13),\n",
       " (u\"hunter's\", 5),\n",
       " (u'healing', 2),\n",
       " (u'Here,', 30),\n",
       " (u'Hey?\"', 1),\n",
       " (u'hindered,', 1),\n",
       " (u\"huntsmen's\", 3),\n",
       " (u'howwible,', 1),\n",
       " (u'he...', 2),\n",
       " (u'his!', 1),\n",
       " (u'hunger-stricken,', 1),\n",
       " (u'huskily,', 1),\n",
       " (u'Horatio!', 5),\n",
       " (u'hearers--who', 1),\n",
       " (u'hurriedly,', 7),\n",
       " (u'Hecuba,', 1),\n",
       " (u'her]', 2),\n",
       " (u\"hand's\", 2),\n",
       " (u'herself!', 4),\n",
       " (u'has--men', 1),\n",
       " (u'Happiness', 1),\n",
       " (u'his;', 1),\n",
       " (u'HERRON', 1),\n",
       " (u'humour', 8),\n",
       " (u'honn\\xeate;', 1),\n",
       " (u'Halt,', 1),\n",
       " (u'have?', 6),\n",
       " (u'himself.\"', 4),\n",
       " (u'hooks,', 1),\n",
       " (u'hollow', 27),\n",
       " (u'hardy', 4),\n",
       " (u'holster', 3),\n",
       " (u'Hooting', 1),\n",
       " (u'human--death--was', 1),\n",
       " (u'honor...\"', 1),\n",
       " (u'Hurrah!\"', 4),\n",
       " (u'harm?', 3),\n",
       " (u'housing', 2),\n",
       " (u'hearing;', 1),\n",
       " (u'hazards', 1),\n",
       " (u'haggard', 1),\n",
       " (u\"hatch'd\", 1),\n",
       " (u'holidays.', 2),\n",
       " (u'Hoist', 1),\n",
       " (u'honor.\\'\"', 1),\n",
       " (u'historian,', 3),\n",
       " (u'handkerchief)', 1),\n",
       " (u'head!\"', 1),\n",
       " (u'home,', 41),\n",
       " (u'hesitates', 1),\n",
       " ...]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = textFile.flatMap(lambda line: line.split())\n",
    "words_subset = words.filter(lambda x: x[0] in ['H','h'])\n",
    "counts = words_subset.map(lambda w: (w,1)).reduceByKey(lambda a,b: a+b)\n",
    "counts.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleanup using a regular expression\n",
    "\n",
    "* remove all non-alphanumeric characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Han 430\n",
      "Hugh 1\n",
      "Howlings 1\n",
      "Helicopter 1\n",
      "Hyrcan 1\n",
      "Heh 2\n",
      "Hoists 1\n",
      "Hoffman 1\n",
      "HUT 1\n",
      "Hmm 6\n",
      "Harness 1\n",
      "Hie 10\n",
      "HARRELL 1\n",
      "Highnesses 1\n",
      "Hurry 11\n",
      "Hiding 2\n",
      "Hecate 4\n",
      "HOLT 2\n",
      "Hussars 3\n",
      "Hold 42\n",
      "Hofmarschal 1\n",
      "Herein 2\n",
      "Hollow 1\n",
      "Hercules 7\n",
      "Hyrcanian 1\n",
      "Hanging 3\n",
      "He 2497\n",
      "Halfway 4\n",
      "Hop 1\n",
      "Hadn 1\n",
      "Heights 8\n",
      "HAVE 2\n",
      "HATCH 1\n",
      "HANKS 1\n",
      "Hurrah 47\n",
      "Huttese 21\n",
      "Hermit 1\n",
      "Honour 1\n",
      "Heat 2\n",
      "Hi 4\n",
      "Herself 3\n",
      "HARTLEY 1\n",
      "HEAD 1\n",
      "Hebrew 1\n",
      "HOSKER 1\n",
      "Honestly 1\n",
      "Historical 1\n",
      "Hosanna 1\n",
      "Honor 6\n",
      "House 3\n",
      "Hollandais 2\n",
      "Hush 3\n",
      "Hectors 1\n",
      "Harpier 1\n",
      "Hack 1\n",
      "Huge 2\n",
      "HODENFIELD 1\n",
      "Hover 1\n",
      "Hmph 1\n",
      "Hecatommythi 1\n",
      "Hadst 4\n",
      "Hang 17\n",
      "HARDBURGER 1\n",
      "Hamlet 86\n",
      "Hors 5\n",
      "Haugwitz 1\n",
      "HOLLY 1\n",
      "Half 16\n",
      "HOME 1\n",
      "HOLE 1\n",
      "HENDERSON 1\n",
      "Hate 2\n",
      "Hood 1\n",
      "Higher 4\n",
      "Hosjeradek 2\n",
      "Havoc 1\n",
      "Hamlets 1\n",
      "Hydra 1\n",
      "HYPERSPACE 2\n",
      "Happy 5\n",
      "Hohenlohe 1\n",
      "Hunts 1\n",
      "Held 1\n",
      "HUGH 1\n",
      "HURREN 1\n",
      "Host 1\n",
      "Hé 1\n",
      "Hear 20\n",
      "Ho 15\n",
      "Historians 1\n",
      "Hamburg 2\n",
      "HOOTKINS 1\n",
      "HOLMAN 1\n",
      "HANCOCK 1\n",
      "Hostilities 1\n",
      "Honest 8\n",
      "Herald 2\n",
      "HOLD 12\n",
      "Hyperion 2\n",
      "History 12\n",
      "HEADQUARTERS 3\n",
      "Helena 4\n",
      "Hill 7\n",
      "HERRON 1\n",
      "HAROLD 1\n",
      "Hide 2\n",
      "Harsh 1\n",
      "Her 132\n",
      "HERMAN 2\n",
      "Hooting 1\n",
      "Hélas 20\n",
      "Hut 3\n",
      "Hoist 1\n",
      "HISS 1\n",
      "Hers 1\n",
      "HOMESTEAD 9\n",
      "Humbug 2\n",
      "Henri 1\n",
      "H 7\n",
      "Hecuba 4\n",
      "Heart 6\n",
      "Help 36\n",
      "HOBBIE 1\n",
      "High 9\n",
      "Hath 62\n",
      "Hussar 2\n",
      "Happiness 1\n",
      "Hunting 1\n",
      "Hillo 2\n",
      "Hand 5\n",
      "HOWARTH 1\n",
      "HENSON 1\n",
      "HUGE 1\n",
      "Hymen 1\n",
      "Holy 19\n",
      "Happily 3\n",
      "HAMILL 3\n",
      "Husbands 1\n",
      "Hm 12\n",
      "Herr 3\n",
      "Haven 6\n",
      "Hardenburg 1\n",
      "Helen 3\n",
      "Heath 1\n",
      "Hollander 2\n",
      "HALF 2\n",
      "Hence 14\n",
      "HUSE 1\n",
      "Hofkriegsrath 7\n",
      "Hungarian 5\n",
      "Hard 5\n",
      "HARLEY 1\n",
      "Habitants 1\n",
      "Human 2\n",
      "HELEN 1\n",
      "Hairdressers 1\n",
      "Holla 2\n",
      "Hallo 2\n",
      "HARGREAVES 1\n",
      "Hated 1\n",
      "HAGON 1\n",
      "Had 64\n",
      "HERBERT 3\n",
      "Henceforth 2\n",
      "Hidden 1\n",
      "HUTTESE 1\n",
      "Hoth 11\n",
      "HALLWAY 13\n",
      "Hairdresser 1\n",
      "HUM 1\n",
      "HILL 2\n",
      "HUMAN 4\n",
      "Hast 12\n",
      "HUME 2\n",
      "HAN 463\n",
      "HUSTON 2\n",
      "HEAR 2\n",
      "Hung 1\n",
      "Husband 3\n",
      "HANGAR 21\n",
      "Hérode 1\n",
      "Hetzelsdorf 1\n",
      "Hyperdrive 1\n",
      "Holà 4\n",
      "Hah 1\n",
      "Heads 1\n",
      "HATT 1\n",
      "Hollabrunn 5\n",
      "Hist 1\n",
      "Horrible 2\n",
      "Herder 1\n",
      "HANSON 1\n",
      "Herculean 2\n",
      "Henceforward 2\n",
      "Highnessness 1\n",
      "However 35\n",
      "Here 191\n",
      "HORATIO 128\n",
      "Highness 56\n",
      "HÉRAUT 3\n",
      "Hokey 1\n",
      "Henry 1\n",
      "Hastily 1\n",
      "Heavy 2\n",
      "Highly 1\n",
      "Heard 1\n",
      "Helene 165\n",
      "Have 153\n",
      "Heavens 4\n",
      "Heloise 2\n",
      "HOLOGRAM 1\n",
      "Hmmm 2\n",
      "Horribly 2\n",
      "HEIDEL 1\n",
      "Has 19\n",
      "Himself 5\n",
      "Hutt 14\n",
      "Hippolyte 51\n",
      "HIRSCH 1\n",
      "Horizontal 1\n",
      "Harvest 1\n",
      "Hall 3\n",
      "HANGER 1\n",
      "Hobbie 2\n",
      "Huttesse 1\n",
      "Haply 5\n",
      "His 429\n",
      "Humbly 2\n",
      "Hoping 2\n",
      "Handkerchief 1\n",
      "Hayne 1\n",
      "Hybla 1\n",
      "Him 26\n",
      "Home 6\n",
      "Hart 4\n",
      "HOTH 60\n",
      "Hendrikhovna 21\n",
      "Highest 1\n",
      "Hit 2\n",
      "Heave 1\n",
      "HECATE 7\n",
      "Hotel 2\n",
      "Hazard 1\n",
      "Hearing 8\n",
      "Hours 1\n",
      "How 532\n",
      "Hope 1\n",
      "Hospital 3\n",
      "HARRY 1\n",
      "Ha 39\n",
      "Hardly 13\n",
      "HIRSH 1\n",
      "HAPPINESS 1\n",
      "Hipparchus 1\n",
      "Horses 1\n",
      "HERALD 1\n",
      "Hasn 4\n",
      "Hur 2\n",
      "Hero 1\n",
      "HONE 1\n",
      "Hum 5\n",
      "HARTNEY 1\n",
      "Horatio 31\n",
      "Honneur 3\n",
      "Heah 1\n",
      "Headquarters 4\n",
      "Honey 1\n",
      "Hot 2\n",
      "Hofs 2\n",
      "HOLDING 2\n",
      "Howe 2\n",
      "Hessian 4\n",
      "Henker 1\n",
      "HORRIGAN 1\n",
      "Halt 3\n",
      "Horse 21\n",
      "HEMLEY 1\n",
      "Holds 4\n",
      "Hey 38\n",
      "HYDE 1\n",
      "Holding 7\n",
      "Historic 1\n",
      "Hills 73\n",
      "Head 3\n",
      "Hullo 3\n",
      "Having 120\n",
      "Hic 1\n",
      "HAGANS 1\n",
      "HANDWERK 1\n",
      "Harder 1\n",
      "Hautboys 4\n",
      "Hessians 2\n",
      "Hearts 1\n",
      "Hacking 1\n",
      "HERD 1\n",
      "HALL 2\n",
      "HAMLET 388\n",
      "Hark 28\n",
      "Hirtius 1\n",
      "HARMAN 1\n",
      "Hoch 2\n",
      "Hellespont 2\n",
      "Hell 2\n",
      "HOPE 1\n",
      "Herod 6\n",
      "Heaven 26\n",
      "Health 3\n",
      "Honnête 4\n",
      "HOUSE 4\n",
      "HOWARD 4\n",
      "HUNTER 1\n",
      "Hail 12\n",
      "Hundreds 5\n",
      "Houses 1\n",
      "HARRIS 1\n",
      "Haste 2\n",
      "Hochgeboren 1\n",
      "HARRISON 3\n",
      "Hands 1\n",
      "HUTT 1\n",
      "Hid 1\n",
      "HATTRICK 1\n",
      "Hello 8\n",
      "Handsome 1\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# Compile a regular expression that matches non-alphanumerics\n",
    "pattern = re.compile('[\\W_]+', re.UNICODE)\n",
    "\n",
    "# Replace all non-alphanumerics with a space, then split into words\n",
    "words = textFile.map(lambda line: pattern.sub(' ',line)).flatMap(lambda line: line.split())\n",
    "\n",
    "#first_letters = set(['H','h','Q','q','s','S'])\n",
    "first_letters = set(['H'])\n",
    "# Count words that start with H\n",
    "words_subset = words.filter(lambda x: x[0] in first_letters)\n",
    "counts = words_subset.map(lambda w: (w,1)).reduceByKey(lambda a,b: a+b)\n",
    "res = counts.collect()\n",
    "for r,c in res:\n",
    "    print r,c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Counting Letter bigrams\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "#textFile = sc.textFile(\"notes/data/books-eng/hamlet.txt\")\n",
    "textFile = sc.textFile(\"data/books-eng\")\n",
    "\n",
    "# Compile a regular expression that matches non-alphanumerics\n",
    "pattern = re.compile(u'[\\W0-9_]+', re.UNICODE)\n",
    "\n",
    "# Replace all non-alphanumerics with a space, then split into words\n",
    "words = textFile.map(lambda line: pattern.sub(' ',line)).flatMap(lambda line: line.split())\n",
    "# Convert to lower case\n",
    "words = words.map(lambda w: w.lower())\n",
    "\n",
    "# Convert to a list of letters list('abc') = ['a','b','c']\n",
    "letters = words.flatMap(lambda word: [pair[0]+pair[1] for pair in zip(list('_'+word),list(word+'_')) ] )\n",
    "counts = letters.map(lambda w: (w,1)).reduceByKey(lambda a,b: a+b)\n",
    "bigrams = counts.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "e_ 172904\n",
      "_t 128820\n",
      "th 106824\n",
      "s_ 102536\n",
      "he 101810\n",
      "d_ 99436\n",
      "_a 97104\n",
      "t_ 92419\n",
      "_s 70618\n",
      "_h 67740\n",
      "in 66176\n",
      "an 64789\n",
      "n_ 63768\n",
      "er 63475\n",
      "_w 56231\n",
      "r_ 53641\n",
      "re 51571\n",
      "_i 49690\n",
      "_o 48646\n",
      "y_ 47589\n",
      "nd 47353\n",
      "o_ 43783\n",
      "on 40815\n",
      "ou 40242\n",
      "en 40018\n",
      "at 39559\n",
      "ha 39329\n",
      "_b 38703\n",
      "_m 36304\n",
      "_c 35617\n",
      "hi 35323\n",
      "to 34885\n",
      "ed 33906\n",
      "is 33101\n",
      "_f 32939\n",
      "ng 32927\n",
      "es 32406\n",
      "_d 31593\n",
      "it 31263\n",
      "or 31073\n",
      "ar 29993\n",
      "g_ 28676\n",
      "_p 28551\n",
      "_l 28257\n",
      "as 28217\n",
      "st 28095\n",
      "te 27913\n",
      "le 27804\n",
      "f_ 26620\n",
      "se 26332\n",
      "nt 25081\n",
      "a_ 24600\n",
      "me 24069\n",
      "h_ 23940\n",
      "l_ 23436\n",
      "_n 23129\n",
      "ve 22985\n",
      "de 22728\n",
      "of 22601\n",
      "ll 22433\n",
      "ro 22372\n",
      "_r 21788\n",
      "ea 21727\n",
      "ne 21470\n",
      "al 21141\n",
      "_e 20797\n",
      "ti 19930\n",
      "ho 19908\n",
      "no 19710\n",
      "co 19261\n",
      "ri 19213\n",
      "ce 18764\n",
      "m_ 18050\n",
      "be 17483\n",
      "om 16762\n",
      "el 16694\n",
      "wh 16330\n",
      "ut 16321\n",
      "ch 16294\n",
      "_g 16157\n",
      "us 15636\n",
      "ot 15593\n",
      "ur 15501\n",
      "ma 15336\n",
      "wi 15255\n",
      "wa 15228\n",
      "sh 15094\n",
      "ad 14907\n",
      "ow 14861\n",
      "lo 14690\n",
      "li 14339\n",
      "et 14115\n",
      "ra 13995\n",
      "si 13900\n",
      "ee 13712\n",
      "ss 13601\n",
      "ai 13570\n",
      "il 13492\n",
      "so 13230\n",
      "ta 13222\n",
      "ie 13145\n",
      "_y 13101\n",
      "un 13033\n",
      "i_ 13028\n",
      "la 12942\n",
      "im 12930\n",
      "pe 12391\n",
      "fo 12378\n",
      "ly 12137\n",
      "rs 12107\n",
      "io 11988\n",
      "nc 11793\n",
      "w_ 11660\n",
      "ic 11593\n",
      "oo 11470\n",
      "yo 11447\n",
      "we 11411\n",
      "ca 11378\n",
      "sa 11265\n",
      "id 11237\n",
      "di 11228\n",
      "mo 11008\n",
      "ac 10779\n",
      "ol 10724\n",
      "pr 10553\n",
      "tr 10312\n",
      "em 10273\n",
      "ke 10134\n",
      "u_ 10083\n",
      "ir 9817\n",
      "ns 9760\n",
      "ld 9748\n",
      "do 9523\n",
      "os 9350\n",
      "ge 9282\n",
      "ul 9189\n",
      "gh 9145\n",
      "ov 8958\n",
      "_u 8930\n",
      "k_ 8838\n",
      "rt 8561\n",
      "ni 8529\n",
      "ay 8467\n",
      "pa 7959\n",
      "_v 7951\n",
      "mi 7838\n",
      "av 7789\n",
      "na 7767\n",
      "po 7746\n",
      "ec 7694\n",
      "fr 7595\n",
      "bu 7533\n",
      "am 7432\n",
      "ig 7412\n",
      "ry 7177\n",
      "rd 7164\n",
      "fi 7113\n",
      "ev 7073\n",
      "wo 7060\n",
      "vi 6722\n",
      "fe 6566\n",
      "ht 6535\n",
      "pi 6432\n",
      "fa 6344\n",
      "rr 6329\n",
      "tt 6256\n",
      "ts 6236\n",
      "go 6227\n",
      "su 6061\n",
      "pl 6040\n",
      "bo 5927\n",
      "_k 5921\n",
      "ag 5852\n",
      "ct 5845\n",
      "tu 5807\n",
      "ap 5804\n",
      "ei 5786\n",
      "ia 5640\n",
      "bl 5628\n",
      "ba 5541\n",
      "qu 5410\n",
      "ey 5342\n",
      "p_ 5293\n",
      "if 5289\n",
      "ep 5281\n",
      "my 5273\n",
      "sp 5241\n",
      "od 5183\n",
      "mp 5121\n",
      "iv 5116\n",
      "ru 5010\n",
      "ab 4893\n",
      "ck 4831\n",
      "ki 4821\n",
      "ex 4740\n",
      "op 4711\n",
      "dr 4704\n",
      "ug 4661\n",
      "ga 4569\n",
      "ak 4568\n",
      "ok 4548\n",
      "gr 4534\n",
      "_j 4451\n",
      "da 4419\n",
      "ue 4408\n",
      "oi 4406\n",
      "ew 4392\n",
      "au 4333\n",
      "up 4295\n",
      "ef 4275\n",
      "ty 4261\n",
      "tl 4168\n",
      "ff 4152\n",
      "rm 4146\n",
      "lu 4130\n",
      "ci 4011\n",
      "ye 4004\n",
      "ds 3957\n",
      "rn 3741\n",
      "_q 3715\n",
      "by 3654\n",
      "gi 3639\n",
      "br 3626\n",
      "pp 3586\n",
      "cr 3541\n",
      "ui 3523\n",
      "sc 3402\n",
      "uc 3400\n",
      "cl 3346\n",
      "vo 3309\n",
      "va 3202\n",
      "nn 3198\n",
      "ny 3176\n",
      "wn 3172\n",
      "v_ 3056\n",
      "aw 3000\n",
      "kn 2998\n",
      "lf 2978\n",
      "eg 2764\n",
      "ft 2736\n",
      "mu 2731\n",
      "oc 2666\n",
      "fu 2637\n",
      "hr 2631\n",
      "ls 2618\n",
      "hu 2610\n",
      "ms 2514\n",
      "cu 2506\n",
      "nk 2493\n",
      "nl 2473\n",
      "sk 2469\n",
      "eo 2418\n",
      "lt 2408\n",
      "rc 2387\n",
      "gu 2374\n",
      "um 2363\n",
      "hy 2356\n",
      "ud 2344\n",
      "mm 2336\n",
      "pt 2301\n",
      "pu 2250\n",
      "af 2208\n",
      "sm 2140\n",
      "rg 2128\n",
      "du 2102\n",
      "fl 2038\n",
      "bi 2034\n",
      "gl 1998\n",
      "ik 1975\n",
      "rk 1949\n",
      "ys 1932\n",
      "eu 1926\n",
      "ua 1923\n",
      "sl 1882\n",
      "dy 1870\n",
      "ju 1869\n",
      "mb 1859\n",
      "oa 1839\n",
      "ps 1828\n",
      "tw 1815\n",
      "uk 1815\n",
      "ob 1804\n",
      "rl 1797\n",
      "je 1785\n",
      "ks 1727\n",
      "ib 1720\n",
      "ip 1676\n",
      "dd 1651\n",
      "xp 1610\n",
      "iu 1510\n",
      "dl 1510\n",
      "c_ 1490\n",
      "nu 1486\n",
      "gn 1484\n",
      "ya 1471\n",
      "gs 1439\n",
      "cc 1431\n",
      "sw 1410\n",
      "lk 1376\n",
      "nf 1360\n",
      "yi 1355\n",
      "ub 1351\n",
      "oy 1257\n",
      "ws 1223\n",
      "tc 1219\n",
      "rv 1203\n",
      "og 1103\n",
      "jo 1099\n",
      "ze 1080\n",
      "nv 1052\n",
      "xt 1027\n",
      "x_ 1022\n",
      "eh 994\n",
      "oe 983\n",
      "ae 922\n",
      "z_ 862\n",
      "yt 861\n",
      "ez 856\n",
      "wr 829\n",
      "iz 823\n",
      "ph 822\n",
      "rf 820\n",
      "kh 818\n",
      "ja 811\n",
      "rp 809\n",
      "xc 780\n",
      "ml 772\n",
      "dg 770\n",
      "gg 759\n",
      "bs 744\n",
      "xi 718\n",
      "à_ 714\n",
      "zo 701\n",
      "ku 695\n",
      "eb 691\n",
      "ka 679\n",
      "lm 657\n",
      "sn 655\n",
      "uz 652\n",
      "lv 636\n",
      "uf 632\n",
      "ko 632\n",
      "rb 628\n",
      "é_ 626\n",
      "oh 623\n",
      "sd 620\n",
      "xe 606\n",
      "lp 593\n",
      "sy 584\n",
      "_à 578\n",
      "lw 553\n",
      "dé 545\n",
      "_é 537\n",
      "vn 532\n",
      "lr 529\n",
      "lc 526\n",
      "ah 519\n",
      "sb 514\n",
      "kl 499\n",
      "az 497\n",
      "ém 481\n",
      "py 481\n",
      "bb 471\n",
      "cy 467\n",
      "dn 451\n",
      "ix 449\n",
      "hm 445\n",
      "xa 440\n",
      "ek 439\n",
      "_x 437\n",
      "wd 433\n",
      "b_ 423\n",
      "dm 419\n",
      "ii 418\n",
      "dv 406\n",
      "nq 405\n",
      "ux 397\n",
      "wl 397\n",
      "rw 393\n",
      "eq 390\n",
      "sf 377\n",
      "uo 368\n",
      "uv 362\n",
      "tm 361\n",
      "bt 359\n",
      "mn 350\n",
      "kp 317\n",
      "hs 315\n",
      "rh 314\n",
      "té 306\n",
      "nb 305\n",
      "sq 303\n",
      "zi 296\n",
      "cb 290\n",
      "ré 283\n",
      "dj 282\n",
      "tf 275\n",
      "ée 269\n",
      "tn 262\n",
      "vs 261\n",
      "yb 246\n",
      "ér 242\n",
      "_z 240\n",
      "gt 238\n",
      "êt 238\n",
      "vr 235\n",
      "ym 234\n",
      "oj 228\n",
      "ét 225\n",
      "yl 224\n",
      "tz 222\n",
      "hn 220\n",
      "yp 218\n",
      "vl 215\n",
      "bj 214\n",
      "aj 214\n",
      "nj 208\n",
      "ox 208\n",
      "nm 207\n",
      "ky 205\n",
      "sé 202\n",
      "nh 202\n",
      "yr 201\n",
      "vy 201\n",
      "gy 198\n",
      "hl 197\n",
      "fs 196\n",
      "mf 189\n",
      "j_ 189\n",
      "za 187\n",
      "èr 178\n",
      "ép 175\n",
      "yc 174\n",
      "mt 171\n",
      "xx 169\n",
      "oz 169\n",
      "né 168\n",
      "éc 161\n",
      "nw 158\n",
      "és 158\n",
      "zh 154\n",
      "np 152\n",
      "cq 146\n",
      "yw 145\n",
      "ln 144\n",
      "cs 136\n",
      "zu 132\n",
      "_ê 130\n",
      "én 130\n",
      "pc 127\n",
      "iq 125\n",
      "fy 125\n",
      "cd 124\n",
      "xv 121\n",
      "là 118\n",
      "nr 115\n",
      "lq 114\n",
      "ax 113\n",
      "vé 112\n",
      "mê 110\n",
      "yf 110\n",
      "lg 109\n",
      "kw 108\n",
      "êm 108\n",
      "lé 107\n",
      "df 106\n",
      "él 105\n",
      "pf 103\n",
      "ej 102\n",
      "zz 102\n",
      "lb 102\n",
      "gm 101\n",
      "kr 101\n",
      "dk 100\n",
      "wb 97\n",
      "ès 97\n",
      "hk 97\n",
      "hf 96\n",
      "mé 96\n",
      "pé 93\n",
      "nx 92\n",
      "gé 91\n",
      "yn 90\n",
      "aî 90\n",
      "mr 90\n",
      "rè 87\n",
      "dw 85\n",
      "ît 85\n",
      "xh 85\n",
      "iè 85\n",
      "uh 83\n",
      "rz 82\n",
      "tp 82\n",
      "sg 80\n",
      "sv 79\n",
      "vu 77\n",
      "hé 76\n",
      "bm 74\n",
      "hd 72\n",
      "hb 71\n",
      "hc 71\n",
      "wk 69\n",
      "ié 67\n",
      "gd 66\n",
      "où 65\n",
      "ù_ 65\n",
      "zm 63\n",
      "wf 63\n",
      "âm 62\n",
      "yu 61\n",
      "sr 61\n",
      "cé 61\n",
      "hh 59\n",
      "év 59\n",
      "mc 59\n",
      "rq 59\n",
      "zl 59\n",
      "dh 57\n",
      "_â 55\n",
      "dt 54\n",
      "ég 54\n",
      "dq 53\n",
      "bv 52\n",
      "ao 52\n",
      "oq 52\n",
      "éd 51\n",
      "uy 51\n",
      "wy 50\n",
      "pm 49\n",
      "ût 48\n",
      "aa 47\n",
      "hw 47\n",
      "rê 47\n",
      "uj 47\n",
      "kc 47\n",
      "ôt 45\n",
      "pè 45\n",
      "cè 43\n",
      "ué 42\n",
      "lh 41\n",
      "cx 40\n",
      "yk 40\n",
      "âc 39\n",
      "lè 38\n",
      "nz 37\n",
      "ço 37\n",
      "iw 36\n",
      "ww 35\n",
      "zy 35\n",
      "nê 34\n",
      "sû 34\n",
      "ûr 33\n",
      "pw 32\n",
      "èn 32\n",
      "dè 32\n",
      "kb 32\n",
      "km 32\n",
      "èl 31\n",
      "yd 30\n",
      "wu 30\n",
      "lx 30\n",
      "xy 29\n",
      "fw 29\n",
      "ji 28\n",
      "bd 28\n",
      "pk 27\n",
      "dc 27\n",
      "kg 27\n",
      "zd 27\n",
      "ât 27\n",
      "rj 27\n",
      "lz 27\n",
      "fb 27\n",
      "pç 26\n",
      "tô 25\n",
      "râ 25\n",
      "aq 25\n",
      "tb 25\n",
      "bh 25\n",
      "yh 24\n",
      "dp 23\n",
      "éa 23\n",
      "èv 23\n",
      "xl 22\n",
      "éf 22\n",
      "bé 21\n",
      "éj 21\n",
      "hè 21\n",
      "wt 20\n",
      "kf 20\n",
      "_ô 20\n",
      "bw 20\n",
      "pg 20\n",
      "xu 20\n",
      "mè 20\n",
      "oû 19\n",
      "tê 19\n",
      "wc 18\n",
      "db 18\n",
      "gw 18\n",
      "jà 18\n",
      "rô 18\n",
      "tè 17\n",
      "tv 17\n",
      "èm 17\n",
      "eû 17\n",
      "pê 17\n",
      "çu 17\n",
      "zn 17\n",
      "xo 17\n",
      "fâ 17\n",
      "ôl 16\n",
      "hâ 16\n",
      "îl 16\n",
      "eç 16\n",
      "_î 16\n",
      "ô_ 15\n",
      "fé 15\n",
      "pb 14\n",
      "èc 14\n",
      "èt 13\n",
      "tk 13\n",
      "yz 13\n",
      "êc 13\n",
      "bn 13\n",
      "yé 13\n",
      "gè 12\n",
      "tg 12\n",
      "mw 12\n",
      "kv 12\n",
      "în 12\n",
      "éb 11\n",
      "yg 10\n",
      "pn 10\n",
      "éq 10\n",
      "aï 10\n",
      "lâ 10\n",
      "cô 10\n",
      "vè 9\n",
      "éi 9\n",
      "èd 9\n",
      "td 9\n",
      "wm 9\n",
      "ih 9\n",
      "cv 9\n",
      "rç 9\n",
      "vk 8\n",
      "ët 8\n",
      "oë 8\n",
      "hp 7\n",
      "fû 7\n",
      "êl 7\n",
      "fk 7\n",
      "kt 7\n",
      "xq 7\n",
      "cw 7\n",
      "sj 6\n",
      "nç 6\n",
      "mk 6\n",
      "vê 6\n",
      "yv 6\n",
      "kd 6\n",
      "sè 6\n",
      "cz 5\n",
      "ïr 5\n",
      "zè 5\n",
      "ça 5\n",
      "âg 5\n",
      "fn 5\n",
      "zv 5\n",
      "sz 5\n",
      "jr 5\n",
      "vô 4\n",
      "ûm 4\n",
      "èg 4\n",
      "zk 4\n",
      "hv 4\n",
      "ên 4\n",
      "lû 4\n",
      "ij 4\n",
      "fî 4\n",
      "ân 3\n",
      "ïs 3\n",
      "cm 3\n",
      "zs 3\n",
      "pâ 3\n",
      "ûl 3\n",
      "fh 3\n",
      "uï 3\n",
      "gf 3\n",
      "bc 3\n",
      "tâ 3\n",
      "mv 3\n",
      "pû 3\n",
      "rû 3\n",
      "ï_ 3\n",
      "dû 3\n",
      "uê 3\n",
      "pd 3\n",
      "èb 3\n",
      "uw 3\n",
      "éu 3\n",
      "vg 3\n",
      "âl 3\n",
      "êv 3\n",
      "uq 3\n",
      "éé 2\n",
      "vv 2\n",
      "tx 2\n",
      "îc 2\n",
      "dî 2\n",
      "aç 2\n",
      "ôn 2\n",
      "gz 2\n",
      "û_ 2\n",
      "gê 2\n",
      "_è 2\n",
      "oï 2\n",
      "îm 2\n",
      "bâ 2\n",
      "wp 2\n",
      "oî 2\n",
      "gâ 2\n",
      "éo 2\n",
      "ïe 2\n",
      "xé 2\n",
      "uu 2\n",
      "fm 2\n",
      "hê 2\n",
      "dz 2\n",
      "gb 2\n",
      "èq 2\n",
      "vt 2\n",
      "bî 2\n",
      "dâ 2\n",
      "oé 2\n",
      "ûc 1\n",
      "ûn 1\n",
      "cn 1\n",
      "xm 1\n",
      "vw 1\n",
      "bf 1\n",
      "ïl 1\n",
      "q_ 1\n",
      "éh 1\n",
      "câ 1\n",
      "mô 1\n",
      "mg 1\n",
      "iy 1\n",
      "pô 1\n",
      "fp 1\n",
      "fè 1\n",
      "xs 1\n",
      "nû 1\n",
      "rî 1\n",
      "hq 1\n",
      "bû 1\n",
      "mâ 1\n",
      "kk 1\n",
      "fê 1\n",
      "cê 1\n",
      "cg 1\n",
      "iâ 1\n",
      "mq 1\n",
      "âb 1\n",
      "nâ 1\n",
      "wg 1\n",
      "ïv 1\n",
      "éâ 1\n",
      "zb 1\n",
      "ôd 1\n",
      "oê 1\n",
      "hg 1\n",
      "zw 1\n"
     ]
    }
   ],
   "source": [
    "for r,c in sorted(bigrams,key=lambda x:x[1],reverse=True):\n",
    "    print r,c\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16694"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigrams[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqgAAAKoCAYAAAC7uA1cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XmYXHWd7/HPN70knaTTCQkQlhAMEGTR0GmWixAJy0UZ\nYJSQrVBhHBnHmRG4yQ1hGcBBGBCIZC5473gVZQaQivCIjvg8OgxD4II6snQAkZ0AsptA0umsvX3v\nH6dau5ssVd3nV+dXqffrefJIn5zz6a+VqupPn6WOubsAAACAWAzLegAAAACgLwoqAAAAokJBBQAA\nQFQoqAAAAIgKBRUAAABRoaACAAAgKhRUAAAARIWCCgAAgKhQUAEAABAVCmrkzGy5md2Y9RxpqaT/\nP2b2HTN738y6zezjWc+zPeV4XEN/DzO71czuSTGv4h8TAKhWtVkPAMTIzD4t6WxJx0l6VdLqbCeK\nwhmSOgPmny/JAuYDKJKZLZe0wt0XZj0LqhMFFdi6/SW94+6/yXqQWLj72sD57SHzgRiYWZ27h/xF\nD9gpcIg/JWb2KTN72MzWmNlqM7vXzKakFF9rZjeb2VozW2VmX08pV5ZYbGYvmdlmM3vNzC5JKXuk\nmd1mZu1m9paZpfabeGHuS8xspZltNLMVZnZmStm3SrpJ0j5m1mNmK9PILWSPNrMfmNl6M3vDzM5L\n8TDxMDO7rnBawjtm9rUUMv+o0g7xbyX/1MJrKBfqewxW4bG9ycyWmtkHZvaumX2p8Br6vpmtK7xG\nP53C9/lfIZ4nZlZf+P/wnpltKrwfHp5S9vLCe2Dq74Nbe59K87neZ/alZrZK0i/SyC1kzzazpwvv\ngavN7D4za0gh91YlR48uKLwHdpvZPinkvmpm5w9YtsLMrkgh+6/M7K2tLP83M7tlCLmnFn6uW+Hr\naYXH5Jo+69xiZrcNMn9C4XV4cZ9lnzCzLWZ2/GDnLuR8ofC8qBuw/Cdm9q9DzJ7c57nR0+fPA0PJ\n7YuCmp5Rkr4pabqkEyR1S/pxStl/oeTQ6hFKDoMuNLMvpZT9DUmLJV0p6SBJ8yS9m1L2EkkzJJ0u\n6WRJM5U8Pmm4VNLnJX1Z0sGSlkq63cxmpJB9vqQrJL0paXclj3talko6WtJpkj6l5DFpTin7HEnr\nJR2p5N/0CjM7MaXsimZmZ0n6gaScu+eznmcbzpa0Ssnz7SZJ35Z0t6RfKnmO3CfpNjMbkcL3CfE8\nuUHJaSBfUDLvy5L+3czGppAtJXOHeB/c2vtUWq/JXmdL2iLpE5K+kkagmU2UdKekWyR9VEmhvEfp\nnCZzgaRfS/qukvfAPSS9kUJuSHdL2qVvqTOzcUreZ+8YQu7DkkbrT8+J45S8Tmf2WeeTkpYPJtzd\nV0v6S0lXmtl0Mxst6TZJN7n7oDL7uFtJz/vz3gVmtqukP5P0vSFm/17SRCXPjYlKHp/3JT00xNw/\ncXf+BPgjaYKkHkkHDzFnuaRnBiy7duCyQWaPlrRJ0hcD/P8fJWmzpFl9lo2TtEHSjUPMrlfyA/ao\nAcu/K+mOlOa/QNLKlB+T0Up+SJ3RZ9mYwv+XoT4myyU9NGDZbyRdk+L8y4c65w7yb5V0T9rzSvpb\nSR9IOjbAzKk8JgP//ZT8UGmX9C99lu1eeE85MrbniaSRhef2vD7LapX8kvc/U3p8Un8fDPk+NWD2\nxwM895qV7AiZlHZ2n7lTfb0rOZ///AHLVki6IqX8H0v6bp+vvyzpjRRyH5e0sPDf90i6uPCzc6Sk\nvQqvy/2G+D1ulvS8kjL9pKS6lB6T/y3pZ32+XijppZT/XYdL+i9JP0kzlz2oKTGz/c3sTjN7xcza\nlLwQXdKQD4so+Yfv69eSDug95DAEBykpe6ntku9jP0l1kh7tXeDuayS9kEL2/kreGP6jcFiu3cza\nley52S+F/FCmKPmh/VjvAndfp3QeE0l6esDX70jaLaXsSjVHSUn97+7+SNbD7MAf//3cvUfJ3ojf\n9ln2XuE/h/pvGuJ5sp+S5/avehe4e5eS1/9BQ8zuFeJ9MOT7VF9PpJwnSU9J+k9Jz5jZXWZ2bop7\nqyvVDySd2eeQ9lmSlqWQ+5D+tMd0hpKS+pykY5XsPX3L3V8Z4ve4UMlraLakszy985S/K+lkM9uj\n8PU5SnYGpOlWJb/sfS7NUApqen6m5Dfvc5UcOjtSyaGW+iyH2oFNWQ8wSKML//tnkqb1+XOwkhd3\ntRr4hubiNd6q5HBcWqfEhLS1f7+t/ZAa6r8pz5Py25B2oLv3uPvJkj4t6XeSzpP0vJlNTvt7pahH\nHz4FoW5rKw7SvUqey6ea2d5KyuRQDu/3elDSsWY2TVKHu7+opLQer+SQfxqHtfeXtKeS+T+SQp4k\nyd2fVPJL6dlmNl3Jz8khnX/al5ldJum/Szrd3VN9nvOmlAIz20XSVElXu/tyd39B0vgUv8VRA74+\nWskueh9i7ktKDm+FOE/xFUld6jN74XygqSlkP6vkcOJkd1854M+HTpKPyEolj8kfz2k1syal85hg\n615R8kPkM2Z2c9bD7MReUVJ8j+ldYGa1Sp7rv0vpe4R4Hwz5PlUW7v5rd79SySH/TiXnAaehQ1JN\nSlm9Vik5Z1GSZGZjlG4Z26Jk7+bnJeUkPe/uT6UQ/bCS07EW6E9l9EEle1WPK/z3oBX2+N6uZG/v\n5ZK+Z2YThpI5wC2Svlj4c39aPyctuTD5Mklz3P21NDL74mOm0rFGyeG4L5vZu5ImKzk/aqgFstc+\nZrZE0ncktUj6qpIXypC4+xYzu07S9WbWqeRijF0lHeLu3x9i9gYz+56kG8zsAyVvTFcrOWdqqHOv\nLzweS82sRtIjkpqU/HBsc/fbh/o9QijM/a+SlpjZGiWPyT8oeUzSeq5gAHd/uXDhxHIz63L3Ib92\n0J+7bzSzf1byel+j5IKaxZIaJA3pvaSP1N8HQ75PhWZmRyrZuXCfpD9I+m9Krn14NqVv8Zqkowp7\nZNdL+iCFnSIPSDrHzH4mqU3JxbldQ8wc6AdKjmgeoqT0DZm7rzWzp5Ucwv67wuL/J+kuJT1qqHtQ\nr1FSgM+TtFHJ0cFblVy4l4Y7lVwMeK6SU+GGzMwOUbIn9jpJz5nZ7oW/6iicJjNkFNQUuLub2Twl\nV97+Vsn5S+driL9V9cYruaKvQcl5Ul2Slrr7oD82o1+4+9cL5fRKJYcX3lFy9XAaLlRyXspPlVzw\n8U0lL8Ihc/fLzewPSk5WnyJprZLDuddsd8PsLVDy+N4raZ2k6yVNUrIneyjKUXArrUT/cV53f7Fw\npXpvSb0w7e8RIKfYZUP9Pmm5WMnh29skNSq5sORkd29LKT/U++DW3qeaUsjtFeoxX6fk/McLlLyv\nvq7kQp77UspfIulflBTeEUr2dP5+iJnXStpXyftfm5K9hfsOMXOgB5RcFHmAkmKWloeUnEr2oJSc\nq2xmz0ra1d1fGmyomR2npC/M7D1EbmZnS3rSzP7a3f/vUAd393Vm9iMlxfffhppXcLiS1+NlhT+9\nHlLySUZDZkP/hQjAYJnZSElvKfnBkvaJ6xXFzO6U1OXuZ2c9C+JiZb6rUbm/HxCamd0v6beVdASp\navagmtkkJb8Juj58krYr+TioN8s+GKqKmR2m5DMLH5U0VsnnrbrS+6224hRO0zhQyTmFae29B4Cq\nV/hkh96Luf4m43FKUjUFVdLbSnbPb+/vgXJYpOQijA4lHz9zrLt/kO1ImTpUyccT/acoqNi6ch/q\n49AidhYrlOwMWTyUUxGywCF+AAAARIWPmQIAAEBUKKgAAACICgUVAAAAUdkpLpIys/GSPqXkg4WH\n+nmSAAAASN8IJZ99++/u/v72VtwpCqqScvqDrIcAAADADn1OO7iRws5SUF+TpDvuuEMHHXTQDlde\nsGCBli5dWnT4mjXF37Xr8ssv11VXXVX0+k1Nxd+wZOHChbrxxhuLXr+np6fodRctWqQlS5YUvf51\n111X9Lq/+tWv9IlPfKLo9Zubm4te94477tDnP//5otevrS3+KX/bbbfp7LOL/8z4TZs2Fb3usmXL\nNH/+/KLXL+Xx+9rXvqYrr7yy6PVHjRpV9LoXX3yxvvGNbxS9fk1N8bfyXrx4sa6//vqi11+xYkXR\n637729/WV77ylaLX32OPPXa8UsE111yjSy+9tOj1t2zZUvS6N9xwgy68sPibXq1fv77odb/1rW/p\nq1/9atHrT5hQ/K3BS31Mxowp/gZzl112ma6++uqi1x87dmzR65b6s6GU5+DNN9+s8847r+j1X3nl\nlaLXvfvuuzVnzpyi199rr72KXrfU187UqVOLXveKK67Q17/+9aLXL+X9pNTnSXt7e9Hr/uM//qP+\n/u//vuj1u7uLv3Putddeq0suuaTo9b/1rW8Vve7TTz+tj3/840Wv/7nPfa7odYt9fr/++uu9Hem1\nHa27sxTUzZJ00EEHafr06Ttcuampqaj1eq1atarodceMGVPSE2D8+PFFr1vq3KUU1FKzd91116LX\nHT58eEnr77vvvkWvO3LkyJLWr6+vLyl7ypQpRa9fSjloaGjQ5MmTi16/lOdUqc/BxsbGkrIPO+yw\notcv5ReCpqamkn45KeXxHjVqlA444ICi1y/l36axsVGHHHJI0euX8otMY2NjUb9091q3bl3R644a\nNUoHHnhg0etPnDix6HVLfUx22WWXotcdM2aMpk3b3kda9xfyPXbjxo1Frzt69OiSHu/Ozs6i121o\naNA+++xT9PqlvK+NHj26pNdOyPeqUt5PSn2erF27tuh1Gxsbdeihhxa9fldXV0nZpbx2xo0bV/S6\ndXV1Ja1fyvO11Oe3ijgdk4ukAAAAEBUKKgAAAKJCQQUAAEBUqrKg5nK5YNlnnHFGsOxSLqop1bx5\n84Jl77///sGyjz766GDZxxxzTLDso446Klj2Zz/72WDZpVyIEVP28ccfHyz7tNNOC5b96U9/Olj2\nSSedFCw75GMya9asYNkhfzaEfLyPOOKIYNkzZ84Mlh3y52XI58npp58eLPvUU08Nlj1p0qRg2SGe\n3+buqYeWm5lNl/TEE088UdIJ7sUq5SKpUpVyAn+pSrlIqlSlXP1bqmOPPTZYdikXSZWqlIt2SnXK\nKacEyy7lIqlSlXJRQ6kefvjhYNmlXCRVqlIukipVKRdJlaqUi6RKVcpFUqUK+R77yCOPBMt+5pln\ngmWXcpFUqUq5MKlUId9PSrlIqlSlXCRVqlI+LaNUCxYsSD3zhRde0LnnnitJLe7eur11q3IPKgAA\nAOJFQQUAAEBUKKgAAACICgUVAAAAUdlZ7iQlKTmhd+CtQ3O5XNArMwEAANDf/fffr/vvv7/fslIu\nJt6pCurSpUuDXMUPAACA4p100kkf+vipPlfx7xCH+AEAABAVCioAAACiQkEFAABAVKI8B9XMJkl6\nVpJLsgF/7ZIOdvc3yz4YAAAAgouyoEp6W9L27pf2drkGAQAAQHlFWVDdvVvSyqznAAAAQPlxDioA\nAACiQkEFAABAVKI8xD9Y7733nt58M/1rp/bee+/UM3u9/Xa402m7urqCZd90003Bsjs7O4Nlb9q0\nKVh2XV1dsOy33norWPaECROCZb/++uvBsmfOnBkse+3atcGye3p6gmUfeeSRwbLr6+uDZYd43+41\nfvz4YNkf+9jHgmU3NzcHyzYbeO1xekI+T0K+Ljds2BAse5999gmWvWzZsmDZW7ZsCZZdDPagAgAA\nICoUVAAAAESFggoAAICoUFABAAAQFQoqAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRibagmtmnzOxh\nM1tjZqvN7F4zm5L1XAAAAAgr2oIqaZSkb0qaLukESd2SfpzpRAAAAAgu2luduvs9fb82s3Ml/cHM\nDnb3ZzMaCwAAAIFFuwfVzPY3szvN7BUza5P0qiSXFO6mtgAAAMhctHtQJf1MSSk9V9LbSsr07yTV\nb2uDK6+8Uo2Njf2WfeYzn9FnP/vZgGMCAACgr7vvvlt33313v2Xr1q0revsoC6qZ7SJpqqQvufsv\nC8uO3dF2X/va1/Sxj30s9HgAAADYjjlz5mjOnDn9lj355JOaMWNGUdtHWVAlrZH0vqQvm9m7kiZL\nulbJIX4AAADsxKI8B9XdXdI8SS2Sfqvkav5FmQ4FAACAsoh1D6rc/QFJhw5YXJPFLAAAACifKPeg\nAgAAoHpRUAEAABAVCioAAACiQkEFAABAVCioAAAAiEq0V/EPxpgxYzRu3LjUc7u6ulLP7DVx4sRg\n2cOGVebvH/X127xZ2JCNGjUqWHZITU1NwbKTT3ULY//99w+WHdKuu+4aLDvEe1Sv2tpwb+khnyeT\nJk0Klh1SyNdlperu7g6WPWHChIrMrlR1dXWpZ44cObLodSuzwQAAAGCnRUEFAABAVCioAAAAiAoF\nFQAAAFGhoAIAACAqURZUM1tuZjdmPQcAAADKL8qCCgAAgOpFQQUAAEBUYi6otWZ2s5mtNbNVZvb1\nrAcCAABAeDEX1L+Q1CnpCEnnS1poZl/KdCIAAAAEF/OtTn/v7gsL//2SmX1c0gJJ38twJgAAAAQW\nc0H9rwFf/1rJXlTzbdwU+qKLLvrQvZFnz56tuXPnBhoRAAAAA+XzeeXz+X7L2trait7ettH1MmVm\nyyW94u7n9ln255LuljRiYEE1s+mSnnjkkUd02GGHpT7P8OHDU8/sNWxYuLMsQmZj5xHyPcDMgmVX\nqq6urmDZtbXh9jnwPEExuru7g2XX1NQEy0Z5tLa2qqWlRZJa3L11e+vG3GCOGvD10ZJe2tbeUwAA\nAOwcYi6o+5jZEjObamY5SV+V9E9ZDwUAAICwYj0H1SXdJqlB0qOSuiQtdfdbMp0KAAAAwUVZUN39\nhD5f/l1mgwAAAKDsYj7EDwAAgCpEQQUAAEBUKKgAAACICgUVAAAAUaGgAgAAICpRXsU/FCHuSNLZ\n2Zl6Zq+GhoZg2ZV6R49KvdNOpdq4cWOw7FGjRgXLrlQhn4MbNmwIlh3yvapS7yTV09MTLDvkPWkq\n9f27Uu8k9cYbbwTLnjRpUrDsrLEHFQAAAFGhoAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBUKKgAA\nAKJCQQUAAEBUKKgAAACIStQF1RKLzewlM9tsZq+Z2SVZzwUAAIBwYr+tzjckfUnS/5D0S0m7STo4\n04kAAAAQVLQF1cxGSzpf0t+6+x2Fxa9K+k12UwEAACC0aAuqpIMk1Ut6oNgNLrroIjU1NfVbNmfO\nHM2dOzfl0QAAALAt+Xxe+Xy+37K2trait4+5oG4qdYPrrrtOzc3NIWYBAABAkXK5nHK5XL9lra2t\namlpKWr7mC+SeknSZkknZj0IAAAAyifaPajuvsXMrpN0vZl1KrlIaldJh7j797OdDgAAAKFEW1Al\nyd2/XiinV0raU9I7kr6d7VQAAAAIKeqCKknufq2ka7OeAwAAAOUR8zmoAAAAqEIUVAAAAESFggoA\nAICoUFABAAAQFQoqAAAAohL9VfylMDOZWeq53d3dqWeWQ01NTbDsnp6eYNnr1q0Llr3LLrsEy8bO\nY+3atcGyx44dGyx75cqVwbIPOuigYNnDhlXmvpKOjo5g2SNGjAiWHVKl/lu6e7DsSZMmBcsOKcRj\nUkpmZT6TAAAAsNOioAIAACAqFFQAAABEhYIKAACAqFBQAQAAEJWKKahmttzMbsx6DgAAAIRVMQUV\nAAAA1YGCCgAAgKhEWVDNbKSZ3WZm7Wb2lpktzHomAAAAlEeUBVXSEkkzJJ0u6WRJMyVNz3IgAAAA\nlEd0tzo1s1GS/lLSWe7+YGHZOZLezHIuAAAAlEd0BVXSfpLqJD3au8Dd15jZCzvacPHixWpqauq3\nbO7cuZo7d27qQwIAAGDr8vm8li1b1m/Z2rVri94+xoI6aNdff72am5uzHgMAAKCq5XI55XK5fsta\nW1t1+OGHF7V9jOegviKpS9JRvQvMbJykqZlNBAAAgLKJbg+qu28ws+9JusHMPpC0StLVkrqznQwA\nAADlEF1BLbhQ0ihJP5XULumbksZkOhEAAADKIsqC6u4bJJ1T+NPrmxmNAwAAgDKK8RxUAAAAVDEK\nKgAAAKJCQQUAAEBUKKgAAACICgUVAAAAUYnyKv7BqqmpUU1NTeq5DQ0NqWf2+tGPfhQse9999w2W\nPXbs2GDZe++9d7DsVatWBcvesGFDsOyf/vSnwbK/+MUvBssOObe7B8v+yEc+Eiz7qaeeCpZ9yCGH\nBMuurQ334yLkYzJt2rRg2a2trcGyJ0+eHCy7vr4+WPaoUaOCZV944YXBss8777xg2aNHjw6WHbKf\nPPLII6lnvvzyy0Wvyx5UAAAARIWCCgAAgKhQUAEAABAVCioAAACiQkEFAABAVCioAAAAiAoFFQAA\nAFGhoAIAACAq0RZUS1xiZivNbKOZrTCzM7OeCwAAAGHFfCepSyWdJenLkl6W9ElJt5vZH9z94Uwn\nAwAAQDBRFlQzq5d0iaQT3f03hcWvmdkMSX8tiYIKAACwk4qyoEraX9JISf9hZtZneZ2kFdvaaNGi\nRWpqauq3bN68eZo3b16QIQEAAPBhDz74oB566KF+yzZu3Fj09rEW1NGF//0zSW8P+Lst29poyZIl\nam5uDjYUAAAAdmzmzJmaOXNmv2Uvv/yyLrjggqK2j7WgPqukiE5290eyHgYAAADlE2VBdff1ZrZE\n0lIzq5H0iKQmScdIanP32zMdEAAAAMFEWVAlyd0vN7M/SLpY0hRJayW1Srom08EAAAAQVLQFVZLc\n/WZJN2c9BwAAAMon2g/qBwAAQHWioAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBVz96xnGDIzmy7p\niV/+8pdB7iQ1YsSI1DN79b+TKySpp6cnWPaGDRuCZTc2NgbLDqmzszNYdl1dXbDskLq6uoJl19aG\n+/CUkO/nH3zwQbDs8ePHB8tGeYV8DvLzsvK1traqpaVFklrcvXV767IHFQAAAFGhoAIAACAqFFQA\nAABEhYIKAACAqFBQAQAAEJWKKahm9h0ze9/Mus3s41nPAwAAgDDCfd5Jiszs05LOlnScpFclrc52\nIgAAAIRSEQVV0v6S3nH332Q9CAAAAMKKvqCa2a2SzpHkZtYj6TV3n5LxWAAAAAgk+oIq6XxJr0j6\nK0mHSwp3myEAAABkLvqC6u7tZtYuqdvdV2U9DwAAAMKKvqCWYvHixWpqauq3bO7cuZo7d25GEwEA\nAFSffD6vfD7fb1lbW1vR2+9UBfX6669Xc3Nz1mMAAABUtVwup1wu129Za2urWlpaitq+Yj4HFQAA\nANWBggoAAICoUFABAAAQlYooqO7+v/jsUwAAgOpQEQUVAAAA1YOCCgAAgKhQUAEAABAVCioAAACi\nQkEFAABAVHaqO0nV1taqrq4u9dzu7u7UM3vV1lbmP4G7B8s2s2DZjY2NwbJ7enqCZQ8bFu53yZBz\nV6pKfV3+/ve/D5Y9efLkYNmVqlJf8yFt2rQpWPbIkSODZSM+lfkKAAAAwE6LggoAAICoUFABAAAQ\nFQoqAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRoaACAAAgKlEWVDMbbWY/MLP1ZvaGmZ1nZsvN7Mas\nZwMAAEBYURZUSUslHS3pNEmfkjRTUnOWAwEAAKA8orufn5mNlnS2pPnu/mBh2RclvZ3lXAAAACiP\n6AqqpClK5nqsd4G7rzOzF3a04aJFi9TU1NRv2bx58zR//vzUhwQAAMDW5fN55fP5fsva2tqK3j7G\ngjpoS5Ys0fTp07MeAwAAoKrlcjnlcrl+y1pbW9XS0lLU9jGeg7pSUpekI3oXmFmTpKmZTQQAAICy\niW4PqruvN7N/lbTEzNZIWiXpHyR1S/IsZwMAAEB4Me5BlaQFkn4l6V5J90l6RNLzkjZnORQAAADC\ni7KguvsGd/+Cuze6+16SvivpQEkvZzwaAAAAAovuEL8kmdlhkj4q6VFJYyVdoeTw/r9lORcAAADC\ni7KgFixScmFUh6QnJB3r7h9kOxIAAABCi7KguvuTkg7Peg4AAACUX5TnoAIAAKB6UVABAAAQlSgP\n8Q/W+vXrS7qNVrHGjx+femY5PPPMM8GyDz300GDZleqNN94Ilj158uRg2cOHDw+WHVJHR0ew7Pr6\n+mDZIYV8noTU3d0dLLumpiZY9rBh4fbxhPhZ1uuBBx4Iln3GGWcEyw6ptbU1WPa0adOCZYd8fm/c\nuDH1zE2bNhW9LntQAQAAEBUKKgAAAKJCQQUAAEBUKKgAAACICgUVAAAAUYmyoJrZcjO7Mes5AAAA\nUH5RFlQAAABULwoqAAAAohJzQR1mZteZ2ftm9o6ZfS3rgQAAABBezAX1HEnrJR0pabGkK8zsxGxH\nAgAAQGgxF9Sn3f0qd3/F3W+X9LgkCioAAMBOrjbrAbbj6QFfvyNpt+1tcNlll2nMmDH9ls2aNUtn\nnnlmyqMBAABgW+666y7dfffd/Za1tbUVvX3MBbVzwNeuHezxvfrqqzVt2rRwEwEAAGCH5s6dq7lz\n5/ZbtmLFCh177LFFbR/zIX4AAABUIQoqAAAAohJrQfWsBwAAAEA2ojwH1d1P2MqyM7KYBQAAAOUV\n6x5UAAAAVCkKKgAAAKJCQQUAAEBUKKgAAACICgUVAAAAUYnyKv7Bqq2tVV1dXeq5GzZsSD2z17Bh\n4X5HmDJlSrDsNWvWBMseeLvaNK1evTpY9vjx44Nlv/jii8Gyp06dGizbPdwnxr3++uvBskM+B2tq\naoJljx49Olh2iPfWXqtWrQqWPXHixGDZIecePnx4sOwTTvjQB+WkZvPmzcGyV65cGSy7tjZcHXr3\n3XeDZe++++7Bsjdu3Jh6ZinPD/agAgAAICoUVAAAAESFggoAAICoUFABAAAQFQoqAAAAolIxBdXM\nlpvZjVnPAQAAgLAq6WOmzpDUmfUQAAAACKtiCqq7r816BgAAAITHIX4AAABEpWIKKgAAAKoDBRUA\nAABRqZhzUItxySWXqKmpqd+y2bNna/bs2RlNBAAAUH3uuece3XPPPf2WrVu3rujtd6qCeu211+qw\nww7LegyLztFgAAAgAElEQVQAAICqNmvWLM2aNavfsqeeekonnXRSUdtziB8AAABRoaACAAAgKpVU\nUD3rAQAAABBexZyD6u4nZD0DAAAAwqukPagAAACoAhRUAAAARIWCCgAAgKhQUAEAABAVCioAAACi\nUjFX8Rdj1KhRamxsTD3XzFLP7OUe7tOzQs49cuTIYNkh7b777lmPMChTp07NeoRBCfkcPOCAA4Jl\no7wmTpyY9QiDsuuuu2Y9QlU5+OCDsx6hqkyYMCH1zHHjxhW9LntQAQAAEBUKKgAAAKJCQQUAAEBU\nKKgAAACICgUVAAAAUamYgmpmt5rZPVnPAQAAgLAq6WOmzpcU7jNrAAAAEIWKKaju3p71DAAAAAiP\nQ/wAAACISsUUVAAAAFQHCioAAACiUjHnoBZjwYIFGjt2bL9l8+fPVy6Xy2giAACA6pPP55XP5/st\na2trK3p7c/e0ZwrCzG6V1OTus7byd9MlPfH4449r+vTpIb536pm9Qj7+IecGAAAoRWtrq1paWiSp\nxd1bt7cuh/gBAAAQFQoqAAAAokJBBQAAQFQqqaAOl7Q+6yEAAAAQVvQF1cxqzOxgSUdL+l3W8wAA\nACCs6AuqpEMlPSbpt5K+nfEsAAAACCz6z0F196ckjcp6DgAAAJRHJexBBQAAQBWhoAIAACAq0R/i\nj0F3d3ew7JqammDZ7e3twbIbGxuDZaO8uJvZh/X09ATLHjYs3H6BkHOHfJ6EfB8MKeTjHVLI52BX\nV1ew7NpaKks1YQ8qAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRoaACAAAgKtEWVDNbbmY3Zj0HAAAA\nyivaggoAAIDqREEFAABAVCqmoJrZqWa21sxyWc8CAACAcCritgxmdpak/yMp5+4/z3oeAAAAhBP9\nHlQz+1tJ35J0GuUUAABg5xf7HtQ5knaVdIy7P7GjlRcsWKCxY8f2WzZ//nzlcpwVAAAAUC75fF75\nfL7fsra2tqK3N3dPe6ZUmNlySeskTZd0r7v/7XbWnS7piccff1zTp09PfZaenp7UM3vV1NQEy25v\nbw+W3djYGCwb5RXyPcDMgmWHFPI1P2xYuANXIecO+TwJ+T4YUsjHO6SQz8Gurq5g2bW1se9Tw460\ntraqpaVFklrcvXV768Z+iP8VScdL+oyZ3Zz1MAAAAAgv+l9H3P1lMzte0nIz63L3BVnPBAAAgHBi\nLqh/PJ7k7i+a2Yn6U0m9MMO5AAAAEFC0BdXdTxjw9fOS9shoHAAAAJRJ7OegAgAAoMpQUAEAABAV\nCioAAACiQkEFAABAVCioAAAAiEq0V/EPRkdHh7Zs2ZJ67qZNm1LPLEf2uHHjgmWHvINKiH/DXpV6\nx5r6+vpg2evXrw+WHfKOY93d3cGyX3311WDZU6ZMCZYdUkdHR7DskHe922233YJlhxTyLmwhXztr\n164Nlj1+/Phg2SFV6h31QtwVrJRM9qACAAAgKhRUAAAARIWCCgAAgKhQUAEAABAVCioAAACiUjEF\n1cyWm9mNWc8BAACAsCqmoAIAAKA6UFABAAAQFQoqAAAAokJBBQAAQFQoqAAAAIhKbdYDpGnx4sVq\namrqt2zOnDmaN29eRhMBAABUn2XLlumHP/xhv2VtbW1Fb79TFdTrr79ezc3NWY8BAABQ1ebPn6/5\n8+f3W9ba2qqjjjqqqO05xA8AAICoUFABAAAQlUoqqJ71AAAAAAivYs5BdfcTsp4BAAAA4VXSHlQA\nAABUgUHvQTWzOkkTJY2UtMrdP0htKgAAAFStkvagmlmjmf2NmT0kaZ2k1yQ9J2mVmb1uZt81syMC\nzAkAAIAqUXRBNbOFSgrpFyXdL+mzkg6TNFXS0ZKuVLJH9j4z+4WZHZD6tAAAANjplXKI/whJn3T3\n323j7x+V9H0z+4qSEjtD0ktDnA8AAABVpuiC6u65ItfbIunbg55oCHp6etTT05N67qhRo1LP7DVu\n3Lhg2ZWqtjbch0vU1dUFy65UZpb1CINSU1MTLHu//fYLlt3Z2Rksu76+Plh2yLl32223YNkhhfh5\n02vYsHDXMId87Qy83XiaKvW9CoPDVfwAAACISqoF1cz2M7MH0swEAABAdUl7D+poScelnAkAAIAq\nUtLJfmZ2/g5W2WsIswAAAAAlf1D/P0l6R1LHNv4+3Bn6AAAAqAqlFtTXJV3k7ndt7S/N7DBJTwx5\nKgAAAFStUs9BfUJSy3b+3iXxORAAAAAYtFL3oF4haeR2/v5ZSR8Z/DgAAACodiXtQXX3Z9398e38\nfae7v15KppktN7ObzGypmX1gZu+a2ZfMbKSZfd/M1pnZS2b26VJyAQAAUJmG/DFTZnaxmY0dYszZ\nklYpuZ3qTUruRHW3pF9KapZ0n6TbzGzEEL8PAAAAIpfG56BeKmmXIWY85e7XuPsrkr4habOkVe7+\nvcKyr0uaIOnjQ/w+AAAAiFwaNz1P46Kop3v/w917zOx9Sb/ts+y9wj14t3vD5osuuuhD9wGeM2eO\n5s6dm8KIAAAAKMayZcv0wx/+sN+ytra2ordPo6CmoXPA176VZdIO9vhed911am5uTm0oAAAAlG7+\n/PmaP39+v2Wtra066qijito+jYJ6sKS3U8gBAAAABncOqpmtNLPxkuTub7h7d2H5WDNbmeaAAAAA\nqC6DvUhqX0k1W1k+XNJeJWb5EJYBAABgJ1PSIX4z+/M+X37KzPqe7Voj6URJr5WS6e4nbGXZlK0s\n21ohBgAAwE6m1HNQf1L4X5f0rwP+rlNJOf2fQ5wJAAAAVaykguruwyTJzF6VdIS7rw4yFQAAAKrW\noK7id/ePpD0IAAAAIJVwkZSZzd/xWn9cd5KZHTO4kQAAAFDNSrmK/2/M7DkzW2xmBw38SzNrMrM/\nM7M7JbVKGp/alAAAAKgaRR/id/fjClfxnyfpWjPbIOk9SZsljZM0UdJqSf8i6VB3fy/9cbevtrZW\ntbXp3xzrySefTD2z15FHHhksu1KtXbs2WPauu+4aLLuzc2s3P0tHXV1dsOzHHnssWPbxxx8fLPu9\n98K9xey+++7Bsuvr64Nld3d3B8tetWpVsOwxY8YEy65Uw4YN9lMgs/Xuu+8Gy95zzz2DZXd0dATL\nbmhoCJYdUk1N+h+eVEpmqRdJ/VTST81sgqQZkvaR1KCkmK6QtMLde0rJBAAAAPoa7O7Gj7n7j7f2\nF2b21+7+f4cwEwAAAKrYYI8h/MLMbjCzPx53NLMJZnavpG+kMxoAAACq0WAL6vGSzpD0mJkdbGan\nSnpG0hhJh6U1HAAAAKrPoAqqu/9KSRF9RskV+z+WtFTSTHd/Pb3xAAAAUG2GcpngVEmHS3pTUpek\nAyWNTGMoAAAAVK9BFVQzu1jSryX9h6RDJR0pqVnS02Z2dHrjAQAAoNoMdg/qBZI+6+7nuftmd39G\nSUm9R9KDaQ0HAACA6jOUj5la3XeBu3dKutDMfjbUocxsuaSnldwE4FxJHZK+7e5XDjUbAAAAcRvs\nRVKrt/N3Dw1+nH7OlrReyZ7ZxZKuMLMTU8oGAABApGK+l9rT7n6Vu7/i7rdLelwSBRUAAGAnl/6N\n69Pz9ICv35G02/Y2WLRokZqamvotmzdvnubNm5fyaAAAANiWfD6vZcuW9Vu2du3aorePuaB2Dvja\ntYM9vkuWLFFzc3O4iQAAALBDuVxOuVyu37LW1lYdfvjhRW0f8yF+AAAAVCEKKgAAAKISa0H1rAcA\nAABANqI8B9XdT9jKsjOymAUAAADlFeseVAAAAFQpCioAAACiQkEFAABAVCioAAAAiAoFFQAAAFGJ\n8ir+wdqyZYs2b96ceu6RRx6ZemavN998M1j2u+++Gyx76tSpwbInTJgQLPv5558Plj1+/Phg2SGF\nfH4/+eSTwbIPPfTQYNkhXztdXV3BshsaGoJl77nnnsGy33jjjWDZkyZNCpa9atWqYNmjR48Olt3e\n3h4sO+TjvXr16mDZ7uE+3TLk6/K5554Llr1ly5bUM1988cWi12UPKgAAAKJCQQUAAEBUKKgAAACI\nCgUVAAAAUaGgAgAAICoUVAAAAESFggoAAICoRFtQzazezG4ys/fMbJOZPWxmh2c9FwAAAMKKtqBK\nukHSGZK+IKlZ0suS/t3MxmY6FQAAAIKKsqCa2UhJX5G0yN3vc/fnJf2VpE2SvpTpcAAAAAgqyoIq\naT8lt2H9Ve8Cd++S9Kikg7IaCgAAAOHVZj1Ami655BI1NTX1WzZ79mzNnj07o4kAAACqz89//nP9\n4he/6Lds/fr1RW8fa0F9RVKnpGMkLZMkM6uVdISkG7e10bXXXqvDDjusLAMCAABg60455RSdcsop\n/ZY999xzOuuss4raPsqC6u4bzeyfJd1gZmskvSFpsaQGSd/PdDgAAAAEFWVBLbhYkkm6TVKjpMcl\nnezubZlOBQAAgKCiLajuvkXS/yj8AQAAQJWI9Sp+AAAAVCkKKgAAAKJCQQUAAEBUKKgAAACICgUV\nAAAAUaGgAgAAICrRfszUYNTV1am+vj713K6urtQze+29994VmR1ST09PsOyPfvSjwbIrlbsHyw55\nZ7eQz5OJEycGy25vbw+W3djYGCw75Pvg2LFjg2WHtMceewTL7uzsDJYd8nmyadOmYNkTJkwIlh3S\nww8/HCx7xowZwbJDKOV9mz2oAAAAiAoFFQAAAFGhoAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBUK\nKgAAAKIS5eegmtlySc8UvvyCpE5J/+zuV2Q3FQAAAMoh5j2oZysppkdIOl/SQjP7UrYjAQAAILQo\n96AWvOHuCwv//ZKZfVzSAknfy3AmAAAABBbzHtT/GvD1ryUdYGaWxTAAAAAoj5j3oJZs8eLFampq\n6rdszpw5mjdvXkYTAQAAVJ98Pq98Pt9vWVtbW9Hbx1xQjxrw9dGSXnJ339YG119/vZqbm8NOBQAA\ngO3K5XLK5XL9lrW2tqqlpaWo7WM+xL+PmS0xs6lmlpP0VUn/lPVQAAAACCvmPai3SWqQ9KikLklL\n3f2WbEcCAABAaDEX1M7CVfx/l/UgAAAAKJ+YD/EDAACgCsVaULd5IRQAAAB2blEe4nf3E7KeAQAA\nANmIdQ8qAAAAqhQFFQAAAFGhoAIAACAqUZ6DOlj19fUaPnx46rlmlnomtm3YMH5vKqdKfX5X6vOk\nsbEx6xEGpbY23I+LSn1MQqqrq8t6hEFpaGjIeoTozJgxI+sRKlJlvsMDAABgp0VBBQAAQFQoqAAA\nAIgKBRUAAABRoaACAAAgKhRUAAAARIWCCgAAgKhEWVDNbKSZ3WZm7Wb2lpktNLPlZnZj1rMBAAAg\nrCgLqqQlkmZIOl3SyZJmSmrOciAAAACUR3R3kjKzUZL+UtJZ7v5gYdk5kt7Mci4AAACUR4x7UPeT\nVCfp0d4F7r5G0guZTQQAAICyiW4P6lAsWLBAY8eO7bds/vz5yuVyGU0EAABQffL5vPL5fL9lbW1t\nRW9v7p72TENSOMT/gZJD/D8qLBsn6Q1J33H3hVvZZrqkJx5//HFNnz49xEypZwIAAFST1tZWtbS0\nSFKLu7dub93o9qC6+wYz+56kG8zsA0mrJF0tqTvbyQAAAFAO0RXUggsljZL0U0ntkr4pqSnTiQAA\nAFAWMV4kJXff4O7nuHuju+/p7t/MeiYAAACUR5QFFQAAANWrkgpqXFdzAQAAIIhYz0H9EHc/IesZ\nAAAAEF4l7UEFAABAFaCgAgAAICoUVAAAAESlYs5BLUZXV5c6OztTz33//fdTz+w1YsSIisweNizc\n7zZvvfVWsOyRI0cGy25oaAiW3djYGCx7y5YtwbJDPiYor7fffjtYdnt7e7DsAw88MFh2yNfO8OHD\ng2WHnPvNN98Mlj1lypRg2SHvqhny5+XOjEcNAAAAUaGgAgAAICoUVAAAAESFggoAAICoUFABAAAQ\nFQoqAAAAolJxBdXM6rKeAQAAAOFE/zmoZrZc0jOSuiR9XtLTkk7MdCgAAAAEE31BLThb0j9L+kTW\ngwAAACCsSimoL7n7xVkPAQAAgPAq5RzUJ7IeAAAAAOVRKXtQNxSz0qJFi9TU1NRv2bx58zRv3rwg\nQwEAAODD8vm88vl8v2VtbW1Fb18pBbUoS5YsUXNzc9ZjAAAAVLVcLqdcLtdvWWtrq1paWoravlIO\n8QMAAKBKVEJB9awHAAAAQPlEf4jf3U/IegYAAACUTyXsQQUAAEAVoaACAAAgKhRUAAAARIWCCgAA\ngKhQUAEAABAVCioAAACiEv3HTJVi/fr1WrduXeq5w4aF6/FjxowJll1TUxMsu6OjI1j2uHHjKjK7\nq6srWHbI5+B7770XLHvfffcNlt3e3h4su6GhIVh2T09PsOzu7u5g2bvsskuw7AkTJgTLDink+2DI\n7JDP70rlXpkfub5ly5Zg2cOHDw+WXQz2oAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBUKKgAAAKJC\nQQUAAEBUKKgAAACIStQF1cxmm9nTZrbRzFab2X1mxge4AQAA7MSi/aB+M5so6U5JiyT9RFKjpBmS\nLMu5AAAAEFa0BVXSHpJqJP3Y3d8oLPtdhvMAAACgDGI+xP+UpP+U9IyZ3WVm55rZ2KyHAgAAQFjR\n7kF19x5JJ5vZ0ZJOlnSepKvN7Ch3f31r21x++eUfurf9rFmzNGvWrODzAgAAIJHP55XP5/sta2tr\nK3r7aAtqL3f/taRfm9lVkl6XdIakf9rauldddZWmTZtWzvEAAAAwQC6XUy6X67estbVVLS0tRW0f\nbUE1syMlnSjpPkl/kPTfJE2Q9GyWcwEAACCsaAuqpHWSPinpAkljlOw9Xeju92U6FQAAAIKKtqC6\n+/OSTsl6DgAAAJRXzFfxAwAAoApRUAEAABAVCioAAACiQkEFAABAVCioAAAAiAoFFQAAAFGJ9mOm\nBqO9vV1r1qxJPbe2NtzDNGxYuN8RSrmlWKmampqCZbt7sOzNmzcHyx4xYkSw7C1btgTLDvkcDKmx\nsTFYdnd3d7Ds+vr6YNnvvvtusOxRo0YFy66rqwuWHVLInw3vv/9+sOyGhoZg2T09PcGyzSxYdk1N\nTbDsF154IVj2/vvvHyw7a5X5kwkAAAA7LQoqAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRib6gmtly\nM7sx6zkAAABQHtEXVAAAAFSXqAuqmd0q6ThJF5hZj5l1m9k+Wc8FAACAcGL/oP4LJE2V9FtJl0sy\nSasynQgAAABBRV1Q3X2dmXVI2ujuFFMAAIAqEPUhfgAAAFSfqPegluqaa6750P25TzvtNJ122mkZ\nTQQAAFB98vm88vl8v2VtbW1Fb18JBbVDUk0xK1566aU65JBDAo8DAACA7cnlcsrlcv2Wtba2qqWl\npajtK+EQ/2uSjjKzyWY23sws64EAAAAQTiUU1CWSuiU9K+kPkiZlOw4AAABCiv4Qv7u/JOmYrOcA\nAABAeVTCHlQAAABUEQoqAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRoaACAAAgKtF/zFQp9t57b+23\n336p57p76pm9Vq5cGSx70qRwHxnb3d0dLLu+vj5Y9pYtW4Jlb968OVh2yMck5PPknnvuCZZ9+umn\nB8sO+Zrv6uoKlr377rsHy3711VeDZe+5557BskPe26WjoyNY9m677RYsO6QDDjgg6xGic+CBB2Y9\nwqBs2rQp9cxSfk6yBxUAAABRoaACAAAgKhRUAAAARIWCCgAAgKhQUAEAABAVCioAAACiQkEFAABA\nVKIsqGb2qpmdP2DZCjO7IquZAAAAUB5RFlQAAABULwoqAAAAokJBBQAAQFRqsx5gG3okDbyBct2O\nNlq4cKGampr6LZs/f75yuVyKowEAAGB77rrrLt111139lrW1tRW9fawFdZWkPXq/MLMxkj6yo41u\nvPFGTZ8+PeRcAAAA2IG5c+dq7ty5/ZatWLFCxxxzTFHbx3qI/wFJXzCzY83sY5L+RVJXtiMBAACg\nHGLdg3qtpH0l3SupTdLlha8BAACwk4uyoLp7u6SzBiy+PYtZAAAAUF6xHuIHAABAlaKgAgAAICoU\nVAAAAESFggoAAICoUFABAAAQFQoqAAAAohLlx0zFpru7O1j2fvvtFyy7UoV8vEeMGBEsu1J1dnYG\ny541a1aw7JDcPVi22cC7OKcn5Nx77bVXsOzhw4cHyw75mFTq8wTl1dUV7j5DtbXhalxDQ0PqmaX8\nDGYPKgAAAKJCQQUAAEBUKKgAAACICgUVAAAAUaGgAgAAICoUVAAAAESFggoAAICoRFlQzeyvzOyt\nrSz/NzO7JYuZAAAAUB5RFlRJd0vaxcyO711gZuMkfUrSHZlNBQAAgOCiLKjuvlbSLySd1WfxHEmr\n3P3BTIYCAABAWURZUAt+IOlMM6srfH2WpGUZzgMAAIAyCHcT16G7V0mBPtXMHpc0Q9IF29tg4cKF\nampq6rds/vz5yuVywYYEAABAf/l8Xvl8vt+ytra2orc3d097ptSY2fcljZH0G0l/4e6HbGO96ZKe\neOyxxzR9+vTU5+jq6ko9s1d9fX2w7ErV3d0dLLumpiZYdqXq7OwMll1XV7fjlSIU8n3RzIJlh5y7\no6MjWPbw4cODZYdUqc8TlFfIDlFbG/N+xg9rbW1VS0uLJLW4e+v21o39/9kPJP1M0iGSbs94FgAA\nAJRBzOegStIDkj6QdICkOzOeBQAAAGUQ9R5UT46f7JX1HAAAACif2PegAgAAoMpQUAEAABAVCioA\nAACiQkEFAABAVCioAAAAiAoFFQAAAFGJ+mOmSrVhwwatW7cu9dwxY8aknolt4w4q5VWpd3sKqVKf\ngyHnDnnHsUq9k1Sl3hUspEp97YRUaXd7igV7UAEAABAVCioAAACiQkEFAABAVCioAAAAiAoFFQAA\nAFGhoAIAACAqFFQAAABEJZqCamanmtkaK3yImplNM7MeM7umzzq3mNlt2U0JAACA0KIpqJIeljRa\nUnPh6+MkrZI0s886n5S0vLxjAQAAoJyiKajuvk7SU/pTIZ0paamkZjMbaWZ7Sdpf0v/LZEAAAACU\nRTQFteAh/amgzpB0j6TnJB2rZO/pW+7+SjajAQAAoBxiu0Hsg5K+aGbTJHW4+4tm9pCk4yWNU1Jg\nt+nSSy/VmDFj+i0788wzNXv27EDjAgAAYKB8Pq98Pt9vWVtbW9Hbm7unPdOgmdlYSasl3SGp3t3P\nMrPPSLpY0lhJ33T3W7ay3XRJTzz44IOaNm1a6nMNLL1pGjYstp3Y2evp6QmWzeMNDM769euDZY8e\nPTpYdqWK6WdzKQrXOQNb1draqpaWFklqcffW7a0b1U9rd18r6WlJn1OyN1VKzjmdLmmqdrAHFQAA\nAJUvqoJa8JCSuR6UJHdfI+lZSe+4+0sZzgUAAIAyiK6guvsCd69x9xf7LGt2972znAsAAADlEV1B\nBQAAQHWjoAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBUKKgAAAKIS261Oh6SxsVFjx47NegwA2Olw\nt6fy4o5MO4+Ojo5g2fX19cGys8YeVAAAAESFggoAAICoUFABAAAQFQoqAAAAokJBBQAAQFQoqAAA\nAIgKBRUAAABRibKgmtkEM3vHzC7us+wTZrbFzI7PcjYAAACEFeUH9bv7ajP7S0k/MbP7JL0o6TZJ\nN7n78mynAwAAQEhRFlRJcvefm9l3JN0p6XFJ6yVdmu1UAAAACC3KQ/x9XKikRM+WdJa7d2Y8DwAA\nAAKLdg9qwf6S9lRSpD8i6dntrbxgwYL/3979hdZ91nEcf3+bZcmiLJlW26o4J8OLuatTi0U3ypT5\nD9yNXuRsXYd/oHqxaSmKyG52M3CXXk5mrX+aBhnM7UJk/qvgTVmSIUI7hP0TlM0Gia5dTJt8vTgJ\nnKRt+ofz/M6T5P2C0J7nPL8v35ZfzvnwnN/5PYyOjq4aa7fbtNvtYg1KkiRptYmJCSYmJlaNzc3N\nXfXxkZm97qknImIQOAnMAC8Bh4A7M/PMJea2gKmpqSlarVazjarnlpaWitXetq32Dw0kSZvJwsJC\nsdo33nhjsdolTE9Ps3v3boDdmTm93tya360fB24GHgaeoBNSj/S1I0mSJBVXZUCNiH3AI8D+zDyb\nnWXeA8BdEXGwv91JkiSppCqvQc3ME8DQmrHXgFv605EkSZKaUuUKqiRJkrYuA6okSZKqYkCVJElS\nVQyokiRJqooBVZIkSVUxoEqSJKkqVd5m6notLi6yuLjY87oXLlzoec0V586dK1Z7bGysWO1adyDb\nrNxd62Ilz8HZ2dlitYeGhq486TqNjIwUq71Rz5OIKFZ7fn6+WO3h4eFitUu8T64YGBgoVrvk73zJ\n82RwcLBY7ZJKvO9cS82N+YojSZKkTcuAKkmSpKoYUCVJklQVA6okSZKqYkCVJElSVQyokiRJqooB\nVZIkSVWpMqBGxIMRcSYiBteMPxMRR/vVlyRJksqrMqACv6TT230rAxHxHuALwFP9akqSJEnlVRlQ\nM3MemAC+0jX8IPBaZv6pP11JkiSpCVUG1GU/Aj4TEbuWHz8EHOljP5IkSWrADf1u4HIy88WI+Atw\nICKeB+4A1r3+9PDhw4yOjq4aGx8fZ3x8vFyjkiRJWmViYoLjx4+vGpubm7vq4yMze91Tz0TEN4Bv\nA88Dt2fm5y8zrwVMnTx5klar1fM+Lly40POaK86dO1es9tjYWLHaNZ8369m2reYPDS5vaWmpWO2N\n+n9S8hycnZ0tVntoaKhY7ZGRkWK1N+p5EhHFas/PzxerPTw8XKz24uJisdoDAwPFapf8nS95nmzU\nvku870xPT7Nnzx6A3Zk5vd7c2l9xjgEfAL6OX46SJEnaEqoOqJn5H+Bp4C3gV31uR5IkSQ2oOqAu\nez/w88w83+9GJEmSVF61X5KKiDHgHmAf8M0+tyNJkqSGVBtQgRlgDPhuZv6t381IkiSpGdUG1My8\nrQRWyMgAAARiSURBVN89SJIkqXkb4RpUSZIkbSEGVEmSJFXFgCpJkqSqVHsN6vUYGBgosotFyZ0x\nSu4qU1LJ3St0sY26i09JJc/B7du3F6utzaPkbk8ludtTszZq3yV20byWXcx815MkSVJVDKiSJEmq\nigFVkiRJVTGgSpIkqSoGVEmSJFXFgCpJkqSqGFAlSZJUlWoDakTcGhFLEbG4/OfKz+/73ZskSZLK\nqflG/a8DO7se7wJ+C5zoTzuSJElqQrUBNTvbUbwJEBFDwLPAnzPzsb42JkmSpKKq/Yh/jSPAO4AH\n+t2IJEmSyqp2BXVFRDwK3Avsycyz6809dOgQo6Ojq8ba7Tbtdrtgh5IkSeo2OTnJ5OTkqrG5ubmr\nPj46n6TXKSK+BPwC+Fxm/nGdeS1gampqilar1VR7kiSpS8lMERHFautiCwsLPa85MzPD3r17AXZn\n5vR6c6tdQY2IjwJHgR8ApyJix/JTC5n57/51JkmSpJJqvgb1Y8BNwKPAP7p+nu5nU5IkSSqr2hXU\nzDxKZwVVkiRJW0jNK6iSJEnaggyokiRJqooBVZIkSVUxoEqSJKkqBlRJkiRVpdpv8V+jYYBTp071\nuw9JkrYsb9S/eZw/f77nNU+fPr3y1+Erza16J6mrFRH309lxSpIkSXV7IDOPrTdhswTUdwOfBV4F\n5vvbjSRJki5hGPgQ8JvMnF1v4qYIqJIkSdo8/JKUJEmSqmJAlSRJUlUMqJIkSaqKAVWSJElVMaBK\nkiSpKgZUSapMRJyIiPErzDkYEc821ZMkNcmAKkkViYj7gPdm5vGusaXl8W4/BloR8clGG5SkBhhQ\nJakuDwNHrjQpM88Dx4BvFe9IkhpmQJWkhkTE9oj4Z0R8r2vsExHxv4i4JyK2A58Cnut6/hUggWeW\nV1Jf7ir5HPDFiBhq6t8gSU0woEpSQzLzDPBV4LGIaEXEO4GfAj/MzD8AdwNnM/NU12F7gAAeAnYu\nP17xAjAIfLyJ/iWpKTf0uwFJ2koy89cR8SSdj+dfAN4Cvr/89AeBN9bMPxMRAHOZ+eaa596OiDng\n1uKNS1KDXEGVpOZ9h84CwZeB+5evJwW4CZi/xlpvAyM97E2S+s6AKknNux14H53X4Nu6xs8At1xj\nrXcB/+pRX5JUBT/il6QGRcQg8DPgOPAS8FRE3Ll8feoMsDMiRjNzruuw88DAJWp9GBhaPk6SNg1X\nUCWpWY8DN9O5ndQTdELqym2lZuisoq69t+mrwKcjYkdEjHWN3w28nJmvFO1YkhpmQJWkhkTEPuAR\nYH9mns3MBA4Ad0XEwcxcAn4C7F9z6GHgXuB1YLprvA08WbxxSWpYdF4fJUk1iIgdwF+BVmb+fZ15\ndwC/Az6Smf9tqj9JaoIrqJJUkcx8A/ganVtOrWcXcMBwKmkzcgVVkiRJVXEFVZIkSVUxoEqSJKkq\nBlRJkiRVxYAqSZKkqhhQJUmSVBUDqiRJkqpiQJUkSVJVDKiSJEmqigFVkiRJVfk/VT2F5g+tOKoA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11011c7d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "# Reduction table\n",
    "my2ascii_table = {\n",
    "    ord(u'â'):\"a\",\n",
    "    ord(u'ä'):\"e\",\n",
    "    ord(u\"à\"):\"a\",\n",
    "    ord(u\"æ\"):\"a\",\n",
    "    ord(u'ç'):\"c\",\n",
    "    ord(u\"é\"):\"e\",\n",
    "    ord(u\"è\"):\"e\",\n",
    "    ord(u\"ê\"):\"e\",\n",
    "    ord(u\"ë\"):\"e\",\n",
    "    ord(u'ğ'):\"g\",\n",
    "    ord(u'ı'):\"i\",\n",
    "    ord(u\"î\"):\"i\",\n",
    "    ord(u'ï'):\"i\",\n",
    "    ord(u'œ'):\"o\",\n",
    "    ord(u\"ô\"):\"o\",\n",
    "    ord(u'ö'):\"o\",\n",
    "    ord(u'ş'):\"s\",\n",
    "    ord(u'ù'):\"u\",\n",
    "    ord(u\"û\"):\"u\",\n",
    "    ord(u'ü'):\"u\",\n",
    "    ord(u'ß'):\"s\"\n",
    "    }\n",
    "\n",
    "\n",
    "def letter2idx(x):\n",
    "    if x=='_':\n",
    "        i = 0\n",
    "    else:\n",
    "        i = ord(x)-ord('a')+1\n",
    "    \n",
    "    if i<0 or i>26:\n",
    "        i = ord(my2ascii_table[ord(x)])-ord('a')+1\n",
    "        \n",
    "    return i\n",
    "\n",
    "T = np.zeros((27,27))\n",
    "# Convert bigrams to a transition matrix\n",
    "for pair in bigrams:\n",
    "    c = pair[1]\n",
    "    s = list(pair[0])\n",
    "    j = letter2idx(s[0])\n",
    "    i = letter2idx(s[1])\n",
    "    T[i,j] += c\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "\n",
    "alphabet=[chr(i+ord('a')) for i in range(26) ]\n",
    "alphabet.insert(0,'_')\n",
    "M = len(alphabet)\n",
    "\n",
    "plt.imshow(T/np.sum(T,axis=0), interpolation='nearest', vmin=0,cmap='gray_r')\n",
    "plt.xticks(range(M), alphabet)\n",
    "plt.xlabel('x(t)')\n",
    "plt.yticks(range(M), alphabet)\n",
    "plt.ylabel('x(t-1)')\n",
    "ax = plt.gca()\n",
    "ax.xaxis.tick_top()\n",
    "#ax.set_title(f, va='bottom')\n",
    "plt.xlabel('x(t)')\n",
    "\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Monte Carlo with Spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pi is roughly 3.158720\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pyspark\n",
    "\n",
    "def sample(p):\n",
    "    x, y = 2*np.random.rand()-1, 2*np.random.rand()-1\n",
    "    return 1 if x*x + y*y < 1 else 0\n",
    "\n",
    "NUM_SAMPLES = 100000\n",
    "\n",
    "count = sc.parallelize(xrange(0, NUM_SAMPLES)).map(sample).reduce(lambda a, b: a + b)\n",
    "\n",
    "print(\"Pi is roughly %f\" % (4.0 * count / NUM_SAMPLES))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Spark application: \n",
    "* a driver program that runs the user’s main function, that executes various parallel operations on a cluster. \n",
    "\n",
    "Main abstraction: \n",
    "* resilient distributed dataset (RDD), \n",
    "* a collection of elements partitioned across the nodes of the cluster that can be operated on in parallel. \n",
    "\n",
    "RDD: \n",
    "* Created from a file in the Hadoop file system (or any other Hadoop-supported file system)\n",
    "* Automatically recover from node failures.\n",
    "\n",
    "Cached: \n",
    "* Ask Spark to persist an RDD in memory\n",
    "\n",
    "Shared variables: \n",
    "* broadcast variables: cache a value in memory on all nodes, \n",
    "* accumulators: counters and sums.\n",
    "\n",
    "\n",
    "\n",
    "Spark’s Python API data formats (as of Nov 2016):\n",
    "\n",
    "* SparkContext.wholeTextFiles:\n",
    "read a directory containing of multiple small text files, return (filename, content) pairs. \n",
    "\n",
    "* RDD.saveAsPickleFile and SparkContext.pickleFile:\n",
    "save an RDD as pickled Python objects. \n",
    "\n",
    "* SequenceFile and Hadoop Input/Output Formats\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading Json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------------------+\n",
      "|   name|          properties|\n",
      "+-------+--------------------+\n",
      "|Product|[[Product identif...|\n",
      "+-------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "#    .config(\"spark.some.config.option\", \"some-value\") \\\n",
    "\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName(\"Python Spark SQL basic example\") \\\n",
    "    .getOrCreate()\n",
    "\n",
    "df = spark.read.json(\"data/products.json\")\n",
    "# Displays the content of the DataFrame to stdout\n",
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|          properties|\n",
      "+--------------------+\n",
      "|[[Product identif...|\n",
      "+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#df.printSchema()\n",
    "df.select(\"properties\").show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark (Spark 2.0.0)",
   "language": "python",
   "name": "pyspark"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}