project1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2016-09-23 06:58:21--  https://raw.githubusercontent.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/master/projects/project-01/women.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.32.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.32.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1053440 (1.0M) [text/plain]\n",
      "Saving to: ‘women.txt’\n",
      "\n",
      "women.txt           100%[===================>]   1.00M  5.07MB/s    in 0.2s    \n",
      "\n",
      "2016-09-23 06:58:21 (5.07 MB/s) - ‘women.txt’ saved [1053440/1053440]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://raw.githubusercontent.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/master/projects/project-01/women.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1347\r\n"
     ]
    }
   ],
   "source": [
    "!grep -wc Jo women.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "457\r\n"
     ]
    }
   ],
   "source": [
    "!grep -wc Beth women.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "683\r\n"
     ]
    }
   ],
   "source": [
    "!grep -wc Meg women.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "643\r\n"
     ]
    }
   ],
   "source": [
    "!grep -wc Amy women.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2016-09-17 22:44:36--  https://raw.githubusercontent.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/master/projects/project-01/romeo.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.32.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.32.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 178983 (175K) [text/plain]\n",
      "Saving to: ‘romeo.txt.6’\n",
      "\n",
      "romeo.txt.6         100%[===================>] 174.79K  --.-KB/s    in 0.02s   \n",
      "\n",
      "2016-09-17 22:44:36 (10.1 MB/s) - ‘romeo.txt.6’ saved [178983/178983]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://raw.githubusercontent.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/master/projects/project-01/romeo.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    117 Jul\r\n",
      "    163 Rom\r\n"
     ]
    }
   ],
   "source": [
    "!grep -oE '\\w{{2,}}' romeo.txt | grep -w \"Rom\\|Jul\" | sort |uniq -c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2016-09-17 22:44:38--  https://github.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/blob/master/projects/project-01/2016q1.csv.zip?raw=true\n",
      "Resolving github.com (github.com)... 192.30.253.113, 192.30.253.113\n",
      "Connecting to github.com (github.com)|192.30.253.113|:443... connected.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: https://github.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/raw/master/projects/project-01/2016q1.csv.zip [following]\n",
      "--2016-09-17 22:44:38--  https://github.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/raw/master/projects/project-01/2016q1.csv.zip\n",
      "Reusing existing connection to github.com:443.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: https://raw.githubusercontent.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/master/projects/project-01/2016q1.csv.zip [following]\n",
      "--2016-09-17 22:44:38--  https://raw.githubusercontent.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/master/projects/project-01/2016q1.csv.zip\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.32.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.32.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 10643003 (10M) [application/octet-stream]\n",
      "Saving to: ‘2016q1.csv.zip?raw=true.6’\n",
      "\n",
      "2016q1.csv.zip?raw= 100%[===================>]  10.15M  12.6MB/s    in 0.8s    \n",
      "\n",
      "2016-09-17 22:44:39 (12.6 MB/s) - ‘2016q1.csv.zip?raw=true.6’ saved [10643003/10643003]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://github.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/blob/master/projects/project-01/2016q1.csv.zip?raw=true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  2016q1.csv.zip?raw=true\n",
      "  inflating: 2016q1.csv              \n"
     ]
    }
   ],
   "source": [
    "!unzip 2016q1.csv.zip?raw=true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  13120 Columbus Circle / Union Station\r\n",
      "   9560 Massachusetts Ave & Dupont Circle NW\r\n",
      "   9388 Lincoln Memorial\r\n",
      "   8138 Jefferson Dr & 14th St SW\r\n",
      "   7479 Thomas Circle\r\n",
      "   7401 15th & P St NW\r\n",
      "   6568 14th & V St NW\r\n",
      "   6491 New Hampshire Ave & T St NW\r\n",
      "   5649 Eastern Market Metro / Pennsylvania Ave & 7th St SE\r\n",
      "   5514 17th & Corcoran St NW\r\n"
     ]
    }
   ],
   "source": [
    "!csvcut -c5 2016q1.csv |sort |uniq -c |sort -rn| head -10 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  13880 Columbus Circle / Union Station\r\n",
      "  11183 Massachusetts Ave & Dupont Circle NW\r\n",
      "   9419 Lincoln Memorial\r\n",
      "   8975 Jefferson Dr & 14th St SW\r\n",
      "   8092 15th & P St NW\r\n",
      "   7267 14th & V St NW\r\n",
      "   6997 Thomas Circle\r\n",
      "   6245 New Hampshire Ave & T St NW\r\n",
      "   5761 5th & K St NW\r\n",
      "   5651 17th & Corcoran St NW\r\n"
     ]
    }
   ],
   "source": [
    "!csvcut -c7 2016q1.csv |sort |uniq -c |sort -rn| head -10 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     17 W22227\r\n",
      "     16 W21867\r\n",
      "     16 W21641\r\n",
      "     16 W21538\r\n",
      "     16 W21239\r\n",
      "     16 W20540\r\n",
      "     16 W00714\r\n",
      "     15 W22080\r\n",
      "     15 W21450\r\n",
      "     15 W21076\r\n",
      "sort: write failed: 'standard output': Broken pipe\r\n",
      "sort: write error\r\n"
     ]
    }
   ],
   "source": [
    "!csvcut -c5,8 2016q1.csv | csvgrep -c1 -m 'Columbus Circle / Union Station' | csvcut -c2 | sort | uniq -c| sort -rn| head -10 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     18 W00485\r\n",
      "     17 W22227\r\n",
      "     16 W22099\r\n",
      "     16 W22080\r\n",
      "     16 W21239\r\n",
      "     16 W21076\r\n",
      "     16 W20425\r\n",
      "     16 W00714\r\n",
      "     15 W21997\r\n",
      "     15 W21867\r\n",
      "sort: write failed: 'standard output': Broken pipe\r\n",
      "sort: write error\r\n"
     ]
    }
   ],
   "source": [
    "!csvcut -c7,8 2016q1.csv | csvgrep -c1 -m 'Columbus Circle / Union Station' | csvcut -c2 | sort | uniq -c| sort -rn| head -10 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2016-09-23 06:53:37--  https://raw.githubusercontent.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/master/projects/project-01/simplefilter.py\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.32.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.32.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 208 [text/plain]\n",
      "Saving to: ‘simplefilter.py.1’\n",
      "\n",
      "simplefilter.py.1   100%[===================>]     208  --.-KB/s    in 0s      \n",
      "\n",
      "2016-09-23 06:53:37 (38.6 MB/s) - ‘simplefilter.py.1’ saved [208/208]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://raw.githubusercontent.com/gwsb-istm-6212-fall-2016/syllabus-and-schedule/master/projects/project-01/simplefilter.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   8155 and\r\n",
      "   7689 the\r\n",
      "   5152 to\r\n",
      "   3523 of\r\n",
      "   3245 her\r\n",
      "   2774 it\r\n",
      "   2503 in\r\n",
      "   2447 you\r\n",
      "   2343 she\r\n",
      "   2233 for\r\n",
      "sort: write failed: 'standard output': Broken pipe\r\n",
      "sort: write error\r\n"
     ]
    }
   ],
   "source": [
    "!cat women.txt | grep -oE '\\w{{2,}}' | tr '[:upper:]' '[:lower:]' | sort | uniq -c | sort -rn | head -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   8155 and\r\n",
      "   7689 the\r\n",
      "   5152 to\r\n",
      "   4531 a\r\n",
      "   4003 i\r\n",
      "   3523 of\r\n",
      "   3245 her\r\n",
      "   2774 it\r\n",
      "   2503 in\r\n",
      "   2447 you\r\n",
      "sort: write failed: 'standard output': Broken pipe\r\n",
      "sort: write error\r\n"
     ]
    }
   ],
   "source": [
    "!cat women.txt | python lower.py | python splitwords.py |sort| uniq -c | sort -rn | head -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   8155 and\r\n",
      "   7689 the\r\n",
      "   5152 to\r\n",
      "   4531 a\r\n",
      "   4003 i\r\n",
      "   3523 of\r\n",
      "   3245 her\r\n",
      "   2774 it\r\n",
      "   2503 in\r\n",
      "   2447 you\r\n",
      "   2343 she\r\n",
      "   2233 for\r\n",
      "   2033 was\r\n",
      "   1978 as\r\n",
      "   1936 that\r\n",
      "   1854 with\r\n",
      "   1680 t\r\n",
      "   1598 he\r\n",
      "   1495 s\r\n",
      "   1469 but\r\n",
      "   1362 jo\r\n",
      "   1135 so\r\n",
      "   1118 his\r\n",
      "   1067 at\r\n",
      "   1063 had\r\n",
      "sort: write failed: 'standard output': Broken pipe\r\n",
      "sort: write error\r\n"
     ]
    }
   ],
   "source": [
    "!cat women.txt | python stopwords.py | python lower.py | python splitwords.py |sort| uniq -c | sort -rn | head -25 #PartB #Python Stopwords converts both "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}