CapitolQuery_7 – Appendix B.1 – Text collection script.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collecting the Congressional Hearings\n",
    "\n",
    "### Before you start\n",
    "Make sure you have the following Python modules installed: <i>os</i>, <i>urllib</i> and <i>selenium</i>.<br>\n",
    "If you have not installed selenium, you will also need to install the <a href=\"https://stackoverflow.com/questions/40208051/selenium-using-python-geckodriver-executable-needs-to-be-in-path\">Geckodriver</a>.<br>\n",
    "\n",
    "### Basic program setup\n",
    "There are two basic steps in the setup:<br>\n",
    "(1) load the Python modules that will be used in the program. This program uses the Operating System module (\"os\") which allows the program to work on your hard drive, the urllib module (\"urllib\") to collect text from a webpage, and the Selenium module (\"selenium\") to mimic the user's click routine on the page.<br>\n",
    "(2) set the working directory to where the text files should be stored."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Created on Fri Oct 23 13:23:47 2015\n",
    "@author: Jeff and Johannes for CapitolQuery\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "import urllib\n",
    "from selenium import webdriver\n",
    "import re\n",
    "\n",
    "\n",
    "## SET your storing directory here \n",
    "os.chdir(\"your storing directory\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Set up the browser\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#to prevent download dialog\n",
    "profile = webdriver.FirefoxProfile()\n",
    "profile.set_preference('browser.download.folderList', 2) # custom location\n",
    "profile.set_preference('browser.download.manager.showWhenStarting', False)\n",
    "profile.set_preference('browser.download.dir', os.getcwd()) # use current working directory\n",
    "profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'text/csv')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start the browser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#open browser driver\n",
    "browser = webdriver.Firefox(profile)\n",
    "browser.implicitly_wait(20)\n",
    "browser.get(\"http://www.gpo.gov/fdsys/browse/collection.action?collectionCode=CHRG\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Loop through all sessions of Congress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#find the elements by congress\n",
    "congressElements = browser.find_elements_by_xpath(\".//div[@class='level1 browse-level']/a\")\n",
    "\n",
    "#loop through congresses\n",
    "counter = 0\n",
    "for counter in range(counter, len(congressElements)):\n",
    "    \n",
    "    #re-establish the element locations    \n",
    "    congressElements = browser.find_elements_by_xpath(\".//div[@class='level1 browse-level']/a\")    \n",
    "    \n",
    "    #save the name of the congress   \n",
    "    congress_name = congressElements[counter].text    \n",
    "    \n",
    "    #click on the chosen congress\n",
    "    congressElements[counter].click()\n",
    "\n",
    "    #record chamber element locations   \n",
    "    chamberElements = browser.find_elements_by_xpath(\".//div[@class='level2 browse-level']/a\")\n",
    "    \n",
    "    #loop through chambers\n",
    "    chamber_counter = 0\n",
    "    for chamber_counter in range(chamber_counter, len(chamberElements)):\n",
    "        \n",
    "        #re-establish the chamber element locations        \n",
    "        chamberElements = browser.find_elements_by_xpath(\".//div[@class='level2 browse-level']/a\")\n",
    "        \n",
    "        #save the name of the chamber        \n",
    "        chamber_name = chamberElements[chamber_counter].text        \n",
    "        \n",
    "        #click the chosen chamber        \n",
    "        chamberElements[chamber_counter].click()\n",
    "        \n",
    "        #record committee element locations        \n",
    "        committeeElements = browser.find_elements_by_partial_link_text(\"Committee\")\n",
    "        \n",
    "        #it is picking up an erroneous committee name from a sidebar\n",
    "        committeeElements.pop(0)\n",
    "        \n",
    "          #loop through committees\n",
    "        committee_counter = 0\n",
    "        for committee_counter in range(committee_counter, len(committeeElements)):\n",
    "            committeeElements = browser.find_elements_by_partial_link_text(\"Committee\")\n",
    "            committeeElements.pop(0)\n",
    "            committee_name = committeeElements[committee_counter].text\n",
    "            committeeElements[committee_counter].click()\n",
    "            \n",
    "        \n",
    "        #record hearing element locations\n",
    "            hearingElements = browser.find_elements_by_xpath(\".//tr[td[span[@class='results-line2']]]\")\n",
    "            \n",
    "            #loop through hearings\n",
    "            hearing_counter = 0\n",
    "            for hearing_counter in range(hearing_counter, len(hearingElements)):\n",
    "                \n",
    "                #re-establish hearing element locations                \n",
    "                hearingElements = browser.find_elements_by_xpath(\".//tr[td[span[@class='results-line2']]]\")\n",
    "                \n",
    "                #navigate to the line with the hearing elements                \n",
    "                hearing_line =  hearingElements[hearing_counter]               \n",
    "                \n",
    "                #save the hearing date & clean it slightly to remove the dot at the end               \n",
    "                hearing_date = hearing_line.find_element_by_xpath(\".//span[@class='results-line2']\").text\n",
    "                hearing_date = re.sub('\\.', '', hearing_date)\n",
    "                \n",
    "                #Storing the hearing text\n",
    "                #find the \"Text\" part of the element and extract the link                \n",
    "                hearing_text_link = hearing_line.find_element_by_partial_link_text(\"Text\").get_attribute('href')\n",
    "                \n",
    "                #download the text file \n",
    "                file_name_elements = [congress_name, chamber_name, committee_name, hearing_date]\n",
    "                file_name = \"_\".join(file_name_elements)\n",
    "                file_name += \".txt\"\n",
    "                \n",
    "                # depricated: \n",
    "                # urllib.request.urlretrieve(\"http://www.gpo.gov/fdsys/pkg/CHRG-115hhrg24324/html/CHRG-115hhrg24324.htm\")\n",
    "                                \n",
    "                hrng=urllib.request.Request(hearing_text_link, headers={'User-Agent': \"Mozilla/56.0.1 (Windows 10; Win64; x64)\"})\n",
    "                con = urllib.request.urlopen( hrng )\n",
    "                text= con.read()\n",
    "                file = open(file_name,'wb')\n",
    "                file.write(text)\n",
    "                file.close()\n",
    "                \n",
    "                #Storing the hearing metadata (MODS)\n",
    "                #Getting the URL & going there\n",
    "                hearing_MODS_page_link = hearing_line.find_element_by_partial_link_text(\"More\").get_attribute('href')\n",
    "                browser.get(hearing_MODS_page_link)\n",
    "                             \n",
    "                #download the text file \n",
    "                file_name_elements = [congress_name, chamber_name, committee_name, hearing_date, 'MODS']\n",
    "                file_name = \"_\".join(file_name_elements)\n",
    "                file_name += \".xml\"\n",
    "                \n",
    "                hearing_MODS_file_link = browser.find_element_by_partial_link_text(\"MODS\").get_attribute('href')\n",
    "                hrng=urllib.request.Request(hearing_MODS_file_link, headers={'User-Agent': \"Mozilla/56.0.1 (Windows 10; Win64; x64)\"})\n",
    "                con = urllib.request.urlopen( hrng )\n",
    "                text= con.read()\n",
    "                file = open(file_name,'wb')\n",
    "                file.write(text)\n",
    "                file.close()\n",
    "                \n",
    "                browser.back()\n",
    "            \n",
    "            browser.back()\n",
    "       \n",
    "        browser.back()\n",
    "    \n",
    "    browser.back()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}