french-nltk.ipynb

{
 "metadata": {
  "name": "NL Functions"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "IMPORTS AND FUNCTION DEFINITIONS (Natural Language Toolkit and Stanford POS Tagger)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Christopher M. Church\n",
      "#PhD Candidate, UC Berkeley, History\n",
      "#Social Science D-Lab, UC Berkeley"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#IMPORTS AND FUNCTION DEFINITIONS\n",
      "\n",
      "#NLTK\n",
      "\n",
      "#imports\n",
      "import nltk #import the natural language toolkit library\n",
      "from nltk.stem.snowball import FrenchStemmer #import the French stemming library\n",
      "from nltk.corpus import stopwords #import stopwords from nltk corpus\n",
      "import re #import the regular expressions library; will be used to strip punctuation\n",
      "from collections import Counter #allows for counting the number of occurences in a list\n",
      "\n",
      "import os #import os module\n",
      "root_path = \"D:\\\\PhD Research - UC Berkeley\\\\Research\\\\Pelee XML\" #define a working directory path\n",
      "os.chdir(root_path) #set the working directory path\n",
      "\n",
      "#reading in the raw text from the file\n",
      "def read_raw_file(path):\n",
      "    '''reads in raw text from a text file using the argument (path), which represents the path/to/file'''\n",
      "    f = open(path,\"r\") #open the file located at \"path\" as a file object (f) that is readonly\n",
      "    raw = f.read().decode('utf8') # read raw text into a variable (raw) after decoding it from utf8\n",
      "    f.close() #close the file now that it isn;t being used any longer\n",
      "    return raw\n",
      "\n",
      "def get_tokens(raw,encoding='utf8'):\n",
      "    '''get the nltk tokens from a text'''\n",
      "    tokens = nltk.word_tokenize(raw) #tokenize the raw UTF-8 text\n",
      "    return tokens\n",
      "\n",
      "def get_nltk_text(raw,encoding='utf8'):\n",
      "    '''create an nltk text using the passed argument (raw) after filtering out the commas'''\n",
      "    #turn the raw text into an nltk text object\n",
      "    no_commas = re.sub(r'[.|,|\\']',' ', raw) #filter out all the commas, periods, and appostrophes using regex\n",
      "    tokens = nltk.word_tokenize(no_commas) #generate a list of tokens from the raw text\n",
      "    text=nltk.Text(tokens,encoding) #create a nltk text from those tokens\n",
      "    return text\n",
      "\n",
      "def get_stopswords(type=\"veronis\"):\n",
      "    '''returns the veronis stopwords in unicode, or if any other value is passed, it returns the default nltk french stopwords'''\n",
      "    if type==\"veronis\":\n",
      "        #VERONIS STOPWORDS\n",
      "        raw_stopword_list = [\"Ap.\", \"Apr.\", \"GHz\", \"MHz\", \"USD\", \"a\", \"afin\", \"ah\", \"ai\", \"aie\", \"aient\", \"aies\", \"ait\", \"alors\", \"apr\u00e8s\", \"as\", \"attendu\", \"au\", \"au-del\u00e0\", \"au-devant\", \"aucun\", \"aucune\", \"audit\", \"aupr\u00e8s\", \"auquel\", \"aura\", \"aurai\", \"auraient\", \"aurais\", \"aurait\", \"auras\", \"aurez\", \"auriez\", \"aurions\", \"aurons\", \"auront\", \"aussi\", \"autour\", \"autre\", \"autres\", \"autrui\", \"aux\", \"auxdites\", \"auxdits\", \"auxquelles\", \"auxquels\", \"avaient\", \"avais\", \"avait\", \"avant\", \"avec\", \"avez\", \"aviez\", \"avions\", \"avons\", \"ayant\", \"ayez\", \"ayons\", \"b\", \"bah\", \"banco\", \"ben\", \"bien\", \"b\u00e9\", \"c\", \"c'\", \"c'est\", \"c'\u00e9tait\", \"car\", \"ce\", \"ceci\", \"cela\", \"celle\", \"celle-ci\", \"celle-l\u00e0\", \"celles\", \"celles-ci\", \"celles-l\u00e0\", \"celui\", \"celui-ci\", \"celui-l\u00e0\", \"cel\u00e0\", \"cent\", \"cents\", \"cependant\", \"certain\", \"certaine\", \"certaines\", \"certains\", \"ces\", \"cet\", \"cette\", \"ceux\", \"ceux-ci\", \"ceux-l\u00e0\", \"cf.\", \"cg\", \"cgr\", \"chacun\", \"chacune\", \"chaque\", \"chez\", \"ci\", \"cinq\", \"cinquante\", \"cinquante-cinq\", \"cinquante-deux\", \"cinquante-et-un\", \"cinquante-huit\", \"cinquante-neuf\", \"cinquante-quatre\", \"cinquante-sept\", \"cinquante-six\", \"cinquante-trois\", \"cl\", \"cm\", \"cm\u00b2\", \"comme\", \"contre\", \"d\", \"d'\", \"d'apr\u00e8s\", \"d'un\", \"d'une\", \"dans\", \"de\", \"depuis\", \"derri\u00e8re\", \"des\", \"desdites\", \"desdits\", \"desquelles\", \"desquels\", \"deux\", \"devant\", \"devers\", \"dg\", \"diff\u00e9rentes\", \"diff\u00e9rents\", \"divers\", \"diverses\", \"dix\", \"dix-huit\", \"dix-neuf\", \"dix-sept\", \"dl\", \"dm\", \"donc\", \"dont\", \"douze\", \"du\", \"dudit\", \"duquel\", \"durant\", \"d\u00e8s\", \"d\u00e9j\u00e0\", \"e\", \"eh\", \"elle\", \"elles\", \"en\", \"en-dehors\", \"encore\", \"enfin\", \"entre\", \"envers\", \"es\", \"est\", \"et\", \"eu\", \"eue\", \"eues\", \"euh\", \"eurent\", \"eus\", \"eusse\", \"eussent\", \"eusses\", \"eussiez\", \"eussions\", \"eut\", \"eux\", \"e\u00fbmes\", \"e\u00fbt\", \"e\u00fbtes\", \"f\", \"fait\", \"fi\", \"flac\", \"fors\", \"furent\", \"fus\", \"fusse\", \"fussent\", \"fusses\", \"fussiez\", \"fussions\", \"fut\", \"f\u00fbmes\", \"f\u00fbt\", \"f\u00fbtes\", \"g\", \"gr\", \"h\", \"ha\", \"han\", \"hein\", \"hem\", \"heu\", \"hg\", \"hl\", \"hm\", \"hm\u00b3\", \"hol\u00e0\", \"hop\", \"hormis\", \"hors\", \"huit\", \"hum\", \"h\u00e9\", \"i\", \"ici\", \"il\", \"ils\", \"j\", \"j'\", \"j'ai\", \"j'avais\", \"j'\u00e9tais\", \"jamais\", \"je\", \"jusqu'\", \"jusqu'au\", \"jusqu'aux\", \"jusqu'\u00e0\", \"jusque\", \"k\", \"kg\", \"km\", \"km\u00b2\", \"l\", \"l'\", \"l'autre\", \"l'on\", \"l'un\", \"l'une\", \"la\", \"laquelle\", \"le\", \"lequel\", \"les\", \"lesquelles\", \"lesquels\", \"leur\", \"leurs\", \"lez\", \"lors\", \"lorsqu'\", \"lorsque\", \"lui\", \"l\u00e8s\", \"m\", \"m'\", \"ma\", \"maint\", \"mainte\", \"maintes\", \"maints\", \"mais\", \"malgr\u00e9\", \"me\", \"mes\", \"mg\", \"mgr\", \"mil\", \"mille\", \"milliards\", \"millions\", \"ml\", \"mm\", \"mm\u00b2\", \"moi\", \"moins\", \"mon\", \"moyennant\", \"mt\", \"m\u00b2\", \"m\u00b3\", \"m\u00eame\", \"m\u00eames\", \"n\", \"n'avait\", \"n'y\", \"ne\", \"neuf\", \"ni\", \"non\", \"nonante\", \"nonobstant\", \"nos\", \"notre\", \"nous\", \"nul\", \"nulle\", \"n\u00ba\", \"n\u00e9anmoins\", \"o\", \"octante\", \"oh\", \"on\", \"ont\", \"onze\", \"or\", \"ou\", \"outre\", \"o\u00f9\", \"p\", \"par\", \"par-del\u00e0\", \"parbleu\", \"parce\", \"parmi\", \"pas\", \"pass\u00e9\", \"pendant\", \"personne\", \"peu\", \"plus\", \"plus_d'un\", \"plus_d'une\", \"plusieurs\", \"pour\", \"pourquoi\", \"pourtant\", \"pourvu\", \"pr\u00e8s\", \"puisqu'\", \"puisque\", \"q\", \"qu\", \"qu'\", \"qu'elle\", \"qu'elles\", \"qu'il\", \"qu'ils\", \"qu'on\", \"quand\", \"quant\", \"quarante\", \"quarante-cinq\", \"quarante-deux\", \"quarante-et-un\", \"quarante-huit\", \"quarante-neuf\", \"quarante-quatre\", \"quarante-sept\", \"quarante-six\", \"quarante-trois\", \"quatorze\", \"quatre\", \"quatre-vingt\", \"quatre-vingt-cinq\", \"quatre-vingt-deux\", \"quatre-vingt-dix\", \"quatre-vingt-dix-huit\", \"quatre-vingt-dix-neuf\", \"quatre-vingt-dix-sept\", \"quatre-vingt-douze\", \"quatre-vingt-huit\", \"quatre-vingt-neuf\", \"quatre-vingt-onze\", \"quatre-vingt-quatorze\", \"quatre-vingt-quatre\", \"quatre-vingt-quinze\", \"quatre-vingt-seize\", \"quatre-vingt-sept\", \"quatre-vingt-six\", \"quatre-vingt-treize\", \"quatre-vingt-trois\", \"quatre-vingt-un\", \"quatre-vingt-une\", \"quatre-vingts\", \"que\", \"quel\", \"quelle\", \"quelles\", \"quelqu'\", \"quelqu'un\", \"quelqu'une\", \"quelque\", \"quelques\", \"quelques-unes\", \"quelques-uns\", \"quels\", \"qui\", \"quiconque\", \"quinze\", \"quoi\", \"quoiqu'\", \"quoique\", \"r\", \"revoici\", \"revoil\u00e0\", \"rien\", \"s\", \"s'\", \"sa\", \"sans\", \"sauf\", \"se\", \"seize\", \"selon\", \"sept\", \"septante\", \"sera\", \"serai\", \"seraient\", \"serais\", \"serait\", \"seras\", \"serez\", \"seriez\", \"serions\", \"serons\", \"seront\", \"ses\", \"si\", \"sinon\", \"six\", \"soi\", \"soient\", \"sois\", \"soit\", \"soixante\", \"soixante-cinq\", \"soixante-deux\", \"soixante-dix\", \"soixante-dix-huit\", \"soixante-dix-neuf\", \"soixante-dix-sept\", \"soixante-douze\", \"soixante-et-onze\", \"soixante-et-un\", \"soixante-et-une\", \"soixante-huit\", \"soixante-neuf\", \"soixante-quatorze\", \"soixante-quatre\", \"soixante-quinze\", \"soixante-seize\", \"soixante-sept\", \"soixante-six\", \"soixante-treize\", \"soixante-trois\", \"sommes\", \"son\", \"sont\", \"sous\", \"soyez\", \"soyons\", \"suis\", \"suite\", \"sur\", \"sus\", \"t\", \"t'\", \"ta\", \"tacatac\", \"tandis\", \"te\", \"tel\", \"telle\", \"telles\", \"tels\", \"tes\", \"toi\", \"ton\", \"toujours\", \"tous\", \"tout\", \"toute\", \"toutefois\", \"toutes\", \"treize\", \"trente\", \"trente-cinq\", \"trente-deux\", \"trente-et-un\", \"trente-huit\", \"trente-neuf\", \"trente-quatre\", \"trente-sept\", \"trente-six\", \"trente-trois\", \"trois\", \"tr\u00e8s\", \"tu\", \"u\", \"un\", \"une\", \"unes\", \"uns\", \"v\", \"vers\", \"via\", \"vingt\", \"vingt-cinq\", \"vingt-deux\", \"vingt-huit\", \"vingt-neuf\", \"vingt-quatre\", \"vingt-sept\", \"vingt-six\", \"vingt-trois\", \"vis-\u00e0-vis\", \"voici\", \"voil\u00e0\", \"vos\", \"votre\", \"vous\", \"w\", \"x\", \"y\", \"z\", \"z\u00e9ro\", \"\u00e0\", \"\u00e7'\", \"\u00e7a\", \"\u00e8s\", \"\u00e9taient\", \"\u00e9tais\", \"\u00e9tait\", \"\u00e9tant\", \"\u00e9tiez\", \"\u00e9tions\", \"\u00e9t\u00e9\", \"\u00e9t\u00e9e\", \"\u00e9t\u00e9es\", \"\u00e9t\u00e9s\", \"\u00eates\", \"\u00eatre\", \"\u00f4\"]\n",
      "    else:\n",
      "        #get French stopwords from the nltk kit\n",
      "        raw_stopword_list = stopwords.words('french') #create a list of all French stopwords\n",
      "    stopword_list = [word.decode('utf8') for word in raw_stopword_list] #make to decode the French stopwords as unicode objects rather than ascii\n",
      "    return stopword_list\n",
      "    \n",
      "\n",
      "def filter_stopwords(text,stopword_list):\n",
      "    '''normalizes the words by turning them all lowercase and then filters out the stopwords'''\n",
      "    words=[w.lower() for w in text] #normalize the words in the text, making them all lowercase\n",
      "    #filtering stopwords\n",
      "    filtered_words = [] #declare an empty list to hold our filtered words\n",
      "    for word in words: #iterate over all words from the text\n",
      "        if word not in stopword_list and word.isalpha() and len(word) > 1: #only add words that are not in the French stopwords list, are alphabetic, and are more than 1 character\n",
      "            filtered_words.append(word) #add word to filter_words list if it meets the above conditions\n",
      "    filtered_words.sort() #sort filtered_words list\n",
      "    return filtered_words\n",
      "    \n",
      "def stem_words(words):\n",
      "    '''stems the word list using the French Stemmer'''\n",
      "    #stemming words\n",
      "    stemmed_words = [] #declare an empty list to hold our stemmed words\n",
      "    stemmer = FrenchStemmer() #create a stemmer object in the FrenchStemmer class\n",
      "    for word in words:\n",
      "        stemmed_word=stemmer.stem(word) #stem the word\n",
      "        stemmed_words.append(stemmed_word) #add it to our stemmed word list\n",
      "    stemmed_words.sort() #sort the stemmed_words\n",
      "    return stemmed_words\n",
      "   \n",
      "def sort_dictionary(dictionary):\n",
      "    '''returns a sorted dictionary (as tuples) based on the value of each key'''\n",
      "    return sorted(dictionary.items(), key=lambda x: x[1], reverse=True)\n",
      "\n",
      "def normalize_counts(counts):\n",
      "    total = sum(counts.values())\n",
      "    return dict((word, float(count)/total) for word,count in counts.items())\n",
      "        \n",
      "def print_sorted_dictionary(tuple_dict):\n",
      "    '''print the results of sort_dictionary'''\n",
      "    for tup in tuple_dict:\n",
      "        print unicode(tup[1])[0:10] + '\\t\\t' + unicode(tup[0])\n",
      "        \n",
      "def print_words(words):\n",
      "    '''clean print the unicode words'''\n",
      "    for word in words:\n",
      "        print word\n",
      "        \n",
      "#USING STANFORD'S FRENCH POS TAGGER, v.3.2\n",
      "#http://nlp.stanford.edu/software/tagger.shtml\n",
      "#note: to get NLTK to find java with the tagger, I had to comment out lines 59 and 85 [config_java(options=self.java_options, verbose=False)] in stanford.py [C:\\Anaconda\\Lib\\site-packages\\nltk\\tag\\stanford.py]\n",
      "#then I had to set the python path directly\n",
      "        \n",
      "import nltk #import the Natural Language Processing Kit\n",
      "from nltk.tag.stanford import POSTagger #Get the Part of Speech tagger from NLP at Stanford, python module that interacts with Java\n",
      "nltk.internals.config_java(\"C:/Program Files/Java/jdk1.7.0_21/bin/java.exe\", options='-mx1000m',verbose=False) #set the path to java (note: i had to edit stanford.py and comment conflicting settings on lines 59 and 85\n",
      "\n",
      "tag_abbreviations = {\n",
      "                    'A': 'adjective',\n",
      "                    'Adv': 'adverb',\n",
      "                    'CC': 'coordinating conjunction',\n",
      "                    'Cl': 'weak clitic pronoun',\n",
      "                    'CS': 'subordinating conjunction',\n",
      "                    'D': 'determiner',\n",
      "                    'ET': 'foreign word',\n",
      "                    'I': 'interjection',\n",
      "                    'NC': 'common noun',\n",
      "                    'NP': 'proper noun',\n",
      "                    'P': 'preposition',\n",
      "                    'PREF': 'prefix',\n",
      "                    'PRO': 'strong pronoun',\n",
      "                    'V': 'verb',\n",
      "                    'PONCT': 'punctuation mark',\n",
      "                    'N': 'noun'}\n",
      "\n",
      "def pos_tag(to_tag,model_path = root_path + \"\\\\stanford-postagger-full-2013-06-20\\\\models\\\\french.tagger\",jar_path = root_path + \"\\\\stanford-postagger-full-2013-06-20\\\\stanford-postagger.jar\"):\n",
      "    '''tag the tokens with part of speech; to_tag is the tags; model_path is the file path to the stanford POS tagger model; and jar_path to the Stanford POS tagger jar file'''\n",
      "    pos_tagger = POSTagger(model_path,jar_path,encoding='utf8') #create an object of class POSTagger that is encoded in UTF-8\n",
      "    tags = pos_tagger.tag(to_tag) #run the tagging algorithm on the tokenized raw text\n",
      "    return tags\n",
      "\n",
      "def print_pos_tags(tags):\n",
      "    '''print all the tags with their part of speech; tag[0] is the word; tag[1] is the Part of Speech'''\n",
      "    for tag in tags: print tag[1]+'\\t',tag[0] \n",
      "            \n",
      "def get_pos_tags(tags,pos='ANY'):\n",
      "    '''get all the tags with their part of speech; tag[0] is the word; tag[1] is the Part of Speech'''\n",
      "    pos=pos.upper()\n",
      "    get_tags = []\n",
      "    if pos=='ANY':\n",
      "        print 'Please specify a tag to get' \n",
      "    else:\n",
      "        tag_abbreviations_upper = {k.upper():v for k,v in tag_abbreviations.items()}\n",
      "        if pos in tag_abbreviations_upper:\n",
      "            for tag in tags: \n",
      "                if tag[1].upper()==pos: get_tags.append(tag[0])\n",
      "        else:\n",
      "            print \"%s is not a valid search term.\" %(pos)\n",
      "    return get_tags\n",
      "            \n",
      "def search_pos(tags,search_term,pos):\n",
      "    '''look for a particular POS word prior to the search term, see what comes after the search term'''\n",
      "    print \"POS\\tPREC\\t\\tS.TERM\\t\\tSUC\\n\"\n",
      "    for i,tag in enumerate(tags):\n",
      "        if tags[i-1][1].upper()==pos.upper() and tag[0].lower()==search_term.lower():\n",
      "            print str(i)+'\\t'+tags[i-1][0]+\"\\t\" + tag[0] + \"\\t\" + tags[i+1][0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "LOAD THE XML FILES"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#this python file goes through the xml doccuments and prints out the text data for a particular element name\n",
      "\n",
      "#included libraries\n",
      "import os #allows for path crawling\n",
      "from xml.dom import minidom #understands XML DOMs\n",
      "\n",
      "#variables\n",
      "xml_path= root_path + \"\\\\xmls\" #set the path that we are going to read through\n",
      "elem_name = \"para\" #name of the dom element to look for\n",
      "xmls=[] #open up an empty array into which we will store our XML filenames\n",
      "\n",
      "#get filenames\n",
      "for root,dirs,files in os.walk(xml_path): #walk through the filepath and look for xml files, storing them in xmls array\n",
      "    for file in files:\n",
      "        if file.endswith('.xml'):\n",
      "            xmls.append(file)\n",
      "print \"Files Loaded: \",xmls\n",
      "\n",
      "\n",
      "documents=[] #initialize an empty documents array\n",
      "all_documents={'tokens':[],'raw':''} #initialize an empty all_documents list\n",
      "\n",
      "#this turns our list of documents read in from the xml files into a list of nltk documents\n",
      "#each document has an index (ex. documents[0]), and within each document is a dictionary with the items: newspaper, date, raw,tokens, and \n",
      "for xml in xmls: # go through each xml document in our xmls array\n",
      "    xmldoc = minidom.parse(xml_path + \"\\\\\" + xml) #parse the XML doc\n",
      "    itemlist = xmldoc.getElementsByTagName(elem_name) #get all paragraph (para) elements\n",
      "    newspaper = xmldoc.getElementsByTagName('newspaper') #get newspaper element\n",
      "    newspaper_name= newspaper[0].attributes['name'].value #set the newspaper_name to the name attribute of the newspaper element\n",
      "    date = newspaper[0].attributes['date'].value #set the newspaper date to the date attribute of the newspaper element\n",
      "    raw = '' #initialize the raw variable\n",
      "    for item in itemlist:\n",
      "        raw += ' '.join(t.nodeValue for t in item.childNodes if t.nodeType == t.TEXT_NODE) # add text from the node's data to the variable raw if the node's data is text\n",
      "    raw = re.sub(r'\\s+', ' ',raw) #remove the excess whitespace from the raw text\n",
      "    tokens = get_tokens(raw) #get the tokens from the text\n",
      "    text = get_nltk_text(raw) #create a nltk text from the xml document's raw text\n",
      "    documents.append({'newspaper':newspaper_name,'date':date,'raw':raw,'tokens':tokens,'text':text}) #add all our elements to the array (documents); each element in the array is a dictionary\n",
      "    all_documents['tokens'].extend(tokens)\n",
      "    all_documents['raw']+=raw\n",
      "    \n",
      "documents = sorted(documents, key=lambda doc: doc['date'])#sort the array according to a document's date\n",
      "#to sort by paper name then date, use documents = sorted(documents, key=lambda doc: (doc['newspaper'],doc['date']))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Set variables common to all functions in analysis\n",
      "french_stopwords = get_stopswords()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "ANALYSIS CODE"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#print a sorted dictionary of all stemmed and filtered words for each text\n",
      "for document in documents:\n",
      "    filtered_words = filter_stopwords(document['text'],french_stopwords)\n",
      "    print '\\n',document['newspaper'],'\\t',document['date'],'\\n---------------'\n",
      "    print_sorted_dictionary(sort_dictionary(Counter(stem_words(filtered_words))))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#print a count of all adjectives filtered for stopwords and stemmed for each text as well as a count of adjectives for all texts\n",
      "def stemmed_adjectives(tokens):\n",
      "     tags = pos_tag(tokens)\n",
      "     adjectives = get_pos_tags(tags,'a') #get all adjectives from the text\n",
      "     adj_text = nltk.Text(adjectives,'utf8') #create an nltk text from those adjectives\n",
      "     filtered_adjectives = filter_stopwords(adj_text,french_stopwords) #filter out stopwwords from the adjectives\n",
      "     stemmed_adjectives = stem_words(filtered_adjectives) #stem the adjectives to normalize them\n",
      "     adjective_count = sort_dictionary(Counter(stemmed_adjectives)) #count and sort the adjectives\n",
      "     print_sorted_dictionary(adjective_count) #print the adjectives\n",
      "        \n",
      "for document in documents:\n",
      "     print '\\n',document['newspaper'],' - ',document['date'],'\\n-----------------------------'\n",
      "     print stemmed_adjectives(document['tokens'])\n",
      "print 'ALL TEXT\\n----------\\n', stemmed_adjectives(all_documents['tokens'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "tags=pos_tag(all_documents['tokens'])\n",
      "nouns=get_pos_tags(tags,'n') #get all the nouns from the pos tagger\n",
      "filtered_nouns = filter_stopwords(nouns,french_stopwords) #filter the veronis stopwords out of the noun tags\n",
      "noun_count = Counter(filtered_nouns) #count the filtered nouns\n",
      "noun_count = normalize_counts(noun_count) #normalize the counts by how many words are in the articles related to Martinique\n",
      "noun_count = sort_dictionary(noun_count) #sort the dictionary into a sorted tuple list\n",
      "print_sorted_dictionary(noun_count) #print the sorted tuples "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "search_pos(tags,'catastrophe','a')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sorted_stemmed_words = sort_dictionary(stemmed_word_count)\n",
      "print_sorted_dictionary(sorted_stemmed_words)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print Counter([tag[1] for tag in tags]) #count the parts of speech"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for document in documents: print document['newspaper'],': ',document['date'],'\\n\\n',document['raw'],'\\n\\n*******\\n\\n' #print out all the paragraphs from each xml file"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# print the concordance of a term for all of our nltk documents that had been read in from the xml files\n",
      "for document in documents:\n",
      "    print \"DATE: %s \\t NEWSPAPER: %s\" %(document['date'],document['newspaper'])\n",
      "    print '------'\n",
      "    print document['text'].concordance('Martinique')\n",
      "    print '\\n'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for document in documents: print \"NEWSPAPER: %s \\t DATE: %s \\t WORD COUNT: %s \\t CHARACTER COUNT: %s\" %(document['newspaper'],document['date'],len(document['tokens']),len(document['raw']))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "NEWSPAPER: Le Temps \t DATE: 1902-05-09 \t WORD COUNT: 209 \t CHARACTER COUNT: 1232\n",
        "NEWSPAPER: Le Temps \t DATE: 1902-05-10 \t WORD COUNT: 3266 \t CHARACTER COUNT: 18512\n",
        "NEWSPAPER: Le Temps \t DATE: 1902-05-11 \t WORD COUNT: 6021 \t CHARACTER COUNT: 34523\n",
        "NEWSPAPER: Le Temps \t DATE: 1902-05-12 \t WORD COUNT: 5873 \t CHARACTER COUNT: 33704\n",
        "NEWSPAPER: Le Temps \t DATE: 1902-05-13 \t WORD COUNT: 5081 \t CHARACTER COUNT: 29378\n",
        "NEWSPAPER: Le Temps \t DATE: 1902-05-13 \t WORD COUNT: 5907 \t CHARACTER COUNT: 33750\n"
       ]
      }
     ],
     "prompt_number": 109
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}