sequence_labelling_codeonly.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read the data in\n",
      "Generate features\n",
      "Generate feature vectors\n",
      "Train the classifier (takes a while for C=0.5 and higher)\n",
      "[LibLinear]Evaluate\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9299287410926366"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import namedtuple\n",
    "import sklearn.svm\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "\n",
    "\n",
    "#Same as tuple but the fields are named for convenience\n",
    "#this says we have four fields\n",
    "OneWord=namedtuple(\"OneWord\",[\"word\",\"pos_label\",\"chunk_label\",\"entity_label\"])\n",
    "\n",
    "def read_conll2003(f_name):\n",
    "    \"\"\"Yield complete sentences\"\"\"\n",
    "    current_sentence=[] #This will be a list of (word,label), which we accumulate for each sentence\n",
    "    with open(f_name) as f:\n",
    "        for line in f:\n",
    "            line=line.strip() #drop whitespace\n",
    "            if line.startswith(\"-DOCSTART-\"): #let's not worry about these for the time being\n",
    "                continue\n",
    "            if not line: #sentence break\n",
    "                if current_sentence: #if we gathered a sentence, we should yield it, because a new starts\n",
    "                    yield current_sentence #much like return, but continues past this line once the element has been consumed\n",
    "                    current_sentence=[] #...and start a new one\n",
    "                continue\n",
    "            #if we made it here, we are on a normal line\n",
    "            columns=line.split() #an actual word line\n",
    "            assert len(columns)==4 #we should have four columns, looking at the data\n",
    "            current_sentence.append(OneWord(*columns))\n",
    "        else: #for ... else -> the else part is executed once, when \"for\" runs out of elements\n",
    "            if current_sentence: #yield also the last one!\n",
    "                yield current_sentence\n",
    "                \n",
    "def generate_sentence_features(sent):\n",
    "    #Given a sentence as a list of (word, label) pairs\n",
    "    #generate the features for every word\n",
    "    #The result should be a list of same length as the sentence\n",
    "    #Each item is a dictionary of {\"feature name\"->feature value} mappings, holding all features of the word at that position\n",
    "    \n",
    "    sent_features=[] #this will be the result\n",
    "    for word_idx, one_word in enumerate(sent):\n",
    "        #We do nothing with label\n",
    "        #it just happens to be around\n",
    "        word_features={}\n",
    "        word_features[\"word_\"+one_word.word]=1 #the word itself is a feature\n",
    "        if word_idx!=0:\n",
    "            word_features[\"left_word_\"+sent[word_idx-1].word]=1\n",
    "        if word_idx!=len(sent)-1:\n",
    "            word_features[\"right_word_\"+sent[word_idx+1].word]=1\n",
    "        sent_features.append(word_features)\n",
    "    return sent_features\n",
    "\n",
    "def prep_data(sentences):\n",
    "    all_labels=[] #here we gather labels for all words in all sentences\n",
    "    all_features=[] #here we gather features for all words in all sentences\n",
    "    for sentence in sentences:\n",
    "        sent_features=generate_sentence_features(sentence)\n",
    "        assert len(sent_features)==len(sentence)\n",
    "        #Now we can get, for every position its label and its features\n",
    "        for one_word,features in zip(sentence,sent_features):\n",
    "            all_labels.append(one_word.pos_label) #label\n",
    "            all_features.append(features)         #and features to go with it\n",
    "    return all_labels, all_features\n",
    "\n",
    "\n",
    "#1) Read data in\n",
    "print(\"Read the data in\")\n",
    "sentences_train=list(read_conll2003(\"CONLL2003/train.txt\"))\n",
    "sentences_dev=list(read_conll2003(\"CONLL2003/valid.txt\"))\n",
    "\n",
    "#2) Generate the features and labels\n",
    "print(\"Generate features\")\n",
    "train_labels,train_features=prep_data(sentences_train)\n",
    "dev_labels,dev_features=prep_data(sentences_dev)\n",
    "\n",
    "#3) Prepare the feature vectors for the classifier\n",
    "print(\"Generate feature vectors\")\n",
    "vectorizer=DictVectorizer()\n",
    "vectorizer.fit(train_features)\n",
    "feature_vectors_train=vectorizer.transform(train_features)\n",
    "feature_vectors_dev=vectorizer.transform(dev_features)\n",
    "\n",
    "#4) Train the classifier\n",
    "print(\"Train the classifier (takes a while for C=0.5 and higher)\")\n",
    "classifier=sklearn.svm.LinearSVC(C=0.5,verbose=1)\n",
    "classifier.fit(feature_vectors_train, train_labels)\n",
    "\n",
    "#5) Evaluate\n",
    "print(\"Evaluate\")\n",
    "classifier.score(feature_vectors_dev,dev_labels)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}