mbox_cluster.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# http://data-mining.philippe-fournier-viger.com/introduction-clustering-k-means-java-code/\n",
    "# http://brandonrose.org/clustering\n",
    "# http://scikit-learn.org/stable/auto_examples/text/document_clustering.html\n",
    "# https://www.codeproject.com/Articles/439890/Text-Documents-Clustering-using-K-Means-Algorithm\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import re\n",
    "\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn import feature_extraction\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 739 ms, sys: 7.41 ms, total: 747 ms\n",
      "Wall time: 749 ms\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('data/dbpedia-discussion-archive-dataset.csv', sep=\"\\t\")\n",
    "\n",
    "stemmer = PorterStemmer()\n",
    "stemmer.stem('DBpedia')\n",
    "\n",
    "def tokenize_and_stem(text):\n",
    "    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]\n",
    "    filtered_tokens = []\n",
    "    for token in tokens:\n",
    "        if re.search('[a-zA-Z]', token):\n",
    "            filtered_tokens.append(token)\n",
    "    stems = [stemmer.stem(term) for term in filtered_tokens]\n",
    "    return stems\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=200000, min_df=5, stop_words='english', use_idf=True, tokenizer=tokenize_and_stem, ngram_range=(1,3))\n",
    "%time tfidf_matrix = tfidf_vectorizer.fit_transform(df['subject'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u\"'s\",\n",
       " u'abstract',\n",
       " u'access',\n",
       " u'add',\n",
       " u'announc',\n",
       " u'announc openlink',\n",
       " u'announc openlink virtuoso',\n",
       " u'answer',\n",
       " u'api',\n",
       " u'arab',\n",
       " u'area',\n",
       " u'articl',\n",
       " u'avail',\n",
       " u'bad',\n",
       " u'base',\n",
       " u'basekb',\n",
       " u'best',\n",
       " u'broken',\n",
       " u'browser',\n",
       " u'bug',\n",
       " u'build',\n",
       " u'case',\n",
       " u'categori',\n",
       " u'chang',\n",
       " u'chapter',\n",
       " u'charact',\n",
       " u'citi',\n",
       " u'class',\n",
       " u'code',\n",
       " u'commun',\n",
       " u'concept',\n",
       " u'connect',\n",
       " u'contain',\n",
       " u'content',\n",
       " u'contribut',\n",
       " u'coordin',\n",
       " u'count',\n",
       " u'countri',\n",
       " u'creat',\n",
       " u'data',\n",
       " u'data dbpedia',\n",
       " u'data qualiti',\n",
       " u'data set',\n",
       " u'databas',\n",
       " u'dataset',\n",
       " u'date',\n",
       " u'dbpedia',\n",
       " u'dbpedia categori',\n",
       " u'dbpedia content',\n",
       " u'dbpedia data',\n",
       " u'dbpedia dataset',\n",
       " u'dbpedia dump',\n",
       " u'dbpedia endpoint',\n",
       " u'dbpedia extract',\n",
       " u'dbpedia extract framework',\n",
       " u'dbpedia freebas',\n",
       " u'dbpedia live',\n",
       " u'dbpedia local',\n",
       " u'dbpedia lookup',\n",
       " u'dbpedia lookup servic',\n",
       " u'dbpedia map',\n",
       " u'dbpedia ontolog',\n",
       " u'dbpedia queri',\n",
       " u'dbpedia releas',\n",
       " u'dbpedia resourc',\n",
       " u'dbpedia sparql',\n",
       " u'dbpedia sparql endpoint',\n",
       " u'dbpedia spotlight',\n",
       " u'dbpedia uri',\n",
       " u'dbpedia-owl',\n",
       " u'dbpedia.org',\n",
       " u'deal',\n",
       " u'descript',\n",
       " u'differ',\n",
       " u'disambigu',\n",
       " u'download',\n",
       " u'dump',\n",
       " u'edit',\n",
       " u'editor',\n",
       " u'editor right',\n",
       " u'encod',\n",
       " u'endpoint',\n",
       " u'english',\n",
       " u'entiti',\n",
       " u'equival',\n",
       " u'error',\n",
       " u'extract',\n",
       " u'extract framework',\n",
       " u'extract problem',\n",
       " u'extractor',\n",
       " u'facet',\n",
       " u'fact',\n",
       " u'feedback',\n",
       " u'fetch',\n",
       " u'file',\n",
       " u'foaf',\n",
       " u'format',\n",
       " u'framework',\n",
       " u'freebas',\n",
       " u'fw',\n",
       " u'gener',\n",
       " u'geo',\n",
       " u'german',\n",
       " u'gsoc',\n",
       " u'ha',\n",
       " u'help',\n",
       " u'host',\n",
       " u'http',\n",
       " u'imag',\n",
       " u'import',\n",
       " u'improv',\n",
       " u'includ',\n",
       " u'inconsist',\n",
       " u'incorrect',\n",
       " u'index',\n",
       " u'indonesian',\n",
       " u'infobox',\n",
       " u'inform',\n",
       " u'instal',\n",
       " u'instanc',\n",
       " u'interfac',\n",
       " u'intern',\n",
       " u'invalid',\n",
       " u'iri',\n",
       " u'issu',\n",
       " u'jena',\n",
       " u'json',\n",
       " u'key',\n",
       " u'keyword',\n",
       " u'label',\n",
       " u'languag',\n",
       " u'larg',\n",
       " u'licens',\n",
       " u'like',\n",
       " u'limit',\n",
       " u'link',\n",
       " u'link data',\n",
       " u'link dbpedia',\n",
       " u'list',\n",
       " u'live',\n",
       " u'live updat',\n",
       " u'load',\n",
       " u'load dbpedia',\n",
       " u'local',\n",
       " u'local dbpedia',\n",
       " u'lod',\n",
       " u'look',\n",
       " u'lookup',\n",
       " u'lookup servic',\n",
       " u'machin',\n",
       " u'mani',\n",
       " u'map',\n",
       " u'map statist',\n",
       " u'map wiki',\n",
       " u'mirror',\n",
       " u'miss',\n",
       " u\"n't\",\n",
       " u'namespac',\n",
       " u'need',\n",
       " u'new',\n",
       " u'number',\n",
       " u'object',\n",
       " u'ontolog',\n",
       " u'open',\n",
       " u'open-sourc',\n",
       " u'open-sourc edit',\n",
       " u'openlink',\n",
       " u'openlink virtuoso',\n",
       " u'order',\n",
       " u'owl',\n",
       " u'page',\n",
       " u'pagelink',\n",
       " u'pars',\n",
       " u'permiss',\n",
       " u'person',\n",
       " u'place',\n",
       " u'pleas',\n",
       " u'point',\n",
       " u'posit',\n",
       " u'predic',\n",
       " u'problem',\n",
       " u'process',\n",
       " u'project',\n",
       " u'properti',\n",
       " u'properti map',\n",
       " u'public',\n",
       " u'qualiti',\n",
       " u'queri',\n",
       " u'queri dbpedia',\n",
       " u'queri result',\n",
       " u'question',\n",
       " u'question dbpedia',\n",
       " u'rang',\n",
       " u'rdf',\n",
       " u'redirect',\n",
       " u'regard',\n",
       " u'relat',\n",
       " u'relationship',\n",
       " u'releas',\n",
       " u'request',\n",
       " u'resourc',\n",
       " u'result',\n",
       " u'retriev',\n",
       " u'return',\n",
       " u'return differ',\n",
       " u'right',\n",
       " u'run',\n",
       " u'samea',\n",
       " u'schema.org',\n",
       " u'script',\n",
       " u'search',\n",
       " u'semant',\n",
       " u'semant web',\n",
       " u'server',\n",
       " u'servic',\n",
       " u'set',\n",
       " u'set dbpedia',\n",
       " u'simpl',\n",
       " u'sko',\n",
       " u'sparql',\n",
       " u'sparql endpoint',\n",
       " u'sparql queri',\n",
       " u'sparql result',\n",
       " u'specif',\n",
       " u'spotlight',\n",
       " u'start',\n",
       " u'statist',\n",
       " u'subject',\n",
       " u'support',\n",
       " u'syntax',\n",
       " u'tabl',\n",
       " u'templat',\n",
       " u'test',\n",
       " u'text',\n",
       " u'thi',\n",
       " u'time',\n",
       " u'tripl',\n",
       " u'turtl',\n",
       " u'type',\n",
       " u'updat',\n",
       " u'uri',\n",
       " u'url',\n",
       " u'use',\n",
       " u'use dbpedia',\n",
       " u'user',\n",
       " u'valu',\n",
       " u'version',\n",
       " u'virtuoso',\n",
       " u'virtuoso open-sourc',\n",
       " u'virtuoso open-sourc edit',\n",
       " u'vs',\n",
       " u'vs.',\n",
       " u'wa',\n",
       " u'web',\n",
       " u'websit',\n",
       " u'whi',\n",
       " u'wiki',\n",
       " u'wikidata',\n",
       " u'wikipedia',\n",
       " u'wikipedia infobox',\n",
       " u'work',\n",
       " u'worldcomp',\n",
       " u'wrong',\n",
       " u'xml',\n",
       " u'yago']"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 's, utf-8, vs. is getting matched\n",
    "# need to decide if dbpedia-* should be filtered out or separated\n",
    "tfidf_vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# How to check convergence to local optimium\n",
    "\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "dist = 1 - cosine_similarity(tfidf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.1 s, sys: 12.1 ms, total: 2.11 s\n",
      "Wall time: 2.12 s\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "km = KMeans(n_clusters=100)\n",
    "\n",
    "%time km.fit(tfidf_matrix)\n",
    "clusters = km.labels_.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7     231\n",
       "12     37\n",
       "17     32\n",
       "9      29\n",
       "18     27\n",
       "6      26\n",
       "4      24\n",
       "13     23\n",
       "22     22\n",
       "21     22\n",
       "15     22\n",
       "58     21\n",
       "24     19\n",
       "51     19\n",
       "57     19\n",
       "92     19\n",
       "30     18\n",
       "8      18\n",
       "75     18\n",
       "82     17\n",
       "65     17\n",
       "48     16\n",
       "11     16\n",
       "71     15\n",
       "2      15\n",
       "39     15\n",
       "35     15\n",
       "20     14\n",
       "26     13\n",
       "98     13\n",
       "     ... \n",
       "55      8\n",
       "23      8\n",
       "78      7\n",
       "91      7\n",
       "3       7\n",
       "81      7\n",
       "79      7\n",
       "25      7\n",
       "42      7\n",
       "76      7\n",
       "29      7\n",
       "94      6\n",
       "1       6\n",
       "97      6\n",
       "86      6\n",
       "87      6\n",
       "60      6\n",
       "89      6\n",
       "96      6\n",
       "95      6\n",
       "59      5\n",
       "5       5\n",
       "77      5\n",
       "93      5\n",
       "61      5\n",
       "66      5\n",
       "33      5\n",
       "54      4\n",
       "99      4\n",
       "46      2\n",
       "Name: cluster, dtype: int64"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['cluster'] = clusters\n",
    "df['cluster'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = df.sort_values('cluster', ascending=1)\n",
    "df[['subject', 'cluster']].to_csv('data/dbpedia-discussion-archive-cluster-results.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}