3_ad_similarity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import psycopg2\n",
    "from sqlalchemy import create_engine\n",
    "import pandas as pd\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from pprint import pprint\n",
    "import networkx as nx\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# change default plot sizes\n",
    "pylab.rcParams['figure.figsize'] = (12, 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create a connection to the database\n",
    "conn = psycopg2.connect(\"\"\"dbname='postgres' \n",
    "                           user='postgres' \n",
    "                           password='apassword' \n",
    "                           host='localhost' \n",
    "                           port='5432'\n",
    "                        \"\"\")\n",
    "\n",
    "# connect to the database\n",
    "engine = create_engine('postgresql+psycopg2://postgres:apassword@localhost/postgres')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# store the results of the query in a dataframe\n",
    "df = pd.read_sql_query(\"\"\"SELECT b.id, b.post_date, b.post_title, b.post_body, b.city, b.state, c.phone\n",
    "                          FROM backpage_adphone as a\n",
    "                          INNER JOIN backpage as b ON a.ad_id = b.id\n",
    "                          INNER JOIN backpage_phone as c ON a.phone_id = c.id\n",
    "                       \"\"\", con=engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# subset the dataframe to only include ads posted after September 19th\n",
    "df = df[df.post_date.dt.date >= datetime.date(2017,9,19)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>post_date</th>\n",
       "      <th>post_title</th>\n",
       "      <th>post_body</th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>phone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2017-09-27 09:28:00</td>\n",
       "      <td>100%Call us: 318-704-6000 100% NEWYOUNG ASIAN ...</td>\n",
       "      <td>HELLO GENTLMEN WELCOME TO Jackson Street Bodyw...</td>\n",
       "      <td>Alexandria</td>\n",
       "      <td>Louisiana</td>\n",
       "      <td>3187046000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>2017-09-26 11:12:00</td>\n",
       "      <td>~  318-321-8820 New Oriental Massage Treatment...</td>\n",
       "      <td>ADD:3402 Jackson St,Ste D,Alexandria,La,71301 ...</td>\n",
       "      <td>Alexandria</td>\n",
       "      <td>Louisiana</td>\n",
       "      <td>3183218820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>2017-09-24 19:33:00</td>\n",
       "      <td>TheUltimateExperience - 33</td>\n",
       "      <td>If You Want SomethingSweet,I Can Satisfy Your ...</td>\n",
       "      <td>Alexandria</td>\n",
       "      <td>Louisiana</td>\n",
       "      <td>3372571723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>2017-09-24 12:53:00</td>\n",
       "      <td>Exclusive Blonde - 30</td>\n",
       "      <td>Hi,Guys I'm Amber Give me a call 337-654-7053</td>\n",
       "      <td>Alexandria</td>\n",
       "      <td>Louisiana</td>\n",
       "      <td>3376547053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>2017-09-20 10:55:00</td>\n",
       "      <td>The girl next door - 24</td>\n",
       "      <td>Hey, I'm just looking to have a good time. My ...</td>\n",
       "      <td>Alexandria</td>\n",
       "      <td>Louisiana</td>\n",
       "      <td>3184160462</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id           post_date                                         post_title  \\\n",
       "0   1 2017-09-27 09:28:00  100%Call us: 318-704-6000 100% NEWYOUNG ASIAN ...   \n",
       "1   3 2017-09-26 11:12:00  ~  318-321-8820 New Oriental Massage Treatment...   \n",
       "2   4 2017-09-24 19:33:00                         TheUltimateExperience - 33   \n",
       "3   5 2017-09-24 12:53:00                              Exclusive Blonde - 30   \n",
       "4   7 2017-09-20 10:55:00                            The girl next door - 24   \n",
       "\n",
       "                                           post_body        city      state  \\\n",
       "0  HELLO GENTLMEN WELCOME TO Jackson Street Bodyw...  Alexandria  Louisiana   \n",
       "1  ADD:3402 Jackson St,Ste D,Alexandria,La,71301 ...  Alexandria  Louisiana   \n",
       "2  If You Want SomethingSweet,I Can Satisfy Your ...  Alexandria  Louisiana   \n",
       "3      Hi,Guys I'm Amber Give me a call 337-654-7053  Alexandria  Louisiana   \n",
       "4  Hey, I'm just looking to have a good time. My ...  Alexandria  Louisiana   \n",
       "\n",
       "        phone  \n",
       "0  3187046000  \n",
       "1  3183218820  \n",
       "2  3372571723  \n",
       "3  3376547053  \n",
       "4  3184160462  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show a sample of the data\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# store the data into a list-of-dicts\n",
    "data = df.to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'city': 'Alexandria',\n",
      " 'id': 1,\n",
      " 'phone': '3187046000',\n",
      " 'post_body': 'HELLO GENTLMEN WELCOME TO Jackson Street Bodywork...TAKE A '\n",
      "              'SHORT TRIP TO ASIAN NEW GRAND OPENING...!New Face, New Feeling '\n",
      "              'and Different Experience One of the best bodyrub in town  100%  '\n",
      "              'Young Asian Girls !  Sweet soft hands give you Dreamly Heavenly '\n",
      "              'feeling!  luxury ROOMS Completely relieve all your stress away  '\n",
      "              'I HAVE BEEN TOLD I AM SUPER SKILLFUL.. CALL NOW: '\n",
      "              '318-704-6000Location: 5401 C Jackson St Alexandria, LA, 71303   '\n",
      "              'OPEN 9:00 am-10:00 pm OPEN 7DAYS',\n",
      " 'post_date': Timestamp('2017-09-27 09:28:00'),\n",
      " 'post_title': '100%Call us: 318-704-6000 100% NEWYOUNG ASIAN - 24',\n",
      " 'state': 'Louisiana'}\n"
     ]
    }
   ],
   "source": [
    "# show a sample of the dict\n",
    "pprint(data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define punctuation to remove punctuation\n",
    "exclude = set(string.punctuation)\n",
    "\n",
    "# clean up the post body\n",
    "for line in data:\n",
    "    \n",
    "    # zap into lowercase\n",
    "    post_clean = line['post_body'].lower()\n",
    "    \n",
    "    # remove punctuation\n",
    "    post_clean = ''.join(ch for ch in post_clean if ch not in exclude)\n",
    "    \n",
    "    # split words up\n",
    "    wordlist = post_clean.split(' ')\n",
    "    \n",
    "    # remove stop words\n",
    "    post_clean = [word for word in wordlist if word not in stopwords.words('english')]\n",
    "    \n",
    "    # remove words less than 3 characters\n",
    "    post_clean = [word for word in post_clean if len(word) > 3]\n",
    "    \n",
    "    # put words into a unique set\n",
    "    unique_words = set(post_clean)\n",
    "    \n",
    "    # add clean post to the data\n",
    "    line['unique_words'] = unique_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'city': 'Alexandria',\n",
      " 'id': 1,\n",
      " 'phone': '3187046000',\n",
      " 'post_body': 'HELLO GENTLMEN WELCOME TO Jackson Street Bodywork...TAKE A '\n",
      "              'SHORT TRIP TO ASIAN NEW GRAND OPENING...!New Face, New Feeling '\n",
      "              'and Different Experience One of the best bodyrub in town  100%  '\n",
      "              'Young Asian Girls !  Sweet soft hands give you Dreamly Heavenly '\n",
      "              'feeling!  luxury ROOMS Completely relieve all your stress away  '\n",
      "              'I HAVE BEEN TOLD I AM SUPER SKILLFUL.. CALL NOW: '\n",
      "              '318-704-6000Location: 5401 C Jackson St Alexandria, LA, 71303   '\n",
      "              'OPEN 9:00 am-10:00 pm OPEN 7DAYS',\n",
      " 'post_date': Timestamp('2017-09-27 09:28:00'),\n",
      " 'post_title': '100%Call us: 318-704-6000 100% NEWYOUNG ASIAN - 24',\n",
      " 'state': 'Louisiana',\n",
      " 'unique_words': {'3187046000location',\n",
      "                  '5401',\n",
      "                  '71303',\n",
      "                  '7days',\n",
      "                  'alexandria',\n",
      "                  'am1000',\n",
      "                  'asian',\n",
      "                  'away',\n",
      "                  'best',\n",
      "                  'bodyrub',\n",
      "                  'bodyworktake',\n",
      "                  'call',\n",
      "                  'completely',\n",
      "                  'different',\n",
      "                  'dreamly',\n",
      "                  'experience',\n",
      "                  'face',\n",
      "                  'feeling',\n",
      "                  'gentlmen',\n",
      "                  'girls',\n",
      "                  'give',\n",
      "                  'grand',\n",
      "                  'hands',\n",
      "                  'heavenly',\n",
      "                  'hello',\n",
      "                  'jackson',\n",
      "                  'luxury',\n",
      "                  'open',\n",
      "                  'openingnew',\n",
      "                  'relieve',\n",
      "                  'rooms',\n",
      "                  'short',\n",
      "                  'skillful',\n",
      "                  'soft',\n",
      "                  'street',\n",
      "                  'stress',\n",
      "                  'super',\n",
      "                  'sweet',\n",
      "                  'told',\n",
      "                  'town',\n",
      "                  'trip',\n",
      "                  'welcome',\n",
      "                  'young'}}\n"
     ]
    }
   ],
   "source": [
    "# show an example of the data\n",
    "pprint(data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def jaccard_index(first_set, second_set):\n",
    "\n",
    "    index = 1.0\n",
    "    if first_set or second_set:\n",
    "        index = (float(len(first_set.intersection(second_set))) \n",
    "             / len(first_set.union(second_set)))\n",
    "\n",
    "    return index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# compute jaccard similarity two each pair of ads\n",
    "similarity = []\n",
    "for x in data:\n",
    "    for y in data:\n",
    "        if x['id'] != y['id']:\n",
    "            row = {'id1': x['id'],\n",
    "                   'id1_post_body': x['post_body'],\n",
    "                   'id1_phone': x['phone'],\n",
    "                   'id1_post_date': x['post_date'],\n",
    "                   'id1_city': x['city'],\n",
    "                   'id2': y['id'],\n",
    "                   'id2_post_body': y['post_body'],\n",
    "                   'id2_phone': y['phone'],\n",
    "                   'id2_post_date': y['post_date'],\n",
    "                   'id2_city': y['city'],\n",
    "                   'jaccard': jaccard_index(x['unique_words'], y['unique_words'])\n",
    "                  }\n",
    "            similarity.append(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id1': 13,\n",
      " 'id1_city': 'New Orleans',\n",
      " 'id1_phone': '3312458056',\n",
      " 'id1_post_body': \"Hey! Welcome Everyone!!!NANA 23 5'2-34D-103#100% as young \"\n",
      "                  'as picture and much beautiful in personI have curve body '\n",
      "                  'and natural breast, open-minded, warm, friendly, and no '\n",
      "                  'rush.  Looking forward to meeting you. Text '\n",
      "                  'me:331-245-8056(SORRY, no african amarican)shower '\n",
      "                  'togetherlickingBJ69nurubody to body massageEverything is '\n",
      "                  'naked!!!',\n",
      " 'id1_post_date': Timestamp('2017-09-27 20:15:00'),\n",
      " 'id2': 1191,\n",
      " 'id2_city': 'New Orleans',\n",
      " 'id2_phone': '9176699832',\n",
      " 'id2_post_body': \"Hey! Welcome Everyone!!!NANA 21 5'2-34D-103#100% as young \"\n",
      "                  'as picture and much beautiful in personI have curve body '\n",
      "                  'and natural breast, open-minded, warm, friendly, and no '\n",
      "                  'rush.  Looking forward to meeting you.Call me: '\n",
      "                  '917-669-9832(SORRY, no african amarican)shower '\n",
      "                  'togetherlicking69nurubody to body massageEverything is '\n",
      "                  'naked!!!',\n",
      " 'id2_post_date': Timestamp('2017-10-09 11:26:00'),\n",
      " 'jaccard': 0.7931034482758621}\n"
     ]
    }
   ],
   "source": [
    "# show a sample of two ads and their similarity score\n",
    "pprint(similarity[43920])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id1': 24,\n",
      " 'id1_city': 'New Orleans',\n",
      " 'id1_phone': '2565680649',\n",
      " 'id1_post_body': 'My name is Diamond  I provide unforgettable services come '\n",
      "                  'spend some time with me and unwind or quick stop n popI '\n",
      "                  'have the skills to take you to a whole new and exciting '\n",
      "                  \"levelI DONT DISAPPOINT in any way I'm honest ,and upfront \"\n",
      "                  'nothing is misleadingneat , clean , no rush atmosphere . '\n",
      "                  'come see for yourself.Diamond (256)568-0649',\n",
      " 'id1_post_date': Timestamp('2017-09-27 18:42:00'),\n",
      " 'id2': 1291,\n",
      " 'id2_city': 'New Orleans',\n",
      " 'id2_phone': '3526197341',\n",
      " 'id2_post_body': \"Hey guys I'm Diamond  I provide unforgettable services come \"\n",
      "                  'spend some time with me and unwind or quick stop n popI '\n",
      "                  'have the skills to take you to a whole new and exciting '\n",
      "                  \"levelI DONT DISAPPOINT in any way I'm honest ,and upfront \"\n",
      "                  'nothing is misleadingneat , clean , no rush atmosphere . '\n",
      "                  'come see for yourself.Diamond (352)619-7341',\n",
      " 'id2_post_date': Timestamp('2017-10-01 16:12:00'),\n",
      " 'jaccard': 0.8666666666666667}\n"
     ]
    }
   ],
   "source": [
    "# show a sample of two ads and their similarity score\n",
    "pprint(similarity[102933])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id1': 109,\n",
      " 'id1_city': 'New Orleans',\n",
      " 'id1_phone': '5045217162',\n",
      " 'id1_post_body': 'Destiny your favorite upscale Chocolate beauty.I love being '\n",
      "                  'your perfect girl.No HassleNo RushIndependentI Have The '\n",
      "                  'Remedy For Every Mans Sweet ToothSmooth Thick Curvy '\n",
      "                  'BodySeductive Eyes That Are Very Alluring & Addictive !If '\n",
      "                  'You Desire Something SweetI Can Satisfy Your CravingUpscale '\n",
      "                  'Clean and Discreet 100% Safe100% REAL PICSNo Law '\n",
      "                  'EnforcementNo Pimps/ThugsNo Blocked CallsCall Destiny (504) '\n",
      "                  '521-7162',\n",
      " 'id1_post_date': Timestamp('2017-09-27 07:25:00'),\n",
      " 'id2': 1972,\n",
      " 'id2_city': 'Lake Charles',\n",
      " 'id2_phone': '8327419470',\n",
      " 'id2_post_body': 'Kim your favorite upscale latina/white beauty.I love being '\n",
      "                  'your perfect girl.No HassleNo RushIndependentI Have The '\n",
      "                  'Remedy For Every Mans Sweet ToothSmooth Thick Curvy '\n",
      "                  'BodySeductive Eyes That Are Very Alluring & Addictive !If '\n",
      "                  'You Desire Something SweetI Can Satisfy Your CravingUpscale '\n",
      "                  'Clean and Discreet 100% Safe100% REAL PICSNo Law '\n",
      "                  'EnforcementNo AA Under 35No Pimps/ThugsNo Blocked Calls     '\n",
      "                  'Call Kimberly 8327419470',\n",
      " 'id2_post_date': Timestamp('2017-10-11 19:52:00'),\n",
      " 'jaccard': 0.7619047619047619}\n"
     ]
    }
   ],
   "source": [
    "# show a sample of two ads and their similarity score\n",
    "pprint(similarity[542817])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load similarity into dataframe\n",
    "sim_df = pd.DataFrame(similarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# sort the dataframe\n",
    "sim_df.sort_values('jaccard', inplace=True, ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remove similarity scores equal to 0 or 1\n",
    "sim_df = sim_df[(sim_df['jaccard'] > 0) & (sim_df['jaccard'] < 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fea75b06eb8>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAHdCAYAAAAekb9wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFvtJREFUeJzt3VHM5XdZ4PHvs61u1CFg7O7EtGgbg7JdK7sygtmQdUaz\na4ubbUwwW2wgsJCGXXD3wk3olV5wg1Gzxgg2DWmIN0yyShSlimY3I26QLDYBSiWwDbDQakIAxR28\nIF1+ezEv5HVs+x7oec87M+fzSSaZc85v5v+0T6b55vQ/58xaKwAA2Hf/4KQHAACAK4EwBgCAhDEA\nAFTCGAAAKmEMAACVMAYAgOqEw3hmHpiZz87MRzY4+19n5oMHPz4+M3+9ixkBANgPc5KfYzwz/7K6\nWP3GWuv7v45f9zPVP19r/ftjGw4AgL1you8Yr7XeW33h8HMz8z0z8wcz89DM/MnMPP9JfunLq3fs\nZEgAAPbC9Sc9wJO4v3rdWut/z8yLq7dWP/rVF2fmu6tbqv9xQvMBAHANuqLCeGZOVf+i+m8z89Wn\n/+Flx+6qfnOt9f92ORsAANe2KyqMu3Rrx1+vtf7Z05y5q3r9juYBAGBPXFEf17bW+pvqkzPzU1Vz\nyQu++vrB/cbfXv3pCY0IAMA16qQ/ru0dXYrc75uZx2bmNdXd1Wtm5kPVI9Wdh37JXdX5dZIfpQEA\nwDXpyI9rm5kHqn9TffbJPlJtZu6u3lhN9X+r/7DW+tAxzAoAAMdmk3eM317d/jSvf7L6kbXWbdWb\nuvSpEgAAcFU58i/frbXeOzM3P83r7zv08P3VTc98LAAA2K1tfyrFa6rf3+TgDTfcsG6++eYtX/7J\nfelLX+rbvu3bdnItTp597x873y/2vV/se78c174feuihz621/tFR57YWxjNzrkth/JKnOXNPdU/V\n6dOn+6Vf+qVtXf5pXbx4sVOnTu3kWpw8+94/dr5f7Hu/2Pd+Oa59nzt37v9scm4rYTwzP1C9rbpj\nrfX5pzq31rq/g3uQz5w5s86ePbuNyx/pwoUL7epanDz73j92vl/se7/Y93456X0/449rm5nvqt5Z\nvWKt9fFnPhIAAOzeke8YH3zW8Nnqhpl5rPr56puq1lr3VT9XfUf11oOvcX5irXXmuAYGAIDjsMmn\nUrz8iNdfW712axMBAMAJuKK+EhoAAE6KMAYAgIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCg\nEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQFXXn/QAJ+Hme9990iPs3Kfe/BMnPQIA\nwBXNO8YAAJAwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAl\njAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAA\nVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgD\nAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiE\nMQAAVBuE8cw8MDOfnZmPPMXrMzO/OjOPzsyHZ+YHtz8mAAAcr03eMX57dfvTvH5H9byDH/dUv/7M\nxwIAgN06MozXWu+tvvA0R+6sfmNd8v7qOTPzndsaEAAAdmEb9xjfWH3m0OPHDp4DAICrxqy1jj40\nc3P1e2ut73+S136vevNa638ePP7v1RvXWn/2JGfv6dLtFp0+ffqF58+ff0bDb+rixYudOnXqa48f\nfvyLO7nuleS2G5990iPszOX75tpn5/vFvveLfe+X49r3uXPnHlprnTnq3PVbuNbj1XMPPb7p4Lm/\nZ611f3V/1ZkzZ9bZs2e3cPmjXbhwocPXetW9797Jda8kn7r77EmPsDOX75trn53vF/veL/a9X056\n39u4leJd1SsPPp3ih6svrrX+cgu/LwAA7MyR7xjPzDuqs9UNM/NY9fPVN1Wtte6rHqxeWj1a/W31\n6uMaFgAAjsuRYbzWevkRr6/q9VubCAAAToBvvgMAgIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWM\nAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABU\nwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMA\nQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQx\nAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBK\nGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFBtGMYz\nc/vMfGxmHp2Ze5/k9WfPzO/OzIdm5pGZefX2RwUAgONzZBjPzHXVW6o7qlurl8/MrZcde33152ut\nF1Rnq1+emW/e8qwAAHBsNnnH+EXVo2utT6y1vlydr+687MyqnjUzU52qvlA9sdVJAQDgGM1a6+kP\nzLysun2t9dqDx6+oXrzWesOhM8+q3lU9v3pW9e/WWu9+kt/rnuqeqtOnT7/w/Pnz2/rneFoXL17s\n1KlTX3v88ONf3Ml1ryS33fjskx5hZy7fN9c+O98v9r1f7Hu/HNe+z50799Ba68xR567f0vV+vPpg\n9aPV91R/NDN/stb6m8OH1lr3V/dXnTlzZp09e3ZLl396Fy5c6PC1XnXv32v2a96n7j570iPszOX7\n5tpn5/vFvveLfe+Xk973JrdSPF4999Djmw6eO+zV1TvXJY9Wn+zSu8cAAHBV2CSMP1A9b2ZuOfgL\ndXd16baJwz5d/VjVzJyuvq/6xDYHBQCA43TkrRRrrSdm5g3Ve6rrqgfWWo/MzOsOXr+velP19pl5\nuJrqjWutzx3j3AAAsFUb3WO81nqwevCy5+479PO/qP71dkcDAIDd8c13AACQMAYAgEoYAwBAJYwB\nAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTC\nGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBA\nJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEA\nAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoY\nAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACo\nhDEAAFQbhvHM3D4zH5uZR2fm3qc4c3ZmPjgzj8zMH293TAAAOF7XH3VgZq6r3lL9q+qx6gMz8661\n1p8fOvOc6q3V7WutT8/MPz6ugQEA4Dhs8o7xi6pH11qfWGt9uTpf3XnZmZ+u3rnW+nTVWuuz2x0T\nAACO1yZhfGP1mUOPHzt47rDvrb59Zi7MzEMz88ptDQgAALswa62nPzDzsi7dIvHag8evqF681nrD\noTO/Vp2pfqz6lupPq59Ya338st/rnuqeqtOnT7/w/PnzW/xHeWoXL17s1KlTX3v88ONf3Ml1ryS3\n3fjskx5hZy7fN9c+O98v9r1f7Hu/HNe+z50799Ba68xR5468x7h6vHruocc3HTx32GPV59daX6q+\nNDPvrV5Q/Z0wXmvdX91fdebMmXX27NkNLv/MXbhwocPXetW9797Jda8kn7r77EmPsDOX75trn53v\nF/veL/a9X05635vcSvGB6nkzc8vMfHN1V/Wuy878TvWSmbl+Zr61enH10e2OCgAAx+fId4zXWk/M\nzBuq91TXVQ+stR6ZmdcdvH7fWuujM/MH1Yerr1RvW2t95DgHBwCAbdrkVorWWg9WD1723H2XPf7F\n6he3NxoAAOyOb74DAICEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAA\nVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgD\nAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiE\nMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCA\nShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMA\nAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoNowjGfm9pn52Mw8OjP3Ps25H5qZJ2bmZdsb\nEQAAjt+RYTwz11Vvqe6obq1ePjO3PsW5X6j+cNtDAgDAcdvkHeMXVY+utT6x1vpydb6680nO/Uz1\nW9VntzgfAADsxCZhfGP1mUOPHzt47mtm5sbqJ6tf395oAACwO9dv6ff5leqNa62vzMxTHpqZe6p7\nqk6fPt2FCxe2dPmnd/Hixb9zrZ+97YmdXPdKsqt/11eCy/fNtc/O94t97xf73i8nve9Nwvjx6rmH\nHt908NxhZ6rzB1F8Q/XSmXlirfXbhw+tte6v7q86c+bMOnv27Dc49tfnwoULHb7Wq+59906ueyX5\n1N1nT3qEnbl831z77Hy/2Pd+se/9ctL73iSMP1A9b2Zu6VIQ31X99OEDa61bvvrzmXl79XuXRzEA\nAFzJjgzjtdYTM/OG6j3VddUDa61HZuZ1B6/fd8wzAgDAsdvoHuO11oPVg5c996RBvNZ61TMfCwAA\ndss33wEAQMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUw\nBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQ\nCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwA\nAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLG\nAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAq\nYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKg2DOOZuX1mPjYzj87MvU/y+t0z8+GZeXhm3jcz\nL9j+qAAAcHyODOOZua56S3VHdWv18pm59bJjn6x+ZK11W/Wm6v5tDwoAAMdpk3eMX1Q9utb6xFrr\ny9X56s7DB9Za71tr/dXBw/dXN213TAAAOF6z1nr6AzMvq25fa7324PErqhevtd7wFOf/S/X8r56/\n7LV7qnuqTp8+/cLz588/w/E3c/HixU6dOvW1xw8//sWdXPdKctuNzz7pEXbm8n1z7bPz/WLf+8W+\n98tx7fvcuXMPrbXOHHXu+m1edGbOVa+pXvJkr6+17u/gNoszZ86ss2fPbvPyT+nChQsdvtar7n33\nTq57JfnU3WdPeoSduXzfXPvsfL/Y936x7/1y0vveJIwfr5576PFNB8/9HTPzA9XbqjvWWp/fzngA\nALAbm9xj/IHqeTNzy8x8c3VX9a7DB2bmu6p3Vq9Ya318+2MCAMDxOvId47XWEzPzhuo91XXVA2ut\nR2bmdQev31f9XPUd1VtnpuqJTe7jAACAK8VG9xivtR6sHrzsufsO/fy11d/7y3YAAHC18M13AACQ\nMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAA\nUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEM\nAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKAS\nxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAA\nKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgKquP+kB2I2b\n7333SY+wMz972xOdPekhAICrjneMAQAgYQwAAJUwBgCAShgDAEC1YRjPzO0z87GZeXRm7n2S12dm\nfvXg9Q/PzA9uf1QAADg+R4bxzFxXvaW6o7q1evnM3HrZsTuq5x38uKf69S3PCQAAx2qTd4xfVD26\n1vrEWuvL1fnqzsvO3Fn9xrrk/dVzZuY7tzwrAAAcm00+x/jG6jOHHj9WvXiDMzdWf/mMpoNv0D59\nbvNXferNP3HSIwDAVW2nX/AxM/d06VaLqosz87EdXfqG6nM7uhYn7D/t6b7nF056ghO1lzvfY/a9\nX+x7vxzXvr97k0ObhPHj1XMPPb7p4Lmv90xrrfur+zcZbJtm5s/WWmd2fV1Ohn3vHzvfL/a9X+x7\nv5z0vje5x/gD1fNm5paZ+ebqrupdl515V/XKg0+n+OHqi2stt1EAAHDVOPId47XWEzPzhuo91XXV\nA2utR2bmdQev31c9WL20erT62+rVxzcyAABs30b3GK+1HuxS/B5+7r5DP1/V67c72lbt/PYNTpR9\n7x873y/2vV/se7+c6L7nUtMCAMB+85XQAADQNRbGvrp6v2yw77sP9vzwzLxvZl5wEnOyHUft+9C5\nH5qZJ2bmZbucj+3aZN8zc3ZmPjgzj8zMH+96RrZrg/+mP3tmfndmPnSwc3+f6So1Mw/MzGdn5iNP\n8fqJ9do1E8a+unq/bLjvT1Y/sta6rXpT7lO7am2476+e+4XqD3c7Idu0yb5n5jnVW6t/u9b6p9VP\n7XxQtmbDP+Ovr/58rfWC6mz1yweflsXV5+3V7U/z+on12jUTxvnq6n1z5L7XWu9ba/3VwcP3d+nz\ntbk6bfLnu+pnqt+qPrvL4di6Tfb909U711qfrlpr2fnVbZOdr+pZMzPVqeoL1RO7HZNtWGu9t0v7\neyon1mvXUhg/1ddSf71nuDp8vbt8TfX7xzoRx+nIfc/MjdVP5v8EXQs2+fP9vdW3z8yFmXloZl65\ns+k4Dpvs/Neqf1L9RfVw9Z/XWl/ZzXjs2In12k6/EhpOwsyc61IYv+SkZ+FY/Ur1xrXWVy69ocQ1\n7vrqhdWPVd9S/enMvH+t9fGTHYtj9OPVB6sfrb6n+qOZ+ZO11t+c7FhcS66lMN7aV1dzVdholzPz\nA9XbqjvWWp/f0Wxs3yb7PlOdP4jiG6qXzswTa63f3s2IbNEm+36s+vxa60vVl2bmvdULKmF8ddpk\n56+u3nzw3QmPzswnq+dX/2s3I7JDJ9Zr19KtFL66er8cue+Z+a7qndUrvIt01Tty32utW9ZaN6+1\nbq5+s/qPoviqtcl/z3+nesnMXD8z31q9uProjudkezbZ+ae79H8ImpnT1fdVn9jplOzKifXaNfOO\nsa+u3i8b7vvnqu+o3nrwLuITa60zJzUz37gN9801YpN9r7U+OjN/UH24+kr1trXWk370E1e+Df+M\nv6l6+8w8XE2Xbp363IkNzTdsZt7RpU8WuWFmHqt+vvqmOvle8813AADQtXUrBQAAfMOEMQAAJIwB\nAKASxgAAUAljAACohDEAAFTCGAAAKmEMAABV/X/T96EB2pLWdAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fea753b0b38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot distribution of similarity scores\n",
    "sim_df.jaccard.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# filter similarity results to only those that have difference phone numbers\n",
    "sim_df_diff_phone = sim_df[(sim_df['id1_phone'] != sim_df['id2_phone']) & (sim_df['id1_phone'] != \"\") & (sim_df['id2_phone'] != \"\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# filter similarity results to those with a jaccard score greater than 0.25\n",
    "filtered = sim_df_diff_phone[sim_df_diff_phone['jaccard'] >= 0.25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11318"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# how many results remain?\n",
    "len(filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id1</th>\n",
       "      <th>id1_city</th>\n",
       "      <th>id1_phone</th>\n",
       "      <th>id1_post_body</th>\n",
       "      <th>id1_post_date</th>\n",
       "      <th>id2</th>\n",
       "      <th>id2_city</th>\n",
       "      <th>id2_phone</th>\n",
       "      <th>id2_post_body</th>\n",
       "      <th>id2_post_date</th>\n",
       "      <th>jaccard</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9413167</th>\n",
       "      <td>1962</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>5045007738</td>\n",
       "      <td>Serious Inquiries Only! No Young AA &amp; Upscaled...</td>\n",
       "      <td>2017-10-10 06:39:00</td>\n",
       "      <td>1136</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>6012154175</td>\n",
       "      <td>Serious Inquiries Only! No AA &amp; Upscaled Gentl...</td>\n",
       "      <td>2017-10-09 19:54:00</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995848</th>\n",
       "      <td>3092</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>6012154175</td>\n",
       "      <td>Serious Inquiries Only! No Young AA &amp; Upscaled...</td>\n",
       "      <td>2017-10-24 13:35:00</td>\n",
       "      <td>1962</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>5045007738</td>\n",
       "      <td>Serious Inquiries Only! No Young AA &amp; Upscaled...</td>\n",
       "      <td>2017-10-10 06:39:00</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5444404</th>\n",
       "      <td>1136</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>6012154175</td>\n",
       "      <td>Serious Inquiries Only! No AA &amp; Upscaled Gentl...</td>\n",
       "      <td>2017-10-09 19:54:00</td>\n",
       "      <td>1962</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>5045007738</td>\n",
       "      <td>Serious Inquiries Only! No Young AA &amp; Upscaled...</td>\n",
       "      <td>2017-10-10 06:39:00</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9414948</th>\n",
       "      <td>1962</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>5045007738</td>\n",
       "      <td>Serious Inquiries Only! No Young AA &amp; Upscaled...</td>\n",
       "      <td>2017-10-10 06:39:00</td>\n",
       "      <td>3092</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>6012154175</td>\n",
       "      <td>Serious Inquiries Only! No Young AA &amp; Upscaled...</td>\n",
       "      <td>2017-10-24 13:35:00</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9418522</th>\n",
       "      <td>1962</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>6012154175</td>\n",
       "      <td>Serious Inquiries Only! No Young AA &amp; Upscaled...</td>\n",
       "      <td>2017-10-10 06:39:00</td>\n",
       "      <td>1136</td>\n",
       "      <td>New Orleans</td>\n",
       "      <td>5045007738</td>\n",
       "      <td>Serious Inquiries Only! No AA &amp; Upscaled Gentl...</td>\n",
       "      <td>2017-10-09 19:54:00</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id1     id1_city   id1_phone  \\\n",
       "9413167   1962  New Orleans  5045007738   \n",
       "14995848  3092  New Orleans  6012154175   \n",
       "5444404   1136  New Orleans  6012154175   \n",
       "9414948   1962  New Orleans  5045007738   \n",
       "9418522   1962  New Orleans  6012154175   \n",
       "\n",
       "                                              id1_post_body  \\\n",
       "9413167   Serious Inquiries Only! No Young AA & Upscaled...   \n",
       "14995848  Serious Inquiries Only! No Young AA & Upscaled...   \n",
       "5444404   Serious Inquiries Only! No AA & Upscaled Gentl...   \n",
       "9414948   Serious Inquiries Only! No Young AA & Upscaled...   \n",
       "9418522   Serious Inquiries Only! No Young AA & Upscaled...   \n",
       "\n",
       "               id1_post_date   id2     id2_city   id2_phone  \\\n",
       "9413167  2017-10-10 06:39:00  1136  New Orleans  6012154175   \n",
       "14995848 2017-10-24 13:35:00  1962  New Orleans  5045007738   \n",
       "5444404  2017-10-09 19:54:00  1962  New Orleans  5045007738   \n",
       "9414948  2017-10-10 06:39:00  3092  New Orleans  6012154175   \n",
       "9418522  2017-10-10 06:39:00  1136  New Orleans  5045007738   \n",
       "\n",
       "                                              id2_post_body  \\\n",
       "9413167   Serious Inquiries Only! No AA & Upscaled Gentl...   \n",
       "14995848  Serious Inquiries Only! No Young AA & Upscaled...   \n",
       "5444404   Serious Inquiries Only! No Young AA & Upscaled...   \n",
       "9414948   Serious Inquiries Only! No Young AA & Upscaled...   \n",
       "9418522   Serious Inquiries Only! No AA & Upscaled Gentl...   \n",
       "\n",
       "               id2_post_date   jaccard  \n",
       "9413167  2017-10-09 19:54:00  0.967742  \n",
       "14995848 2017-10-10 06:39:00  0.967742  \n",
       "5444404  2017-10-10 06:39:00  0.967742  \n",
       "9414948  2017-10-24 13:35:00  0.967742  \n",
       "9418522  2017-10-09 19:54:00  0.967742  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show a sample of the similiarty data\n",
    "filtered.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create a network from the dataframe\n",
    "g = nx.from_pandas_dataframe(filtered, 'id1_phone', 'id2_phone', edge_attr=['jaccard', \n",
    "                                                                            'id1_phone', \n",
    "                                                                            'id2_phone', \n",
    "                                                                            'id1_post_body', \n",
    "                                                                            'id2_post_body'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "768"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# count the number of nodes in the graph\n",
    "g.number_of_nodes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1133"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# count the number of edges in the graph\n",
    "g.number_of_edges()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create a dict for node ids for edges\n",
    "node_dict = {}\n",
    "for i, node in enumerate(g.nodes(data=True)):\n",
    "    node_dict[node[0]] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create a dict of node and edge data\n",
    "edge_data = {\"nodes\": [], \n",
    "             \"links\": []}\n",
    "\n",
    "for node in g.nodes():\n",
    "    edge_data[\"nodes\"].append({\"name\": node, \"group\": 1})\n",
    "    \n",
    "for edge in g.edges(data=True):\n",
    "    edge_data[\"links\"].append({\"source\": node_dict[edge[0]], \"target\": node_dict[edge[1]], \"value\":edge[2]['jaccard']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# write edge data to JSON\n",
    "with open('ad_data_name.json', 'w') as f:\n",
    "    json.dump(edge_data, f, indent=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}