Featurization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JHuoPajdN5f3"
   },
   "source": [
    "<p style=\"font-size:16px;text-align:left\"> <b>Graph Featurization</b> </p>\n",
    "\n",
    "Generated features using concepts of graph theory and other techniques:\n",
    "\n",
    "- Jaccard distance\n",
    "- Cosine distance\n",
    "- Page rank\n",
    "- Shortest path\n",
    "- Weakly connected components\n",
    "- Adar/Adamic Index\n",
    "- Kartz centrality\n",
    "- Follows back\n",
    "- HITS score\n",
    "- Weight features\n",
    "- SVD features\n",
    "- Preferential attachment features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 1131,
     "status": "ok",
     "timestamp": 1621878696427,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "9Z7ns4EEOHzE"
   },
   "outputs": [],
   "source": [
    "#Importing Libraries\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import csv\n",
    "import pandas as pd\n",
    "import datetime \n",
    "import time \n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pylab as plt\n",
    "from matplotlib import rcParams # size of plots\n",
    "import seaborn as sns\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "\n",
    "import networkx as nx # library to study graphs and networks\n",
    "import pdb\n",
    "import pickle\n",
    "from pandas import HDFStore,DataFrame\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from pandas import HDFStore,DataFrame\n",
    "from pandas import read_hdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1znHayNeVFFt"
   },
   "source": [
    "# 1. Reading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 52707,
     "status": "ok",
     "timestamp": 1621878751481,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "Uq9HbHwEVFFv",
    "outputId": "0dab0dae-f970-4dd4-a132-e02c0908322c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: \n",
      "Type: DiGraph\n",
      "Number of nodes: 1780722\n",
      "Number of edges: 7550015\n",
      "Average in degree:   4.2399\n",
      "Average out degree:   4.2399\n"
     ]
    }
   ],
   "source": [
    "if os.path.isfile('data/after_eda/train_pos_after_eda.csv'):\n",
    "    train_graph=nx.read_edgelist('data/after_eda/train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n",
    "    print(nx.info(train_graph))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HmlUa64tVFF7"
   },
   "source": [
    "# 2. Similarity measures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ivVMUMiWVFF9"
   },
   "source": [
    "## 2.1 Jaccard Distance:\n",
    "http://www.statisticshowto.com/jaccard-index/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NoWCYuRBVFF_"
   },
   "source": [
    "\\begin{equation}\n",
    "j = \\frac{|X\\cap Y|}{|X \\cup Y|} \n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "executionInfo": {
     "elapsed": 348,
     "status": "ok",
     "timestamp": 1621874467386,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "Seo4z5SnVFGB"
   },
   "outputs": [],
   "source": [
    "# for followees\n",
    "def jaccard_for_followees(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n",
    "                                    (len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n",
    "    except:\n",
    "        return 0\n",
    "    return sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 347,
     "status": "ok",
     "timestamp": 1621874473500,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "Oa9FMlS8VFGF",
    "outputId": "b0a08df5-d7a2-434c-b087-cbc466f1db69"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "# one test case\n",
    "print(jaccard_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "executionInfo": {
     "elapsed": 1137,
     "status": "ok",
     "timestamp": 1621874486029,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "LO-a5ZkKVFGO"
   },
   "outputs": [],
   "source": [
    "#for followers\n",
    "def jaccard_for_followers(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n",
    "                                 (len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 371,
     "status": "ok",
     "timestamp": 1621874496911,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "DlbX2t0jVFGQ",
    "outputId": "f5d40471-fa79-4206-b041-6d74cfbd67c2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(jaccard_for_followers(273084,470294))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_MH6HvphQqBD"
   },
   "source": [
    "Higher the Jaccard distance, more is the chance of a and b following each other."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MnH2my2UVFGX"
   },
   "source": [
    "## 2.2 Cosine distance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XNvdBGS2VFGY"
   },
   "source": [
    "\\begin{equation}\n",
    "CosineDistance = \\frac{|X\\cap Y|}{|X|\\cdot|Y|} \n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "Iznz67EdVFGZ"
   },
   "outputs": [],
   "source": [
    "# for followees\n",
    "\n",
    "def cosine_for_followees(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n",
    "                                    (math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "H55ALjkMVFGc",
    "outputId": "531fceba-60f4-4e6b-97f4-f37733dc468f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "q0RGKgJFVFGf",
    "outputId": "41202fc6-f4aa-4a1d-d8f6-84f960a3fbba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followees(273084,1635354))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "KJ_yGxA0VFGj"
   },
   "outputs": [],
   "source": [
    "def cosine_for_followers(a,b):\n",
    "    try:\n",
    "        \n",
    "        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n",
    "                                     (math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "75QrFJb6VFGm",
    "outputId": "f01e0558-f1e3-465f-ab14-0e4ca764f4aa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.02886751345948129\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followers(2,470294))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-ut4k_F0VFGq",
    "outputId": "8bc9607a-7262-43e2-9de8-f71d276762fc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followers(669354,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DaIHhWh6VFGv"
   },
   "source": [
    "## 3. Ranking Measures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6nfV1SprVFGx"
   },
   "source": [
    "https://networkx.github.io/documentation/networkx-1.10/reference/generated/networkx.algorithms.link_analysis.pagerank_alg.pagerank.html\n",
    "\n",
    "PageRank computes a ranking of the nodes in the graph G based on the structure of the incoming links.\n",
    "\n",
    "Mathematical PageRanks for a simple network, expressed as percentages. (Google uses a logarithmic scale.) Page C has a higher PageRank than Page E, even though there are fewer links to C; the one link to C comes from an important page and hence is of high value. If web surfers who start on a random page have an 85% likelihood of choosing a random link from the page they are currently visiting, and a 15% likelihood of jumping to a page chosen at random from the entire web, they will reach Page E 8.1% of the time. <b>(The 15% likelihood of jumping to an arbitrary page corresponds to a damping factor of 85%.) Without damping, all web surfers would eventually end up on Pages A, B, or C, and all other pages would have PageRank zero. In the presence of damping, Page A effectively links to all pages in the web, even though it has no outgoing links of its own.</b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GkkfYYZ6VFGy"
   },
   "source": [
    "## 3.1 Page Ranking\n",
    "\n",
    "https://en.wikipedia.org/wiki/PageRank\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "AtvqwZ34VFGy"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/page_rank.p'):\n",
    "    pr = nx.pagerank(train_graph, alpha=0.85) # here alpha is a hyperparameter\n",
    "    pickle.dump(pr,open('data/fea_sample/page_rank.p','wb'))\n",
    "else:\n",
    "    pr = pickle.load(open('data/fea_sample/page_rank.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lXGKYYf6VFG2",
    "outputId": "bb3d1b7a-81f9-44ab-dbe7-3214ccd47179"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 1.6556497245737814e-07\n",
      "max 2.7098251341935827e-05\n",
      "mean 5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "print('min',pr[min(pr, key=pr.get)])\n",
    "print('max',pr[max(pr, key=pr.get)])\n",
    "print('mean',float(sum(pr.values())) / len(pr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5xwlah4oVFG4",
    "outputId": "992fdfad-7ff6-4626-c9ee-d9bce220a680"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "# for imputing to nodes which are not there in train data\n",
    "mean_pr = float(sum(pr.values())) / len(pr)\n",
    "print(mean_pr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HhPbSL1tVFG7"
   },
   "source": [
    "# 4. Other Graph Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AgsorCl7VFG8"
   },
   "source": [
    "## 4.1 Shortest path:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "E7teH2LCVFG9"
   },
   "source": [
    "Getting Shortest path between two nodes, if nodes have direct path i.e directly connected then we are removing that edge and calculating path. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "RA076ovzVFG9"
   },
   "outputs": [],
   "source": [
    "# if has direct edge then deleting that edge and calculating shortest path\n",
    "\n",
    "def compute_shortest_path_length(a,b):\n",
    "    p=-1\n",
    "    try:\n",
    "        if train_graph.has_edge(a,b):\n",
    "            train_graph.remove_edge(a,b)\n",
    "            p= nx.shortest_path_length(train_graph,source=a,target=b)\n",
    "            train_graph.add_edge(a,b)\n",
    "        else:\n",
    "            p= nx.shortest_path_length(train_graph,source=a,target=b)\n",
    "        return p\n",
    "    except:\n",
    "        return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AxnKId11VFG_",
    "outputId": "15ca223a-6a04-4549-d010-54619b472a9e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# testing\n",
    "compute_shortest_path_length(77697, 826021)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0huWCNtRVFHC",
    "outputId": "6debfa4f-2067-48bc-84b3-ab86e2d9dea6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 22,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# testing\n",
    "compute_shortest_path_length(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "baE_95bzVFHF"
   },
   "source": [
    "## 4.2 Checking for same community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "15CIQqAbVFHG"
   },
   "outputs": [],
   "source": [
    "# getting weakly connected edges from graph \n",
    "\n",
    "wcc=list(nx.weakly_connected_components(train_graph))\n",
    "\n",
    "def belongs_to_same_wcc(a,b):\n",
    "    index = []\n",
    "    if train_graph.has_edge(b,a):\n",
    "      return 1\n",
    "    if train_graph.has_edge(a,b):\n",
    "        for i in wcc:\n",
    "            if a in i:\n",
    "                index= i\n",
    "                break\n",
    "        if (b in index):\n",
    "            train_graph.remove_edge(a,b)\n",
    "            if compute_shortest_path_length(a,b)==-1:\n",
    "                train_graph.add_edge(a,b)\n",
    "                return 0\n",
    "            else:\n",
    "                train_graph.add_edge(a,b)\n",
    "                return 1\n",
    "        else:\n",
    "            return 0\n",
    "    else:\n",
    "        for i in wcc:\n",
    "            if a in i:\n",
    "                index= i\n",
    "                break\n",
    "        if(b in index):\n",
    "            return 1\n",
    "        else:\n",
    "            return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fAzOHtCFVFHI",
    "outputId": "2b043a87-b460-42bf-f37e-4c04bbed6586"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 24,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "belongs_to_same_wcc(861, 1659750)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HMdYpPuGVFHK",
    "outputId": "2005e22c-b60f-48d7-839b-650bf97cae35"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 25,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "belongs_to_same_wcc(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "q74nth0OVFHN"
   },
   "source": [
    "## 4.3 Adamic/Adar Index:\n",
    "Adamic/Adar measures is defined as inverted sum of degrees of common neighbours for given two vertices.\n",
    "$$A(x,y)=\\sum_{u \\in N(x) \\cap N(y)}\\frac{1}{log(|N(u)|)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "CeS98LI5VFHO"
   },
   "outputs": [],
   "source": [
    "# adar index\n",
    "def calc_adar_in(a,b):\n",
    "    sum=0\n",
    "    try:\n",
    "        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n",
    "        if len(n)!=0:\n",
    "            for i in n:\n",
    "                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n",
    "            return sum\n",
    "        else:\n",
    "            return 0\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KezFeRmyVFHQ",
    "outputId": "2f9c0e11-02d9-4f28-d67a-65e3d4943e99"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 27,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_adar_in(1,189226)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vj_m89bBVFHV",
    "outputId": "68a0a099-2954-402f-c80f-6d436ffa1aba"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 28,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_adar_in(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pBUudhFAVFHY"
   },
   "source": [
    "## 4.4 Is persion was following back:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "j_mwmopLVFHZ"
   },
   "outputs": [],
   "source": [
    "def follows_back(a,b):\n",
    "    if train_graph.has_edge(b,a):\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LdjUXIfbVFHb",
    "outputId": "ed3d8640-9834-4a95-e712-804292da70e9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 30,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "follows_back(1,189226)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PmZtL65YVFHf",
    "outputId": "18ea6fe2-3f96-42c0-d116-ecb76ddba4b5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "follows_back(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "29Vrq2EXVFHi"
   },
   "source": [
    "## 4.5 Katz Centrality:\n",
    "https://en.wikipedia.org/wiki/Katz_centrality\n",
    "\n",
    "https://www.geeksforgeeks.org/katz-centrality-centrality-measure/\n",
    " Katz centrality computes the centrality for a node \n",
    "    based on the centrality of its neighbors. It is a \n",
    "    generalization of the eigenvector centrality. The\n",
    "    Katz centrality for node `i` is\n",
    " \n",
    "$$x_i = \\alpha \\sum_{j} A_{ij} x_j + \\beta,$$\n",
    "where `A` is the adjacency matrix of the graph G \n",
    "with eigenvalues $$\\lambda$$.\n",
    "\n",
    "The parameter $$\\beta$$ controls the initial centrality and \n",
    "\n",
    "$$\\alpha < \\frac{1}{\\lambda_{max}}.$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "CN5OSqrkVFHj"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/katz.p'):\n",
    "    katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n",
    "    pickle.dump(katz,open('data/fea_sample/katz.p','wb'))\n",
    "else:\n",
    "    katz = pickle.load(open('data/fea_sample/katz.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gcU83vw7VFHm",
    "outputId": "05f49ad4-46fe-4cf6-f32a-2fe4846b0714"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 0.0007313532484065916\n",
      "max 0.003394554981699122\n",
      "mean 0.0007483800935562018\n"
     ]
    }
   ],
   "source": [
    "print('min',katz[min(katz, key=katz.get)])\n",
    "print('max',katz[max(katz, key=katz.get)])\n",
    "print('mean',float(sum(katz.values())) / len(katz))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qcboIksiVFHt",
    "outputId": "99f52422-9edb-479a-d5d9-e33401160da7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0007483800935562018\n"
     ]
    }
   ],
   "source": [
    "mean_katz = float(sum(katz.values())) / len(katz)\n",
    "print(mean_katz)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SRZqGFgYVFHx"
   },
   "source": [
    "## 4.6 Hits Score\n",
    "The HITS algorithm computes two numbers for a node. Authorities estimates the node value based on the incoming links. Hubs estimates the node value based on outgoing links.\n",
    "\n",
    "https://en.wikipedia.org/wiki/HITS_algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "WXNHRdzUVFHz"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/hits.p'):\n",
    "    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n",
    "    pickle.dump(hits,open('data/fea_sample/hits.p','wb'))\n",
    "else:\n",
    "    hits = pickle.load(open('data/fea_sample/hits.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PSUwSZBVVFH3",
    "outputId": "77448253-5409-4229-f0be-b8dbc14d7f46"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 0.0\n",
      "max 0.004868653378780953\n",
      "mean 5.615699699344123e-07\n"
     ]
    }
   ],
   "source": [
    "print('min',hits[0][min(hits[0], key=hits[0].get)])\n",
    "print('max',hits[0][max(hits[0], key=hits[0].get)])\n",
    "print('mean',float(sum(hits[0].values())) / len(hits[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZZtowOLZVFH6"
   },
   "source": [
    "# 5. Featurization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "o6NnRWmLVFH6"
   },
   "source": [
    "## 5. 1 Reading a sample of Data from both train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "executionInfo": {
     "elapsed": 25310,
     "status": "ok",
     "timestamp": 1621878946351,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "wgHje1UVVFH8"
   },
   "outputs": [],
   "source": [
    "# Reference: https://stackoverflow.com/a/22259008/4084039\n",
    "\n",
    "if os.path.isfile('data/after_eda/train_after_eda.csv'):\n",
    "    filename = \"data/after_eda/train_after_eda.csv\"\n",
    "    n_train =  15100028 # total no of datapoints in train\n",
    "    s = 100000 # desired sample size\n",
    "    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "executionInfo": {
     "elapsed": 5625,
     "status": "ok",
     "timestamp": 1621878951968,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "zOzuRFFlVFH-"
   },
   "outputs": [],
   "source": [
    "# Reference: https://stackoverflow.com/a/22259008/4084039\n",
    "\n",
    "if os.path.isfile('data/after_eda/train_after_eda.csv'):\n",
    "    filename = \"data/after_eda/test_after_eda.csv\"\n",
    "    n_test = 3775006 # total no of datapoints in test\n",
    "    s = 50000 # desired sample size\n",
    "    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1621878953843,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "3D_SeUCOVFH_",
    "outputId": "8ddb6aaa-aa2a-4216-9963-5d4dee3876df"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows in the train data file: 15100028\n",
      "Number of rows we are going to elimiate in train data are 15000028\n",
      "Number of rows in the test data file: 3775006\n",
      "Number of rows we are going to elimiate in test data are 3725006\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of rows in the train data file:\", n_train)\n",
    "print(\"Number of rows we are going to elimiate in train data are\",len(skip_train))\n",
    "print(\"Number of rows in the test data file:\", n_test)\n",
    "print(\"Number of rows we are going to elimiate in test data are\",len(skip_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 124
    },
    "executionInfo": {
     "elapsed": 8245,
     "status": "ok",
     "timestamp": 1621878964406,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "pCisf6PpVFID",
    "outputId": "6a3f9888-9011-4ed6-d4c4-848aac76ab6f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our train matrix size  (100002, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273084</td>\n",
       "      <td>1505602</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>122637</td>\n",
       "      <td>7211</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link\n",
       "0       273084           1505602               1\n",
       "1       122637              7211               1"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final_train = pd.read_csv('data/after_eda/train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\n",
    "df_final_train['indicator_link'] = pd.read_csv('data/train_y.csv', skiprows=skip_train, names=['indicator_link'])\n",
    "print(\"Our train matrix size \",df_final_train.shape)\n",
    "df_final_train.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 124
    },
    "executionInfo": {
     "elapsed": 2623,
     "status": "ok",
     "timestamp": 1621878966995,
     "user": {
      "displayName": "Amit Singh",
      "photoUrl": "",
      "userId": "08452780927821262547"
     },
     "user_tz": -330
    },
    "id": "tFn1RkdyVFIH",
    "outputId": "9639276f-848d-4254-cb73-8422a4e74291"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our test matrix size  (50002, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>848424</td>\n",
       "      <td>784690</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1282653</td>\n",
       "      <td>659527</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link\n",
       "0       848424            784690               1\n",
       "1      1282653            659527               1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final_test = pd.read_csv('data/after_eda/test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\n",
    "df_final_test['indicator_link'] = pd.read_csv('data/test_y.csv', skiprows=skip_test, names=['indicator_link'])\n",
    "print(\"Our test matrix size \",df_final_test.shape)\n",
    "df_final_test.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gIaOWDaDVFIJ"
   },
   "source": [
    "## 5.2 Adding a set of features\n",
    "\n",
    "__we will create each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>jaccard_followers</li>\n",
    "<li>jaccard_followees</li>\n",
    "<li>cosine_followers</li>\n",
    "<li>cosine_followees</li>\n",
    "<li>num_followers_s</li>\n",
    "<li>num_followees_s</li>\n",
    "<li>num_followers_d</li>\n",
    "<li>num_followees_d</li>\n",
    "<li>inter_followers</li>\n",
    "<li>inter_followees</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "2qTkOiBcVFIJ"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage1.h5'):\n",
    "    # mapping jaccard followers to train and test data\n",
    "    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    # mapping jaccard followees to train and test data\n",
    "    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    \n",
    "\n",
    "    # mapping cosine followers to train and test data\n",
    "    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    # mapping cosine followees to train and test data\n",
    "    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "fz2eZpSnVFIL"
   },
   "outputs": [],
   "source": [
    "def compute_features_stage1(df_final):\n",
    "    # calculating no of followers followees for source and destination\n",
    "    # calculating intersection of followers and followees for source and destination\n",
    "    num_followers_s=[]\n",
    "    num_followees_s=[]\n",
    "    num_followers_d=[]\n",
    "    num_followees_d=[]\n",
    "    inter_followers=[]\n",
    "    inter_followees=[]\n",
    "    for i,row in df_final.iterrows():\n",
    "        try:\n",
    "            s1=set(train_graph.predecessors(row['source_node']))\n",
    "            s2=set(train_graph.successors(row['source_node']))\n",
    "        except:\n",
    "            s1 = set()\n",
    "            s2 = set()\n",
    "        try:\n",
    "            d1=set(train_graph.predecessors(row['destination_node']))\n",
    "            d2=set(train_graph.successors(row['destination_node']))\n",
    "        except:\n",
    "            d1 = set()\n",
    "            d2 = set()\n",
    "        num_followers_s.append(len(s1))\n",
    "        num_followees_s.append(len(s2))\n",
    "\n",
    "        num_followers_d.append(len(d1))\n",
    "        num_followees_d.append(len(d2))\n",
    "\n",
    "        inter_followers.append(len(s1.intersection(d1)))\n",
    "        inter_followees.append(len(s2.intersection(d2)))\n",
    "    \n",
    "    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "VFc60kcRVFIN"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage1.h5'):\n",
    "    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n",
    "    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n",
    "    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)\n",
    "    \n",
    "    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n",
    "    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n",
    "    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)\n",
    "    \n",
    "    hdf = HDFStore('data/fea_sample/storage_sample_stage1.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf('data/fea_sample/storage_sample_stage1.h5', 'train_df',mode='r')\n",
    "    df_final_test = read_hdf('data/fea_sample/storage_sample_stage1.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "go_e8hxxVFIO"
   },
   "source": [
    "## 5.3 Adding new set of features\n",
    "\n",
    "__we will create each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>adar index</li>\n",
    "<li>is following back</li>\n",
    "<li>belongs to same weakly connect components</li>\n",
    "<li>shortest path between source and destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "LqB0Peg0VFIP"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage2.h5'):\n",
    "    # mapping adar index on train\n",
    "    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n",
    "    # mapping adar index on test\n",
    "    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    # mapping followback or not on train\n",
    "    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    # mapping followback or not on test\n",
    "    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    # mapping same component of wcc or not on train\n",
    "    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    # mapping same component of wcc or not on train\n",
    "    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n",
    "    \n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    # mapping shortest path on train \n",
    "    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n",
    "    # mapping shortest path on test\n",
    "    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    hdf = HDFStore('data/fea_sample/storage_sample_stage2.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf('data/fea_sample/storage_sample_stage2.h5', 'train_df',mode='r')\n",
    "    df_final_test = read_hdf('data/fea_sample/storage_sample_stage2.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HJ8Dbma_VFIR"
   },
   "source": [
    "## 5.4 Adding new set of features\n",
    "\n",
    "__we will create each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>Weight Features\n",
    "    <ul>\n",
    "        <li>weight of incoming edges</li>\n",
    "        <li>weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges + weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges * weight of outgoing edges</li>\n",
    "        <li>2*weight of incoming edges + weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges + 2*weight of outgoing edges</li>\n",
    "    </ul>\n",
    "</li>\n",
    "<li>Page Ranking of source</li>\n",
    "<li>Page Ranking of dest</li>\n",
    "<li>katz of source</li>\n",
    "<li>katz of dest</li>\n",
    "<li>hubs of source</li>\n",
    "<li>hubs of dest</li>\n",
    "<li>authorities_s of source</li>\n",
    "<li>authorities_s of dest</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iVHI2jtNVFIS"
   },
   "source": [
    "#### Weight Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rXmUYF9FVFIT"
   },
   "source": [
    "In order to determine the similarity of nodes, an edge weight value was calculated between nodes. Edge weight decreases as the neighbor count goes up. Intuitively, consider one million people following a celebrity on a social network then chances are most of them never met each other or the celebrity. On the other hand, if a user has 30 contacts in his/her social network, the chances are higher that many of them know each other. \n",
    "`credit` - Graph-based Features for Supervised Link Prediction\n",
    "William Cukierski, Benjamin Hamner, Bo Yang"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Qzbs2no7VFIV"
   },
   "source": [
    "\\begin{equation}\n",
    "W = \\frac{1}{\\sqrt{1+|X|}}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kkzUPrWaVFIV"
   },
   "source": [
    "it is directed graph so calculated Weighted in and Weighted out differently"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FgNMzzTbVFIW",
    "outputId": "7e8e6d88-8bd6-45f6-f80e-82b093c18974"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████| 1780722/1780722 [00:11<00:00, 152682.24it/s]\n"
     ]
    }
   ],
   "source": [
    "# weight for source and destination of each link\n",
    "Weight_in = {}\n",
    "Weight_out = {}\n",
    "for i in  tqdm(train_graph.nodes()):\n",
    "    s1=set(train_graph.predecessors(i))\n",
    "    w_in = 1.0/(np.sqrt(1+len(s1)))\n",
    "    Weight_in[i]=w_in\n",
    "    \n",
    "    s2=set(train_graph.successors(i))\n",
    "    w_out = 1.0/(np.sqrt(1+len(s2)))\n",
    "    Weight_out[i]=w_out\n",
    "    \n",
    "# for imputing with mean\n",
    "mean_weight_in = np.mean(list(Weight_in.values()))\n",
    "mean_weight_out = np.mean(list(Weight_out.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "AF4yPhIOVFIY"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage3.h5'):\n",
    "    # mapping to pandas train\n",
    "    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "\n",
    "    # mapping to pandas test\n",
    "    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "\n",
    "\n",
    "    # some features engineerings on the in and out weights\n",
    "    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n",
    "    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n",
    "    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n",
    "    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n",
    "\n",
    "    # some features engineerings on the in and out weights\n",
    "    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n",
    "    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n",
    "    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n",
    "    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "uhxzhQ9aVFIa"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage3.h5'):\n",
    "    \n",
    "    # page rank for source and destination in Train and Test\n",
    "    # if anything not there in train graph then adding mean page rank \n",
    "    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "\n",
    "    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    #================================================================================\n",
    "\n",
    "    # Katz centrality score for source and destination in Train and test\n",
    "    # if anything not there in train graph then adding mean katz score\n",
    "    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "\n",
    "    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    #================================================================================\n",
    "\n",
    "    # Hits algorithm score for source and destination in Train and test\n",
    "    # if anything not there in train graph then adding 0\n",
    "    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "\n",
    "    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "    #================================================================================\n",
    "\n",
    "    # Hits algorithm score for source and destination in Train and Test\n",
    "    # if anything not there in train graph then adding 0\n",
    "    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "\n",
    "    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "    #================================================================================\n",
    "\n",
    "    hdf = HDFStore('data/fea_sample/storage_sample_stage3.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf('data/fea_sample/storage_sample_stage3.h5', 'train_df',mode='r')\n",
    "    df_final_test = read_hdf('data/fea_sample/storage_sample_stage3.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p6xkDfD-VFIb"
   },
   "source": [
    "## 5.5 Adding new set of features\n",
    "\n",
    "__we will create each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>SVD features for both source and destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "WQO6E65eVFIc"
   },
   "outputs": [],
   "source": [
    "def svd(x, S):\n",
    "    try:\n",
    "        z = sadj_dict[x]\n",
    "        return S[z]\n",
    "    except:\n",
    "        return [0,0,0,0,0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "9sOyLwvNVFId"
   },
   "outputs": [],
   "source": [
    "# for svd features to get feature vector creating a dict node val and index in svd vector\n",
    "sadj_col = sorted(train_graph.nodes())\n",
    "sadj_dict = { val:idx for idx,val in enumerate(sadj_col)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "zLSt8fGVVFIg"
   },
   "outputs": [],
   "source": [
    "Adj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "soq-VAHlVFIh",
    "outputId": "3f9bfb32-004f-4698-e415-469243250130"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adjacency matrix Shape (1780722, 1780722)\n",
      "U Shape (1780722, 6)\n",
      "V Shape (6, 1780722)\n",
      "s Shape (6,)\n"
     ]
    }
   ],
   "source": [
    "U, s, V = svds(Adj, k = 6)\n",
    "print('Adjacency matrix Shape',Adj.shape)\n",
    "print('U Shape',U.shape)\n",
    "print('V Shape',V.shape)\n",
    "print('s Shape',s.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "ls5fqLFhVFIm"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/fea_sample/storage_sample_stage4.h5'):\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n",
    "    df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    \n",
    "    df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n",
    "    df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n",
    "    df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "\n",
    "    df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n",
    "    df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n",
    "    df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    \n",
    "    df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n",
    "    df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n",
    "    df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "\n",
    "    df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n",
    "    df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "\n",
    "    hdf = HDFStore('data/fea_sample/storage_sample_stage4.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "DaIHhWh6VFGv",
    "GkkfYYZ6VFGy",
    "AgsorCl7VFG8",
    "baE_95bzVFHF",
    "pBUudhFAVFHY",
    "29Vrq2EXVFHi",
    "SRZqGFgYVFHx"
   ],
   "name": "FB_featurization.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}