1. Creating ENTNERPTDOI.nt from ENTDOI.csv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "13ae3111",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random \n",
    "import matplotlib.pyplot as plt\n",
    "from fuzzywuzzy import fuzz\n",
    "from collections import defaultdict\n",
    "from rdflib import Graph, Literal, Namespace, RDF, URIRef, BNode\n",
    "import openai\n",
    "import ast\n",
    "import pickle\n",
    "from urllib.parse import quote\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "349e268d",
   "metadata": {},
   "source": [
    "# Load the file ENTPTNERDOI.csv.tar.gz from https://zenodo.org/records/10022727"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "41b17cb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "ENTDOI = pd.read_csv('ENTPTNERDOI.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "969a76c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Entity</th>\n",
       "      <th>Part_of_text</th>\n",
       "      <th>NER_Tag</th>\n",
       "      <th>DOI</th>\n",
       "      <th>Preferred_Entity</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7966059</th>\n",
       "      <td>calcium</td>\n",
       "      <td>Abs</td>\n",
       "      <td>CHM</td>\n",
       "      <td>10.1111/j.1551-2916.2006.01203.x</td>\n",
       "      <td>Calcium</td>\n",
       "      <td>2006.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83090376</th>\n",
       "      <td>sulfur</td>\n",
       "      <td>Cap</td>\n",
       "      <td>CHM</td>\n",
       "      <td>10.1016/j.apsusc.2017.10.076</td>\n",
       "      <td>Sulfur</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23313622</th>\n",
       "      <td>anodes</td>\n",
       "      <td>Abs</td>\n",
       "      <td>APL</td>\n",
       "      <td>10.1016/j.jpowsour.2014.07.081</td>\n",
       "      <td>Anodes</td>\n",
       "      <td>2014.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39090930</th>\n",
       "      <td>p+</td>\n",
       "      <td>Abs</td>\n",
       "      <td>PRO</td>\n",
       "      <td>10.1016/s0169-4332(01)00526-8</td>\n",
       "      <td>P+</td>\n",
       "      <td>2001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22326316</th>\n",
       "      <td>catalysts</td>\n",
       "      <td>Abs</td>\n",
       "      <td>APL</td>\n",
       "      <td>10.1186/s11671-018-2824-7</td>\n",
       "      <td>Catalyst</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Entity Part_of_text NER_Tag                               DOI  \\\n",
       "7966059     calcium          Abs     CHM  10.1111/j.1551-2916.2006.01203.x   \n",
       "83090376     sulfur          Cap     CHM      10.1016/j.apsusc.2017.10.076   \n",
       "23313622     anodes          Abs     APL    10.1016/j.jpowsour.2014.07.081   \n",
       "39090930         p+          Abs     PRO     10.1016/s0169-4332(01)00526-8   \n",
       "22326316  catalysts          Abs     APL         10.1186/s11671-018-2824-7   \n",
       "\n",
       "         Preferred_Entity    Year  \n",
       "7966059           Calcium  2006.0  \n",
       "83090376           Sulfur  2018.0  \n",
       "23313622           Anodes  2014.0  \n",
       "39090930               P+  2001.0  \n",
       "22326316         Catalyst  2018.0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ENTDOI.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50252ec0",
   "metadata": {},
   "source": [
    "### Analysing ENTDOI.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "243a1022",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows : 84659831\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of rows : {}\".format(ENTDOI.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbc230a9",
   "metadata": {},
   "source": [
    "### Extracting all the raw entities for cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "498937e1",
   "metadata": {},
   "source": [
    "Let's extract all raw entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "37e3214c",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_entities = list(set(ENTDOI['Entity'].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "954df95a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of entities : 197623\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of entities : {}\".format(len(all_entities)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d64b8ed4",
   "metadata": {},
   "source": [
    "Now, the entities are clubbed according to their Levenshtein distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67d49253",
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters = defaultdict(set)\n",
    "threshold = 90\n",
    "for entity in tqdm(entity_list, total = len(entity_list)):\n",
    "    found = False\n",
    "    for key in clusters.keys():\n",
    "        if fuzz.ratio(key, entity) >= threshold:\n",
    "            clusters[key].add(entity)\n",
    "            found = True\n",
    "            break\n",
    "    if not found:\n",
    "        clusters[entity].add(entity)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d1096f2",
   "metadata": {},
   "source": [
    "These clustered entities are cleaned using OpenAI API"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19dbcb0d",
   "metadata": {},
   "source": [
    "### Grouping and cleaning clusters using OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a35da84",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_key = '<enter api key>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8a9703f",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"\"\"\n",
    "Given a list of scientific terms as input, return a list of normalized terms as output. The normalization should take into\\\n",
    "account minor spelling variations, plural forms, special characters, and word order. Additionally, the normalization should\\\n",
    "preserve distinctions in terms that have specific additional descriptors. Each input term should have a corresponding\\\n",
    "output term in title case.\n",
    "\n",
    "Examples:\n",
    "Input: ['nanobeam', 'nanobeams', 'gap activity', 'gpx activity', 'alp activity', 'vibration amplitude', 'vibration amplitudes',\\\n",
    " 'photo images', 'Photo images', 'photo image', 'pebbles', 'pebble', 'pressure drops', 'pressure - drop',\\\n",
    " 'dark-field image.']\n",
    "Output: ['Nanobeam', 'Nanobeam', 'Gap Activity', 'GPX Activity', 'ALP Activity', 'Vibration Amplitude', 'Vibration Amplitude',\\\n",
    " 'Photo Images', 'Photo Images', 'Photo Images', 'Pebbles', 'Pebbles', 'Pressure Drop', 'Pressure Drop', 'Dark Field Image']\n",
    "Please provide the output as a list for the given input terms\n",
    "Input Terms:\n",
    "{}\n",
    "Output Terms:\n",
    "\"\"\"\n",
    "\n",
    "def return_prompt(input_):\n",
    "    return prompt.format(input_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2221240",
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_dict = {}\n",
    "failed_lists = []\n",
    "final_list = [list(value) for key, value in clusters.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73643636",
   "metadata": {},
   "outputs": [],
   "source": [
    "for input_list in tqdm(final_lists, total = len(final_lists)):\n",
    "    Run = True\n",
    "    for x in input_list:\n",
    "        if x in mapping_dict:\n",
    "            Run = False\n",
    "    try:\n",
    "        if Run:\n",
    "            response = openai.Completion.create(\n",
    "                                model=\"text-davinci-003\",\n",
    "                                prompt = return_prompt(input_list),\n",
    "                                max_tokens=1024,\n",
    "                                temperature=0.1,\n",
    "                            )\n",
    "            result=response.choices[0].text\n",
    "            result_list = ast.literal_eval(result)\n",
    "\n",
    "            if len(result_list) == len(input_list):\n",
    "                for A, B in zip(input_list, result_list):\n",
    "                    mapping_dict[A] = B\n",
    "            else:\n",
    "                failed_lists.append([input_list, result_list])\n",
    "\n",
    "            if random.random()<0.01:\n",
    "                time.sleep(100)\n",
    "                pickle.dump(mapping_dict, open('final_rdf_mapping_dict.pickle', 'wb'))\n",
    "                pickle.dump(failed_lists, open('final_rdf_error_lists.pickle', 'wb'))\n",
    "        else:\n",
    "            continue\n",
    "    except:\n",
    "        failed_lists.append([input_list, result_list])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f0a0f25",
   "metadata": {},
   "source": [
    "The mapping dict contains the (raw_entity, preferred_entity) pairings. \\\n",
    "Note that many clusters have only value. These can be filtered to trim the number of API calls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ba9d9833",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_pref = list(set(ENTDOI['Preferred_Entity'].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0341e86a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding this data to the DataFrame\n",
    "ENTDOI['Preferred_Entity'] = ENTDOI['Preferred_Entity'].apply(lambda x: mapping_dict[x] if x in mapping_dict else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bd7d4558",
   "metadata": {},
   "outputs": [],
   "source": [
    "ENTDOI.to_csv('ENTDOI.csv', index = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "06145fb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sanitize_entity(entity):\n",
    "    return quote(entity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b7b6f3c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Entity</th>\n",
       "      <th>Part_of_text</th>\n",
       "      <th>NER_Tag</th>\n",
       "      <th>DOI</th>\n",
       "      <th>Preferred_Entity</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>electrospray ionization mass spectrometry</td>\n",
       "      <td>Abs</td>\n",
       "      <td>APL</td>\n",
       "      <td>10.1016/s0925-4005(02)00161-2</td>\n",
       "      <td>Electrospray Ionization Mass Spectrometry</td>\n",
       "      <td>2002.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>wsoc</td>\n",
       "      <td>Abs</td>\n",
       "      <td>APL</td>\n",
       "      <td>10.1016/j.atmosenv.2007.06.034</td>\n",
       "      <td>WSOC</td>\n",
       "      <td>2007.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pyridinium</td>\n",
       "      <td>Abs</td>\n",
       "      <td>APL</td>\n",
       "      <td>10.1021/jo0712401</td>\n",
       "      <td>Pyridinium</td>\n",
       "      <td>2007.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>navy blue</td>\n",
       "      <td>Abs</td>\n",
       "      <td>APL</td>\n",
       "      <td>10.1016/j.ecoleng.2013.07.005</td>\n",
       "      <td>Navy Blue</td>\n",
       "      <td>2013.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gantt chart</td>\n",
       "      <td>Abs</td>\n",
       "      <td>APL</td>\n",
       "      <td>10.1021/ie970286p</td>\n",
       "      <td>Gantt Chart</td>\n",
       "      <td>1997.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Entity Part_of_text NER_Tag  \\\n",
       "0  electrospray ionization mass spectrometry          Abs     APL   \n",
       "1                                       wsoc          Abs     APL   \n",
       "2                                 pyridinium          Abs     APL   \n",
       "3                                  navy blue          Abs     APL   \n",
       "4                                gantt chart          Abs     APL   \n",
       "\n",
       "                              DOI                           Preferred_Entity  \\\n",
       "0   10.1016/s0925-4005(02)00161-2  Electrospray Ionization Mass Spectrometry   \n",
       "1  10.1016/j.atmosenv.2007.06.034                                       WSOC   \n",
       "2               10.1021/jo0712401                                 Pyridinium   \n",
       "3   10.1016/j.ecoleng.2013.07.005                                  Navy Blue   \n",
       "4               10.1021/ie970286p                                Gantt Chart   \n",
       "\n",
       "     Year  \n",
       "0  2002.0  \n",
       "1  2007.0  \n",
       "2  2007.0  \n",
       "3  2013.0  \n",
       "4  1997.0  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ENTDOI.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7e210546",
   "metadata": {},
   "outputs": [],
   "source": [
    "NER, pref_entity = ENTDOI[['NER_Tag', 'Preferred_Entity']].values.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c80725d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(84659831, 84659831)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(NER), len(pref_entity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b621a117",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 84659831/84659831 [02:59<00:00, 471315.03it/s]\n"
     ]
    }
   ],
   "source": [
    "# Dictionary to keep track of entity URI mappings\n",
    "entity_uri_mapping = {}\n",
    "\n",
    "for ner, pref_ent in tqdm(zip(NER, pref_entity), total = len(NER)):\n",
    "    if pref_ent:\n",
    "\n",
    "        # Add Preferred_Entity, creating a URI based on NER tag\n",
    "        preferred_entity_uri = URIRef(\"http://example.com/{}/{}\".format(ner, sanitize_entity(pref_ent)))\n",
    "\n",
    "        entity_uri_mapping[pref_ent] = preferred_entity_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dc782a1",
   "metadata": {},
   "source": [
    "### Creating RDF Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a9178e25",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a RDF graph\n",
    "# Create an empty RDF graph\n",
    "g = Graph()\n",
    "\n",
    "# Define namespaces\n",
    "n = Namespace(\"http://example.com/\")\n",
    "doi = Namespace(\"https://doi.org/\")\n",
    "entity_uri_dict= {}\n",
    "\n",
    "# Iterate through each row in the DataFrame to create RDF triples\n",
    "for index, row in tqdm(ENTDOI.iterrows(), total = ENTDOI.shape[0]):\n",
    "    \n",
    "    if any(pd.isnull(row)):\n",
    "        continue\n",
    "    \n",
    "    \n",
    "    try:\n",
    "        node_uri = URIRef(f\"http://example.com/row{index}\")\n",
    "\n",
    "        # Add Entity\n",
    "        g.add((node_uri, n.hasEntity, Literal(row['Entity'])))\n",
    "        \n",
    "        # Add Part_of_text\n",
    "        g.add((node_uri, n.partOfText, Literal(row['Part_of_text'])))\n",
    "\n",
    "        # Add NER_Tag\n",
    "        g.add((node_uri, n.hasNER_Tag, Literal(row['NER_Tag'])))\n",
    "        \n",
    "\n",
    "\n",
    "        # Add DOI, encoding it first\n",
    "        encoded_doi = quote(row['DOI'])\n",
    "        doi_uri     =  URIRef(f\"https://doi.org/{encoded_doi}\")\n",
    "        g.add((node_uri, n.hasDOI, doi_uri))\n",
    "        # Add Year to DOI       \n",
    "        g.add((doi_uri, n.hasYear, Literal(row['Year'])))\n",
    "        \n",
    "        # Add Preferred_Entity, creating a URI based on NER tag\n",
    "        preferred_entity_uri =entity_uri_mapping[row['Preferred_Entity']]\n",
    "        g.add((node_uri, n.hasPreferredEntity, preferred_entity_uri))\n",
    "        entity_uri_dict[row['Preferred_Entity']] = preferred_entity_uri\n",
    "        \n",
    "        if (index+1)%10000000==0:\n",
    "            g.serialize(destination='ENTPTNERDOI.nt', format='nt')\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        pass\n",
    "\n",
    "g.serialize(destination='ENTPTNERDOI.nt', format='nt')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "98a262db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dc74abf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "vineeth_10",
   "language": "python",
   "name": "vineeth_10"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}