NLP text search using hugging face transformer model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b0585b4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd, json\n",
    "from elasticsearch import Elasticsearch\n",
    "from getpass import getpass\n",
    "from urllib.request import urlopen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2e74133b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'name': 'liuxgm.local', 'cluster_name': 'elasticsearch', 'cluster_uuid': 'lM5ZgEQNTkO8phtT5qZAHQ', 'version': {'number': '8.10.0', 'build_flavor': 'default', 'build_type': 'tar', 'build_hash': 'e338da74c79465dfdc204971e600342b0aa87b6b', 'build_date': '2023-09-07T08:16:21.960703010Z', 'build_snapshot': False, 'lucene_version': '9.7.0', 'minimum_wire_compatibility_version': '7.17.0', 'minimum_index_compatibility_version': '7.0.0'}, 'tagline': 'You Know, for Search'}\n"
     ]
    }
   ],
   "source": [
    "ELASTCSEARCH_CERT_PATH = \"/Users/liuxg/elastic/elasticsearch-8.10.0/config/certs/http_ca.crt\"\n",
    " \n",
    "es = Elasticsearch(  ['https://localhost:9200'],\n",
    "    basic_auth = ('elastic', 'vXDWYtL*my3vnKY9zCfL'),\n",
    "    ca_certs = ELASTCSEARCH_CERT_PATH,\n",
    "    verify_certs = True)\n",
    "print(es.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "99da8e70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'acknowledged': True})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ingest pipeline definition\n",
    "PIPELINE_ID=\"vectorize_blogs\"\n",
    "\n",
    "es.ingest.put_pipeline(id=PIPELINE_ID, processors=[{\n",
    "        \"inference\": {\n",
    "          \"model_id\": \"sentence-transformers__all-minilm-l6-v2\",\n",
    "          \"target_field\": \"text_embedding\",\n",
    "          \"field_map\": {\n",
    "            \"title\": \"text_field\"\n",
    "          }\n",
    "        }\n",
    "      }])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d6eb66c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting existing blogs\n",
      "Creating index blogs\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'acknowledged': True, 'shards_acknowledged': True, 'index': 'blogs'})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define index name\n",
    "INDEX_NAME=\"blogs\"\n",
    "\n",
    "# flag to check if index has to be deleted before creating\n",
    "SHOULD_DELETE_INDEX=True\n",
    "\n",
    "# define index mapping\n",
    "INDEX_MAPPING = {\n",
    "    \"properties\": {\n",
    "      \"title\": {\n",
    "        \"type\": \"text\",\n",
    "        \"fields\": {\n",
    "          \"keyword\": {\n",
    "            \"type\": \"keyword\",\n",
    "            \"ignore_above\": 256\n",
    "          }\n",
    "        }\n",
    "      },\n",
    "      \n",
    "      \"text_embedding\": {\n",
    "        \"properties\": {\n",
    "          \"is_truncated\": {\n",
    "            \"type\": \"boolean\"\n",
    "          },\n",
    "          \"model_id\": {\n",
    "            \"type\": \"text\",\n",
    "            \"fields\": {\n",
    "              \"keyword\": {\n",
    "                \"type\": \"keyword\",\n",
    "                \"ignore_above\": 256\n",
    "              }\n",
    "            }\n",
    "          },\n",
    "          \"predicted_value\": {\n",
    "            \"type\": \"dense_vector\",\n",
    "            \"dims\": 384,\n",
    "            \"index\": True,\n",
    "            \"similarity\": \"l2_norm\"\n",
    "          }\n",
    "        }\n",
    "      }\n",
    "    }\n",
    "  }\n",
    "\n",
    "INDEX_SETTINGS = {\n",
    "    \"index\": {\n",
    "      \"number_of_replicas\": \"1\",\n",
    "      \"number_of_shards\": \"1\",\n",
    "      \"default_pipeline\": PIPELINE_ID\n",
    "    }\n",
    "}\n",
    "\n",
    "# check if we want to delete index before creating the index\n",
    "if(SHOULD_DELETE_INDEX):\n",
    "  if es.indices.exists(index=INDEX_NAME):\n",
    "    print(\"Deleting existing %s\" % INDEX_NAME)\n",
    "    es.options(ignore_status=[400, 404]).indices.delete(index=INDEX_NAME)\n",
    "\n",
    "print(\"Creating index %s\" % INDEX_NAME)\n",
    "es.options(ignore_status=[400,404]).indices.create(index=INDEX_NAME, mappings=INDEX_MAPPING, settings=INDEX_SETTINGS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a02021bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'title': 'Pulp Fiction', 'runtime': '154', 'plot': 'The lives of two mob hitmen, a boxer, a gangster and his wife, and a pair of diner bandits intertwine in four tales of violence and redemption.', 'keyScene': \"John Travolta is forced to inject adrenaline directly into Uma Thurman's heart after she overdoses on heroin.\", 'genre': 'Crime, Drama', 'released': '1994'}, {'title': 'The Dark Knight', 'runtime': '152', 'plot': 'When the menace known as the Joker wreaks havoc and chaos on the people of Gotham, Batman must accept one of the greatest psychological and physical tests of his ability to fight injustice.', 'keyScene': \"Batman angrily responds 'I’m Batman' when asked who he is by Falcone.\", 'genre': 'Action, Crime, Drama, Thriller', 'released': '2008'}, {'title': 'Fight Club', 'runtime': '139', 'plot': 'An insomniac office worker and a devil-may-care soapmaker form an underground fight club that evolves into something much, much more.', 'keyScene': 'Brad Pitt explains the rules of Fight Club to Edward Norton. The first rule of Fight Club is: You do not talk about Fight Club. The second rule of Fight Club is: You do not talk about Fight Club.', 'genre': 'Drama', 'released': '1999'}, {'title': 'Inception', 'runtime': '148', 'plot': 'A thief who steals corporate secrets through the use of dream-sharing technology is given the inverse task of planting an idea into thed of a C.E.O.', 'keyScene': \"Leonardo DiCaprio explains the concept of inception to Ellen Page by using a child's spinning top.\", 'genre': 'Action, Adventure, Sci-Fi, Thriller', 'released': '2010'}, {'title': 'The Matrix', 'runtime': '136', 'plot': 'A computer hacker learns from mysterious rebels about the true nature of his reality and his role in the war against its controllers.', 'keyScene': 'Red pill or blue pill? Morpheus offers Neo a choice between the red pill, which will allow him to learn the truth about the Matrix, or the blue pill, which will return him to his former life.', 'genre': 'Action, Sci-Fi', 'released': '1999'}, {'title': 'The Shawshank Redemption', 'runtime': '142', 'plot': 'Two imprisoned men bond over a number of years, finding solace and eventual redemption through acts of common decency.', 'keyScene': 'Andy Dufresne escapes from Shawshank prison by crawling through a sewer pipe.', 'genre': 'Drama', 'released': '1994'}, {'title': 'Goodfellas', 'runtime': '146', 'plot': 'The story of Henry Hill and his life in the mob, covering his relationship with his wife Karen Hill and his mob partners Jimmy Conway and Tommy DeVito in the Italian-American crime syndicate.', 'keyScene': \"Joe Pesci's character Tommy DeVito shoots young Spider in the foot for not getting him a drink.\", 'genre': 'Biography, Crime, Drama', 'released': '1990'}, {'title': 'Se7en', 'runtime': '127', 'plot': 'Two detectives, a rookie and a veteran, hunt a serial killer who uses the seven deadly sins as his motives.', 'keyScene': \"Brad Pitt's character David Mills shoots John Doe after he reveals that he murdered Mills' wife.\", 'genre': 'Crime, Drama, Mystery, Thriller', 'released': '1995'}, {'title': 'The Silence of the Lambs', 'runtime': '118', 'plot': 'A young F.B.I. cadet must receive the help of an incarcerated and manipulative cannibal killer to help catch another serial killer, a madman who skins his victims.', 'keyScene': \"Hannibal Lecter explains to Clarice Starling that he ate a census taker's liver with some fava beans and a nice Chianti.\", 'genre': 'Crime, Drama, Thriller', 'released': '1991'}, {'title': 'The Godfather', 'runtime': '175', 'plot': \"An organized crime dynasty's aging patriarch transfers control of his clandestine empire to his reluctant son.\", 'keyScene': \"James Caan's character Sonny Corleone is shot to death at a toll booth by a number of machine gun toting enemies.\", 'genre': 'Crime, Drama', 'released': '1972'}, {'title': 'The Departed', 'runtime': '151', 'plot': 'An undercover cop and a mole in the police attempt to identify each other while infiltrating an Irish gang in South Boston.', 'keyScene': \"Leonardo DiCaprio's character Billy Costigan is shot to death by Matt Damon's character Colin Sullivan.\", 'genre': 'Crime, Drama, Thriller', 'released': '2006'}, {'title': 'The Usual Suspects', 'runtime': '106', 'plot': 'A sole survivor tells of the twisty events leading up to a horrific gun battle on a boat, which began when five criminals met at a seemingly random police lineup.', 'keyScene': \"Kevin Spacey's character Verbal Kint is revealed to be the mastermind behind the crime, when his limp disappears as he walks away from the police station.\", 'genre': 'Crime, Mystery, Thriller', 'released': '1995'}]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(12, [])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from elasticsearch import helpers\n",
    "\n",
    "# Load data into a JSON object\n",
    "with open('data.json') as f:\n",
    "   data_json = json.load(f)\n",
    " \n",
    "print(data_json)\n",
    " \n",
    "# Prepare the documents to be indexed\n",
    "documents = []\n",
    "for doc in data_json:\n",
    "    documents.append({\n",
    "        \"_index\": \"blogs\",\n",
    "        \"_source\": doc,\n",
    "    })\n",
    " \n",
    "# Use helpers.bulk to index\n",
    "helpers.bulk(es, documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "09a05b77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>_score</th>\n",
       "      <th>fields.title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>W7_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.936852</td>\n",
       "      <td>[The Dark Knight]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZL_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.488973</td>\n",
       "      <td>[The Departed]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Xr_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.467606</td>\n",
       "      <td>[The Matrix]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Y7_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.466467</td>\n",
       "      <td>[The Godfather]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>YL_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.465213</td>\n",
       "      <td>[Goodfellas]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>X7_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.464054</td>\n",
       "      <td>[The Shawshank Redemption]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Wr_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.446297</td>\n",
       "      <td>[Pulp Fiction]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>XL_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.443128</td>\n",
       "      <td>[Fight Club]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Zb_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.431134</td>\n",
       "      <td>[The Usual Suspects]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Xb_SDIsByaLf0EuTEs2L</td>\n",
       "      <td>0.426258</td>\n",
       "      <td>[Inception]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    _id    _score                fields.title\n",
       "0  W7_SDIsByaLf0EuTEs2L  0.936852           [The Dark Knight]\n",
       "1  ZL_SDIsByaLf0EuTEs2L  0.488973              [The Departed]\n",
       "2  Xr_SDIsByaLf0EuTEs2L  0.467606                [The Matrix]\n",
       "3  Y7_SDIsByaLf0EuTEs2L  0.466467             [The Godfather]\n",
       "4  YL_SDIsByaLf0EuTEs2L  0.465213                [Goodfellas]\n",
       "5  X7_SDIsByaLf0EuTEs2L  0.464054  [The Shawshank Redemption]\n",
       "6  Wr_SDIsByaLf0EuTEs2L  0.446297              [Pulp Fiction]\n",
       "7  XL_SDIsByaLf0EuTEs2L  0.443128                [Fight Club]\n",
       "8  Zb_SDIsByaLf0EuTEs2L  0.431134        [The Usual Suspects]\n",
       "9  Xb_SDIsByaLf0EuTEs2L  0.426258                 [Inception]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "INDEX_NAME=\"blogs\"\n",
    "\n",
    "source_fields = [ \"id\", \"title\"]\n",
    "\n",
    "query = {\n",
    "  \"field\": \"text_embedding.predicted_value\",\n",
    "  \"k\": 10,\n",
    "  \"num_candidates\": 50,\n",
    "  \"query_vector_builder\": {\n",
    "    \"text_embedding\": {\"model_id\": \"sentence-transformers__all-minilm-l6-v2\",\n",
    "      \"model_text\": \"dark knight\"\n",
    "    }\n",
    "  }\n",
    "}\n",
    "\n",
    "response = es.search(\n",
    "    index=INDEX_NAME,\n",
    "    fields=source_fields,\n",
    "    knn=query,\n",
    "    source=False)\n",
    "\n",
    "\n",
    "results = pd.json_normalize(json.loads(json.dumps(response.body['hits']['hits'])))\n",
    "\n",
    "# shows the result\n",
    "results[['_id', '_score', 'fields.title']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb722be7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}