rekhta_parser.ipynb

{
 "cells": [
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJ0AAAA5CAYAAAAsqykuAAABRGlDQ1BJQ0MgUHJvZmlsZQAAKJFjYGASSSwoyGFhYGDIzSspCnJ3UoiIjFJgf8LAwyAGhDwMEonJxQWOAQE+QCUMMBoVfLvGwAiiL+uCzDI4uuSIqPiv/9X8L212njonjakeBXClpBYnA+k/QJyaXFBUwsDAmAJkK5eXFIDYHUC2SBHQUUD2HBA7HcLeAGInQdhHwGpCgpyB7BtAtkByRiLQDMYXQLZOEpJ4OhIbai8IcDsGhwYrhAUbGVsScC0ZoCS1ogREO+cXVBZlpmeUKDgCQylVwTMvWU9HwcjAyICBARTmENWfb4DDklGMAyFWtJOBwbqIgYH5MUIs+gMDw4atQG/8RYhp6DIwCN9kYNjnW5BYlAh3AOM3luI0YyMIm3s7AwPrtP//P4czMLBrMjD8vf7//+/t////XQY0/xYDw4FvAOKkYeSnpfBgAAACA2lUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNS40LjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczpleGlmPSJodHRwOi8vbnMuYWRvYmUuY29tL2V4aWYvMS4wLyIKICAgICAgICAgICAgeG1sbnM6dGlmZj0iaHR0cDovL25zLmFkb2JlLmNvbS90aWZmLzEuMC8iPgogICAgICAgICA8ZXhpZjpQaXhlbFlEaW1lbnNpb24+NTc8L2V4aWY6UGl4ZWxZRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+MTU3PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcmllbnRhdGlvbj4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+CpbbJMIAAAq1SURBVHgB7VsJUBVHGv5BEEQUhIhHVCjAeANuxN1wCCquETaaKLFi5PBIooAmArqJJouUK6hopFIRzbGpeKGAF945Sl1UlBWzUWOJ90W0xPvCE539/17m1XvzZh7XezOY13/VY2b6+Lv776+7/6OxEZCAE5eAihKwVbEt3hSXAJMABx0HguoS4KBTXeS8QQ46jgHVJcBBp7rIeYMcdBwDqkuAg051kfMGOeg4BlSXAAed6iLnDXLQcQyoLgEOOtVFzhvkoOMYUF0CHHSqi5w3yEHHMaC6BDjoVBc5b5CDjmNAdQlw0Kkuct4gBx3HgOoS4KBTXeS8Qbs/mgiKdu+GSZMnw9OnT9nQbt+5w55PnjyB+5WV7H1tfj4MfeONeg89NDwcrl29yuo/fPgQHiNvolu3bwP9y4m3lxccP3aMpZn7z6lTp+BKRQWEhoSYm7Vq/P5woOvRowfkLFrEBLhy5UpYumKF2YU5b84cqKqqgitXrkBMfLzZ+ZtiOGv2bGjTpg0HnSkhqZ33krs79AsNZc3u3rPHIs0HvfYa43vhwgWL8FdievjwYShYuxYmJyUpFXkh0rlO90JME8DVa9cgJi6OHd8vSJcVu2nR4/XevXtMSC1btmQd+P3SJTh9+jTY29uDv58fODs7K3bs+PHjsHnLFig9eBDOnDkDxIPq3bp1CxybNQP/Xr0gODgYoiIjWZ4iIwtmODg61sid+rtp82Yo3rcPysvL4TbqfTRuDw8PCPD3hxHDh4O3t7cinwcPHsDGTZtgZno6nL94UbGcXMaNmzehtLQUjqF+ee7cOahAPfTu3bvw/PlzaN68Obi1agU9e/aEoKAg+HPfvnIsLJJmUdC998EHTPd4ffBg+PsnnzBBB/bpA3dQuc+aPx/64PsMTG/SpIlucHv27oW0mTOheP9+lubr4wOk+Hfv3p19P378GDJQp5qblQXffPcdtHJ1heSPPoLUlBQGSh2jOryQYh5XC91samoq/HXQIB1nRwcH3bv0hQyKnMWL4Ztvv4XQ0BDwQWA5NG0Ka9atg5sIRCJ6/wzH+t64sZD9+UJoivlEtND+kZYGN27cgJNoODx89Iili38KCwvhyJEj4ifExcRADP5EWot8/4WycXNzgxBcmH64wGmB2trawvnz52H7Dz9A3po1UFVtbFG9rq+8ApkZGfC3qCiRjeWe9B/+lqBDhw4JTZ2chI+nTxeCQkMFtLqEkydPCsH9+gn2zZoJXbp1Ezp6egrps2ax5nH1CZ+lpbE8yhd/aI3Kdi965EhdGSo7JCpKqKysNCj7z4wMgzIiT9w5DMohkIX9JSVCXn6+MCAiQlfHycVFeD0yUliUkyPs3LVLQBAY1MNdTFdW5E1PGltySoqQnJoq4E5lUOdiebng7etrVG/U6NG6cjQOBIeAu6NROeKfOm0ay6cy9KN+iISGBqvTyctLCAoJEWajDGh8Uvp3UZHg7OpqxD931SppUbN/0/FndsJtXPALCGADcnV3F/YWF7M2QsLCjAbZrkMHlpeWnm6UR0JTov8cOGBUfsy4cQbFaws6/UrTP/1UcHFzE+hJ4zBFSqBzcHYWEiclKVbFXcio7wSmLVu3GtQhQOmDWXyncckRuoVkgUTAk6PY+Hgj/jQf0oUiV7chaWY1JMiNkF9QAH9B667sxAm2PdPxGIw6A9EB1C+kZIdHK+6K7LiU5g3GY1mJ+rz6KrhU64pimdzVq6G+FivpTrgrw7Zt26CkuBgy0TXh0bq1yLpOz+fPnkHixATFOgP695fNoyOxoURHuJQqUH2QI0cZnfQ6Hun//fVXueJmS6s36Ei3wmOH/ebMnQtjx48Hn86dIXbMGPj98mVdB/WdmCHV4NNl4ktSYiLjgStHP5m9d/b1NUoTE0g/6dihg/ipe1Kf6kqkqPdGEGd/8QX0Dw+Hrl271pWFQXkHBweTPFqhAi9HB1GXawiRofUljkF/MfZCvyXpu1KiDWDd+vXSZPaNapBsurkSG2xIXEKAfZ6drdif9u3b6/JWLF8O02fMgD3oP3NFA2Dc2LEMdL6oxMqRO/rcTNEz3FGkhLoXs5htbGykWUbfZNGlTJ0KW7dv1+XlfPUVhCPwhg0dqkur64sr7sD6xlFt61egW6ShNOqdd+DNYcMAj2agncwLoyOiLFBXZEZKARoRBHDyIOxFq1pKVM6SVG/Q0WqehE5KsqJMgU5/C2/frh0s+/57o/GUoytFjlC5NmmRUthJSnfRTUPWMYFaiR6hNZiJFnDWggXwAMNYUpqIuy+5Mzw9PaVZtfomd4SWRHI5/NtvcPToUeYqIVfNJZQxuVDIxRQXG8t2xCnJybKgI1XDklRv0JmzU82dnKBSZqCJCQnw9ogRdW6qRYsWJusQqEjXjEYf2fLcXKOyNDmj0RG78+efdW4Mo0KNMOH+/fuQhHHnPNSr9dUVW9SbU9Ct9CHmtcUQmkhKfsYneq4Usaw5n40CdLSriH45/cHREUU+ptoSOT3JAV3T0UbH+tzMTDYxZeiELv3lF6MmSOf5GH2I2QsXGuU11oQPp0yB1ejTlFLusmXMCS1NJ3lpQfU2JMzZ2XdHjZJlV1JSIpuulLgNdTO6YVITUWzWzs6OHd3Lly6Flgo7I+l36zdsqImdxfLt0DCQIzmwUKRhlQzg/hQQIAs44ntZz+CTa8dSaY0CdGMwGhCI1qOUdhUVQVlZmTRZ9pv0EIpkpKCeUhfywWM2G3U7JUpAvZUMDi2IjkJ7GRcIAUxKpMuSq0ZKSpYyXfXaJ2NESOtb4rtRgI5M/fy8POjWpYvBGEkvicej8Nr16wbp0g8Khg976y0gv154WJg0u8bvWFSsR0ZHy5YjpXw05pOLSEouLi7SJLN+k5oQJnNvjixPUWejxVa4cSO0a9sWOr78slH7GGlhKoc0g8KIdEVKC7LBzgv1aXgkmuZEZCnSjqREfhhQFgPa09A9QbFXJaILAuhtZ3FD8cIllSVhkqUcOWQIdOrUCchyJicm7YJb0Zm7AY/AZNRnEiZOZAAVj9gT6KAWndT6bVJwmyaJ4rnpGONcvGQJC6qbGkffwEAg65usPwwhgehWKEQfn5Sc0XqNGDiQJbdu/RLkfPn/+33ky6R6dMFU302jX//NalcNWc7z580DuviAoTi4jHf39CkMVQR/1IV37NgBWVhuUEQEi6m+izFYqVHWHf2OCzDWTeXp8gSNly4c0KWJ8RMm6LNl77T4u+CvADcCS1C9QUc3J+pKgThxNNk1EVlhP/70E+zctRNvSJSxmyn3ME10b7TAWxrkGCbQDBwwAKLRwhV3HapLAfHakAeudArg78PLBWdxMmpD5Dg+ffYsVFXfFq6pjhMCcDjuwkRkVda2nis6kMXgOy1silaQD5JcH3RTmaIlJM8J778PndEpL9JZ7NuSr7+GIlwYp/BGjwhAdwz+0+KnBReLrqjevXsDlTV1xOpfIhD5m+NZb9CZo3HOwzol0Ch0OusUvfWOmoPOeudes5Fz0GkmeuttmIPOeudes5Fz0GkmeuttmIPOeudes5Fz0GkmeuttmIPOeudes5Fz0GkmeuttmIPOeudes5Fz0GkmeuttmIPOeudes5Fz0GkmeuttmIPOeudes5Fz0GkmeuttmIPOeudes5Fz0Gkmeutt+H8xpNH/8HPtYwAAAABJRU5ErkJggg=="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All data credits belong to the wonderful work done by **Rekhta foundation**. \n",
    "Data has been parsed into Urdu, Hindi and English translieration thanks to their excellent data organization.\n",
    "Consider supporting them for their great work in pushing the urdu language.\n",
    "\n",
    "![image.png](attachment:image.png)\n",
    "\n",
    "Credits to these authors for their wonderful original creations:\n",
    "\n",
    "*'mirza-ghalib','allama-iqbal','faiz-ahmad-faiz','sahir-ludhianvi','meer-taqi-meer',\n",
    "'dagh-dehlvi','kaifi-azmi','gulzar','bahadur-shah-zafar','parveen-shakir',\n",
    "'jaan-nisar-akhtar','javed-akhtar','jigar-moradabadi','jaun-eliya',\n",
    " 'ahmad-faraz','meer-anees','mohsin-naqvi','firaq-gorakhpuri','fahmida-riaz','wali-mohammad-wali',\n",
    " 'waseem-barelvi','akbar-allahabadi','altaf-hussain-hali','ameer-khusrau','naji-shakir','naseer-turabi',\n",
    " 'nazm-tabatabai','nida-fazli','noon-meem-rashid', 'habib-jalib'*\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from bs4.element import Comment\n",
    "import urllib.request\n",
    "\n",
    "\n",
    "# credits to this stackoverflow answer https://stackoverflow.com/questions/1936466/beautifulsoup-grab-visible-webpage-text\n",
    "\n",
    "allowed_sections=['style', 'script', 'head', 'title', 'meta', '[document]']\n",
    "\n",
    "\n",
    "def tag_visible(element):\n",
    "    if element.parent.name in allowed_sections:\n",
    "        return False\n",
    "    if isinstance(element, Comment):\n",
    "        return False\n",
    "    return True\n",
    "\n",
    "\n",
    "def text_from_html(body):\n",
    "    soup = BeautifulSoup(body, 'html.parser')\n",
    "    texts = soup.findAll(text=True)\n",
    "    visible_texts = filter(tag_visible, texts)  \n",
    "    return u\" \".join(t.strip() for t in visible_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "#captures the different <p> tags as newlines\n",
    "def text_with_newlines(elem):\n",
    "    text = ''\n",
    "    for e in elem.descendants:\n",
    "        if isinstance(e, str):\n",
    "            text += e\n",
    "        elif e.name == 'br' or e.name == 'p':\n",
    "            text += '\\n'\n",
    "    return text\n",
    "\n",
    "\n",
    "def parse_ghazal(url):\n",
    "    html = urllib.request.urlopen(url).read()\n",
    "    soup= BeautifulSoup(html, 'html.parser')\n",
    "    mydivs = soup.find(\"div\", {\"class\": \"pMC\"})\n",
    "    \n",
    "    #this section removes some of the English translations present on the webpage\n",
    "    # mixing language data would add noise, and make it difficult for the model to learn\n",
    "    #BUT in future these urdu to english translations could be a valuable resource to prepare machine translation data \n",
    "    for div in mydivs.find_all(\"div\", {'class':'t'}): \n",
    "        div.decompose()\n",
    "    \n",
    "    mydivs= text_with_newlines(mydivs)\n",
    "    return mydivs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "sitāroñ se aage jahāñ aur bhī haiñ \n",
      "abhī ishq ke imtihāñ aur bhī haiñ \n",
      "tahī zindagī se nahīñ ye fazā.eñ \n",
      "yahāñ saikḌoñ kārvāñ aur bhī haiñ \n",
      "qanā.at na kar ālam-e-rañg-o-bū par \n",
      "chaman aur bhī āshiyāñ aur bhī haiñ \n",
      "agar kho gayā ik nasheman to kyā ġham \n",
      "maqāmāt-e-āh-o-fuġhāñ aur bhī haiñ \n",
      "tū shāhīñ hai parvāz hai kaam terā \n",
      "tire sāmne āsmāñ aur bhī haiñ \n",
      "isī roz o shab meñ ulajh kar na rah jā \n",
      "ki tere zamān o makāñ aur bhī haiñ \n",
      "ga.e din ki tanhā thā maiñ anjuman meñ \n",
      "yahāñ ab mire rāz-dāñ aur bhī haiñ \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\n    okay I have a problem the way this urdu font is rendered. It is absolutely terrible to read.\\n    There have to be better font options.\\n'"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_english='https://www.rekhta.org/ghazals/sitaaron-se-aage-jahaan-aur-bhii-hain-allama-iqbal-ghazals-1'\n",
    "url_urdu='https://www.rekhta.org/ghazals/sitaaron-se-aage-jahaan-aur-bhii-hain-allama-iqbal-ghazals-1?lang=ur'\n",
    "url_hindi='https://www.rekhta.org/ghazals/sitaaron-se-aage-jahaan-aur-bhii-hain-allama-iqbal-ghazals-1?lang=hi'\n",
    "\n",
    "ghazal = parse_ghazal(url_english)\n",
    "print(ghazal)\n",
    "\n",
    "'''\n",
    "    okay I have a problem the way this urdu font is rendered. It is absolutely terrible to read.\n",
    "    There have to be better font options.\n",
    "'''\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsed text samples\n",
    "\n",
    "### English transliteration\n",
    "\n",
    "sitāroñ se aage jahāñ aur bhī haiñ abhī ishq ke imtihāñ aur bhī haiñ\n",
    "\n",
    "tū shāhīñ hai parvāz hai kaam terā tire sāmne āsmāñ aur bhī haiñ \n",
    "\n",
    "isī roz o shab meñ ulajh kar na rah jā ki tere zamān o makāñ aur bhī haiñ\n",
    "\n",
    "### Urdu \n",
    "\n",
    "ستاروں سے آگے جہاں اور بھی ہیں\n",
    "\n",
    "تو شاہیں ہے پرواز ہے کام تیرا ترے سامنے آسماں اور بھی ہیں\n",
    "\n",
    "اسی روز و شب میں الجھ کر نہ رہ جا کہ تیرے زمان و مکاں اور بھی ہیں\n",
    "\n",
    "\n",
    "### Hindi \n",
    "सितारों से आगे जहाँ और भी हैं अभी इश्क़ के इम्तिहाँ और भी हैं\n",
    "\n",
    "तू शाहीं है परवाज़ है काम तेरा तिरे सामने आसमाँ और भी हैं\n",
    "\n",
    "इसी रोज़ ओ शब में उलझ कर न रह जा कि तेरे ज़मान ओ मकाँ और भी हैं"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Observing the structure of urls to make sure the same script would work across different poets on the webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "hazāroñ ḳhvāhisheñ aisī ki har ḳhvāhish pe dam nikle \n",
      "bahut nikle mire armān lekin phir bhī kam nikle \n",
      "Dare kyuuñ merā qātil kyā rahegā us kī gardan par \n",
      "vo ḳhuuñ jo chashm-e-tar se umr bhar yuuñ dam-ba-dam nikle \n",
      "nikalnā ḳhuld se aadam kā sunte aa.e haiñ lekin \n",
      "bahut be-ābrū ho kar tire kūche se ham nikle \n",
      "bharam khul jaa.e zālim tere qāmat kī darāzī kā \n",
      "agar is turra-e-pur-pech-o-ḳham kā pech-o-ḳham nikle \n",
      "magar likhvā.e koī us ko ḳhat to ham se likhvā.e \n",
      "huī sub.h aur ghar se kaan par rakh kar qalam nikle \n",
      "huī is daur meñ mansūb mujh se bāda-ashāmī \n",
      "phir aayā vo zamāna jo jahāñ meñ jām-e-jam nikle \n",
      "huī jin se tavaqqo ḳhastagī kī daad paane kī \n",
      "vo ham se bhī ziyāda ḳhasta-e-teġh-e-sitam nikle \n",
      "mohabbat meñ nahīñ hai farq jiine aur marne kā \n",
      "usī ko dekh kar jiite haiñ jis kāfir pe dam nikle \n",
      "kahāñ mai-ḳhāne kā darvāza 'ġhālib' aur kahāñ vaa.iz \n",
      "par itnā jānte haiñ kal vo jaatā thā ki ham nikle \n"
     ]
    }
   ],
   "source": [
    "#The folder structure has urls as https://www.rekhta.org/ghazals/\n",
    "#https://www.rekhta.org/ghazals/tire-ishq-kii-intihaa-chaahtaa-huun-allama-iqbal-ghazals\n",
    "#Iqbal\n",
    "url='https://www.rekhta.org/ghazals/tire-ishq-kii-intihaa-chaahtaa-huun-allama-iqbal-ghazals'\n",
    "url= 'https://www.rekhta.org/ghazals/kabhii-ai-haqiiqat-e-muntazar-nazar-aa-libaas-e-majaaz-men-allama-iqbal-ghazals'\n",
    "\n",
    "##Ghalib's ghazals\n",
    "## there are currently 234 ghazals by this poet on the page\n",
    "## which makes it a rich resource for training a text model\n",
    "url= 'https://www.rekhta.org/ghazals/hazaaron-khvaahishen-aisii-ki-har-khvaahish-pe-dam-nikle-mirza-ghalib-ghazals'\n",
    "url ='https://www.rekhta.org/ghazals/hazaaron-khvaahishen-aisii-ki-har-khvaahish-pe-dam-nikle-mirza-ghalib-ghazals?lang=ur'\n",
    "url='https://www.rekhta.org/ghazals/hazaaron-khvaahishen-aisii-ki-har-khvaahish-pe-dam-nikle-mirza-ghalib-ghazals?lang=hi'\n",
    "url='https://www.rekhta.org/ghazals/har-ek-baat-pe-kahte-ho-tum-ki-tuu-kyaa-hai-mirza-ghalib-ghazals'\n",
    "\n",
    "url='https://www.rekhta.org/ghazals/ishq-mujh-ko-nahiin-vahshat-hii-sahii-mirza-ghalib-ghazals'\n",
    "url='https://www.rekhta.org/ghazals/ishq-mujh-ko-nahiin-vahshat-hii-sahii-mirza-ghalib-ghazals?lang=ur'\n",
    "url='https://www.rekhta.org/ghazals/ishq-mujh-ko-nahiin-vahshat-hii-sahii-mirza-ghalib-ghazals?lang=hi'\n",
    "\n",
    "url='https://www.rekhta.org/ghazals/koii-din-gar-zindagaanii-aur-hai-mirza-ghalib-ghazals'\n",
    "url='https://www.rekhta.org/ghazals/koii-din-gar-zindagaanii-aur-hai-mirza-ghalib-ghazals?lang=ur'\n",
    "\n",
    "url='https://www.rekhta.org/ghazals/hai-bazm-e-butaan-men-sukhan-aazurda-labon-se-mirza-ghalib-ghazals'\n",
    "url='https://www.rekhta.org/ghazals/hai-bazm-e-butaan-men-sukhan-aazurda-labon-se-mirza-ghalib-ghazals?lang=ur'\n",
    "\n",
    "url='https://www.rekhta.org/ghazals/ghar-jab-banaa-liyaa-tire-dar-par-kahe-bagair-mirza-ghalib-ghazals'\n",
    "url='https://www.rekhta.org/ghazals/ghar-jab-banaa-liyaa-tire-dar-par-kahe-bagair-mirza-ghalib-ghazals?lang=hi'\n",
    "url='https://www.rekhta.org/ghazals/ghar-jab-banaa-liyaa-tire-dar-par-kahe-bagair-mirza-ghalib-ghazals?lang=ur'\n",
    "\n",
    "## Sahir Ludhianvi\n",
    "url='https://www.rekhta.org/ghazals/kabhii-khud-pe-kabhii-haalaat-pe-ronaa-aayaa-sahir-ludhianvi-ghazals'\n",
    "url='https://www.rekhta.org/ghazals/kabhii-khud-pe-kabhii-haalaat-pe-ronaa-aayaa-sahir-ludhianvi-ghazals?lang=ur'\n",
    "url= 'https://www.rekhta.org/ghazals/kabhii-khud-pe-kabhii-haalaat-pe-ronaa-aayaa-sahir-ludhianvi-ghazals?lang=hi'\n",
    "\n",
    "\n",
    "\n",
    "## Faiz\n",
    "en_url='https://www.rekhta.org/ghazals/gulon-men-rang-bhare-baad-e-nau-bahaar-chale-faiz-ahmad-faiz-ghazals'\n",
    "hindi_url='https://www.rekhta.org/ghazals/gulon-men-rang-bhare-baad-e-nau-bahaar-chale-faiz-ahmad-faiz-ghazals?lang=hi'\n",
    "urdu_url='https://www.rekhta.org/ghazals/gulon-men-rang-bhare-baad-e-nau-bahaar-chale-faiz-ahmad-faiz-ghazals?lang=ur'\n",
    "\n",
    "\n",
    "\n",
    "url='https://www.rekhta.org/ghazals/hazaaron-khvaahishen-aisii-ki-har-khvaahish-pe-dam-nikle-mirza-ghalib-ghazals'#?lang=ur'\n",
    "ghazal = parse_ghazal(url)\n",
    "print(ghazal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [],
   "source": [
    "#testing the homepage for parsing all poet names now\n",
    "#for ghalib\n",
    "\n",
    "def parse_webpage_at_given_scroll(html):\n",
    "    ctr=0\n",
    "    soup= BeautifulSoup(html, 'html.parser')\n",
    "    mydivs = soup.find(\"div\", {\"class\": \"contentListBody\"})\n",
    "    titles=[]\n",
    "    for a in mydivs.find_all('a', href=True):\n",
    "        t=a['href']\n",
    "        if t not in titles:\n",
    "            if ctr%5==0:\n",
    "                print(\"Found the URL:\", t)\n",
    "            titles.append(t)\n",
    "            ctr+=1\n",
    "    print('=============================')    \n",
    "    print('number of titles',len(titles))\n",
    "    print('=============================')\n",
    "    \n",
    "    return titles\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [],
   "source": [
    "#language argument can be ur or hi for urdu or hindi\n",
    "def read_and_write_web(author,language='ur'):\n",
    "    \n",
    "    lang=language\n",
    "    #author='mirza-ghalib'\n",
    "    author_lan=author+'/'+lang\n",
    "    if not os.path.exists(author_lan):\n",
    "        os.makedirs(author_lan)\n",
    "\n",
    "    for url in titles:\n",
    "        name_poem=url.split('https://www.rekhta.org/ghazals/')[1]\n",
    "        path_poem= author_lan+'/'+name_poem\n",
    "        if os.path.exists(path_poem):\n",
    "            pass\n",
    "        else:\n",
    "            f= open(path_poem,\"w+\")\n",
    "            if lang=='en':\n",
    "                    url_for_lang= url\n",
    "            else:\n",
    "                url_for_lang= url+'?lang='+lang\n",
    "            ghazal = parse_ghazal(url_for_lang)\n",
    "            f.write(ghazal)\n",
    "            f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found the URL: https://www.rekhta.org/ghazals/aah-ko-chaahiye-ik-umr-asar-hote-tak-mirza-ghalib-ghazals\n",
      "Found the URL: https://www.rekhta.org/ghazals/har-ek-baat-pe-kahte-ho-tum-ki-tuu-kyaa-hai-mirza-ghalib-ghazals\n",
      "Found the URL: https://www.rekhta.org/ghazals/rahiye-ab-aisii-jagah-chal-kar-jahaan-koii-na-ho-mirza-ghalib-ghazals\n",
      "Found the URL: https://www.rekhta.org/ghazals/muddat-huii-hai-yaar-ko-mehmaan-kiye-hue-mirza-ghalib-ghazals\n",
      "Found the URL: https://www.rekhta.org/ghazals/gair-len-mahfil-men-bose-jaam-ke-mirza-ghalib-ghazals\n",
      "Found the URL: https://www.rekhta.org/ghazals/sab-kahaan-kuchh-laala-o-gul-men-numaayaan-ho-gaiin-mirza-ghalib-ghazals\n",
      "Found the URL: https://www.rekhta.org/ghazals/dil-se-tirii-nigaah-jigar-tak-utar-gaii-mirza-ghalib-ghazals\n",
      "Found the URL: https://www.rekhta.org/ghazals/diivaangii-se-dosh-pe-zunnaar-bhii-nahiin-mirza-ghalib-ghazals\n",
      "Found the URL: https://www.rekhta.org/ghazals/jahaan-teraa-naqsh-e-qadam-dekhte-hain-mirza-ghalib-ghazals\n",
      "Found the URL: https://www.rekhta.org/ghazals/main-unhen-chheduun-aur-kuchh-na-kahen-mirza-ghalib-ghazals\n",
      "=============================\n",
      "number of titles 50\n",
      "=============================\n"
     ]
    }
   ],
   "source": [
    "##Parsing based on home page of authors\n",
    "url_base='https://www.rekhta.org/poets/'\n",
    "\n",
    "## TODO : Later\n",
    "## or just iterate through the list of all poets on the index, instead of hand curated list\n",
    "authors=['mirza-ghalib','allama-iqbal','faiz-ahmad-faiz','sahir-ludhianvi','meer-taqi-meer',\n",
    "         'dagh-dehlvi','kaifi-azmi','gulzar','bahadur-shah-zafar','parveen-shakir',\n",
    "         'jaan-nisar-akhtar','javed-akhtar','jigar-moradabadi','jaun-eliya',\n",
    "         'ahmad-faraz','meer-anees','mohsin-naqvi','firaq-gorakhpuri','fahmida-riaz','wali-mohammad-wali',\n",
    "        'waseem-barelvi','akbar-allahabadi','altaf-hussain-hali','ameer-khusrau','naji-shakir','naseer-turabi'\n",
    "        ,'nazm-tabatabai','nida-fazli','noon-meem-rashid','habib-jalib']\n",
    "\n",
    "\n",
    "\n",
    "for author in authors:\n",
    "    url_home_page= url_base +author+ '/ghazals'\n",
    "    html = urllib.request.urlopen(url_home_page).read()\n",
    "    titles= parse_webpage_at_given_scroll(html)\n",
    "    read_and_write_web(author,'en')\n",
    "    read_and_write_web(author,'ur')\n",
    "    read_and_write_web(author,'hi')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "ہم کو مٹا سکے یہ زمانے میں دم نہیں \n",
    "ہم سے زمانہ خود ہے زمانے سے ہم نہیں\n",
    "\n",
    "हम को मिटा सके ये ज़माने में दम नहीं \n",
    "हम से ज़माना ख़ुद है ज़माने से हम नहीं \n",
    "            \n",
    "            Jigar Moradabadi\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}