NHS Jobtrac webscraper.ipynb

{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "# This is an NHS Jobtrac webscraper, it will pull all job details other than hiring manager details (they are censored against webscraping even if this code was changed to allow this)\n",
    "\n",
    "This is a neutered version of another webscraper, that extracts all hiring manager details, including Mobile numbers, Names, emails ect. This will not be published due to the nature of it.\n",
    "\n",
    "\n",
    "When code is run, input the below for it to generate an example of how it works.\n",
    "https://www.nhsjobs.com/job_list?JobSearch_q=&JobSearch_d=&JobSearch_g=&JobSearch_re=_POST&JobSearch_re_0=1&JobSearch_re_1=1-_-_-&JobSearch_re_2=1-_-_--_-_-&JobSearch_Submit=Search&_tr=JobSearch&_ts=1336&_pg="
   ],
   "id": "8a77807f92f26ee"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "import sys\n",
    "import os\n",
    "sys.path.append(os.path.join(os.getcwd(), 'relative/path/to/package'))"
   ],
   "id": "1a7895034a37fc53",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from urllib.parse import urljoin, urlparse\n",
    "from lxml import html\n",
    "import datetime\n",
    "import tkinter\n",
    "from tkinter import simpledialog\n",
    "from tkinter import filedialog\n",
    "\n",
    "#The below is used for the full version:\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.chrome.service import Service as ChromeService\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "import openpyxl\n",
    "import time\n",
    "import random"
   ],
   "id": "7c74094a99fac77e"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "\n",
    "base_url = simpledialog.askstring(\"Input\", \"Enter the NHS Jobs URL (without page number):\")\n",
    "base_url += \"pg=\"  #add \"&page=\" part to URL\n",
    "num_pages_str = simpledialog.askstring(\"Input\", \"Enter the number of pages you want to extract:\")\n",
    "num_pages = int(num_pages_str) if num_pages_str is not None else 1\n",
    "# Define the XPath expressions for data extraction\n",
    "xpaths = {\n",
    "    \"employer_name\": '//*[@id=\"hj-job-summary\"]/div/div/div/dl[2]/dd[1]',\n",
    "    \"job_title\": '//*[@id=\"hj-enquiry-html\"]/div/dl/dd[2]',\n",
    "    'area recruiting for': '//*[@id=\"hj-job-summary\"]/div/div/div/dl[1]/dd[1]',\n",
    "    'job recruiting for': '//*[@id=\"hj-job\"]/div/div[1]/h2',\n",
    "    'recruitment close date': '//*[@id=\"hj-job-summary\"]/div/div/div/dl[2]/dd[7]',\n",
    "    'contract': '//*[@id=\"hj-job-summary\"]/div/div/div/dl[1]/dd[3]',\n",
    "    'reference': '//*[@id=\"hj-job-summary\"]/div/div/div/dl[1]/dd[5]',\n",
    "    'Hours': '//*[@id=\"hj-job-summary\"]/div/div/div/dl[1]/dd[4]',\n",
    "    'address line 1': '//*[@id=\"hj-job-summary\"]/div/div/div/dl[2]/dd[3]',\n",
    "    'town': '//*[@id=\"hj-job-summary\"]/div/div/div/dl[2]/dd[4]',\n",
    "    'salary': '//*[@id=\"hj-job-summary\"]/div/div/div/dl[2]/dd[5]',\n",
    "    'Band': '//*[@id=\"hj-job-summary\"]/div/div/div/dl[1]/dd[2]',\n",
    "    'personal spec': '//*[@id=\"hj-job-role-requirement\"]/div',\n",
    "    'job description': '//*[@id=\"hj-job-advert\"]'\n",
    "\n",
    "\n",
    "}"
   ],
   "id": "48db041c52cbd6d"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "dataframes = [] # initialise empty dataframe",
   "id": "7e9bc78adbd4af45"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "\n",
    "session = requests.Session()\n",
    "session.max_redirects = 2  # Sets a limit on the number of redirects\n",
    "# Goes through url, adds page numbers and extracts xpath data from all ads on those pages\n",
    "for page in range(1, num_pages + 1):\n",
    "    url = base_url + str(page)\n",
    "    response = session.get(url)\n",
    "    if response.status_code == 200:\n",
    "        print(f\" successfully located\" +url + '/'+num_pages_str)\n",
    "        soup = BeautifulSoup(response.text, \"html.parser\")\n",
    "        job_adverts = soup.find_all(\"a\", href=True, attrs={\"href\": \"/job/UK/\"})\n",
    "\n",
    "        for advert in job_adverts:\n",
    "            advert_relative_url = advert[\"href\"]\n",
    "            if \"/job/UK/\" in advert_relative_url:\n",
    "                advert_url = urljoin(base_url, advert_relative_url)\n",
    "                url_parts = urlparse(advert_url)\n",
    "                if url_parts.scheme:\n",
    "                    try:\n",
    "                        advert_response = session.get(advert_url)\n",
    "                        advert_response.raise_for_status()\n",
    "                        advert_tree = html.fromstring(advert_response.text)\n",
    "                        advert_data = {}\n",
    "                        for key, xpath in xpaths.items():\n",
    "                            elements = advert_tree.xpath(xpath)\n",
    "                            if elements:\n",
    "                                advert_data[key] = elements[0].text_content()\n",
    "                            else:\n",
    "                                print(f\"No specified XPath elements detected for {key}\")\n",
    "                        dataframes.append(pd.DataFrame(advert_data, index=[0]))\n",
    "                    except requests.exceptions.HTTPError as http_err:\n",
    "                        print(f\"HTTP error occurred for URL {advert_url}: {http_err}\")\n",
    "                    except Exception as err:\n",
    "                        print(f\"An error occurred for URL {advert_url}: {err}\")\n",
    "                else:\n",
    "                    print(f\"Invalid URL: {advert_url}\")"
   ],
   "id": "1b6a17346eaed3e2"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "\n",
    "# Concatenate dataframe\n",
    "if dataframes:\n",
    "    final_dataframe = pd.concat(dataframes, ignore_index=True)\n",
    "    # Setup the root Tkinter window\n",
    "    root = tkinter.Tk()\n",
    "    root.withdraw()  # Hide the root window\n",
    "    # Ask the user to select the save location and filename using file explorer\n",
    "    current_datetime = datetime.datetime.now()\n",
    "    default_filename = f\"job_adverts_{current_datetime.strftime('%Y%m%d_%H%M%S')}.xlsx\"\n",
    "    file_path = filedialog.asksaveasfilename(\n",
    "        initialdir=os.path.expanduser(\"~/Desktop\"),  # Set initial directory to the Desktop\n",
    "        title=\"Select file\",\n",
    "        filetypes=((\"Excel files\", \"*.xlsx\"), (\"All files\", \"*.*\")),\n",
    "        defaultextension=\".xlsx\",\n",
    "        initialfile=default_filename\n",
    "    )"
   ],
   "id": "1a0c6c21f24f808c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-08T14:30:23.502236Z",
     "start_time": "2024-09-08T14:29:53.614363Z"
    }
   },
   "cell_type": "code",
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for +=: 'NoneType' and 'str'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[1], line 24\u001B[0m\n\u001B[0;32m     21\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mrandom\u001B[39;00m\n\u001B[0;32m     23\u001B[0m base_url \u001B[38;5;241m=\u001B[39m simpledialog\u001B[38;5;241m.\u001B[39maskstring(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mInput\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mEnter the NHS Jobs URL (without page number):\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m---> 24\u001B[0m base_url \u001B[38;5;241m+\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpg=\u001B[39m\u001B[38;5;124m\"\u001B[39m  \u001B[38;5;66;03m#add \"&page=\" part to URL\u001B[39;00m\n\u001B[0;32m     25\u001B[0m num_pages_str \u001B[38;5;241m=\u001B[39m simpledialog\u001B[38;5;241m.\u001B[39maskstring(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mInput\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mEnter the number of pages you want to extract:\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m     26\u001B[0m num_pages \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mint\u001B[39m(num_pages_str) \u001B[38;5;28;01mif\u001B[39;00m num_pages_str \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;241m1\u001B[39m\n",
      "\u001B[1;31mTypeError\u001B[0m: unsupported operand type(s) for +=: 'NoneType' and 'str'"
     ]
    }
   ],
   "execution_count": 1,
   "source": [
    "\n",
    "    # Check if a file path was provided (i.e., user didn't cancel the dialog)\n",
    "    if file_path:\n",
    "        # Save the DataFrame to an Excel file at the specified path\n",
    "        final_dataframe.to_excel(file_path, index=False)\n",
    "        print(f\"File saved successfully at {file_path}\")\n",
    "    else:\n",
    "        print(\"File save cancelled.\")\n",
    "else:\n",
    "    simpledialog.messagebox.showinfo(\"Completion\", \"No data extracted.\")\n"
   ],
   "id": "fbc121e30a2defb3"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}