diff --git a/api/failing.json b/api/failing.json index eabc2063ba..d4023ef460 100644 --- a/api/failing.json +++ b/api/failing.json @@ -1 +1 @@ -[{"council_id":"BDG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:59:29] Fetching Scraper for: BDG handlers.py:23\n Begin attempting to scrape: BDG handlers.py:27\n[08:59:30] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:59:31] ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgov.lbbd.gov.uk/internet/mgWebService.asmx/GetC \n ouncillorsByWard \n[08:59:46] [Errno 110] Connection timed out handlers.py:36\n[08:59:47] Finished attempting to scrape: BDG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-12-12 08:59:29.650248","end":"2024-12-12 08:59:47.263901","duration":17}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:57:27] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n Deleting existing data... base.py:257\n[09:57:28] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:57:29] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-12-12 09:57:27.320535","end":"2024-12-12 09:57:29.515766","duration":2}},{"council_id":"BRT","missing":false,"latest_run":{"status_code":1,"log_text":"[10:17:50] Fetching Scraper for: BRT handlers.py:23\n[10:18:00] Begin attempting to scrape: BRT handlers.py:27\n[10:18:02] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:18:03] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.broxtowe.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[10:18:19] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BRT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-12-12 10:17:50.554943","end":"2024-12-12 10:18:19.545956","duration":28}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:11:54] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n[09:11:55] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:11:56] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[09:11:58] list index out of range handlers.py:36\n Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-12-12 09:11:54.631061","end":"2024-12-12 09:11:58.942507","duration":4}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:41:22] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n[08:41:23] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:41:24] ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[08:41:25] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n[08:41:26] Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-12-12 08:41:22.833352","end":"2024-12-12 08:41:26.279463","duration":3}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[09:18:54] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n Deleting existing data... base.py:257\n[09:18:55] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:18:56] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[09:18:57] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n[09:18:58] 'NoneType' object has no attribute 'find_parent' handlers.py:36\n Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-12-12 09:18:54.508813","end":"2024-12-12 09:18:58.245643","duration":3}},{"council_id":"EDU","missing":false,"latest_run":{"status_code":1,"log_text":"[08:56:28] Fetching Scraper for: EDU handlers.py:23\n Begin attempting to scrape: EDU handlers.py:27\n Deleting existing data... base.py:257\n[08:56:29] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:56:30] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.eastdunbarton.gov.uk/residents/council-democra \n cy/committees-and-councillors/councillors-2017 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.eastdunbarton.gov.uk/residents/council-de \n mocracy/committees-and-councillors/councillors-2017' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: EDU base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.eastdunbarton.gov.uk/residents/council-democracy/committees-and-councillors/councillors-2017'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-12-12 08:56:28.128005","end":"2024-12-12 08:56:30.500120","duration":2}},{"council_id":"ELS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:19:32] Fetching Scraper for: ELS handlers.py:23\n Begin attempting to scrape: ELS handlers.py:27\n Deleting existing data... base.py:257\n[08:19:33] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:19:34] ...data deleted. base.py:264\n Scraping from base.py:49\n https://cne-siar.gov.uk/home/your-council/council-members/ \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.cne-siar.gov.uk/home/your-council/council \n -members/' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n[08:19:35] Finished attempting to scrape: ELS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.cne-siar.gov.uk/home/your-council/council-members/'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-12-12 08:19:32.550330","end":"2024-12-12 08:19:35.018421","duration":2}},{"council_id":"FYL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:13:02] Fetching Scraper for: FYL handlers.py:23\n Begin attempting to scrape: FYL handlers.py:27\n[10:13:03] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:13:04] Getting all files in Councillors/json... base.py:209\n ...found 17 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 17 files in Councillors/raw base.py:225\n ...found 35 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 35 files base.py:236\n[10:13:05] ...data deleted. base.py:264\n Scraping from base.py:49\n https://fylde.cmis.uk.com/fylde/CouncillorsandMP.aspx \n[10:13:06] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[10:13:09] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[10:13:10] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:13] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:15] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[10:13:18] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[10:13:19] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:22] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:25] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:28] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:29] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[10:13:32] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[10:13:34] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:36] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:37] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[10:13:40] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[10:13:42] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:45] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:48] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:51] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:52] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:55] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[10:13:57] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[10:14:00] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[10:14:01] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[10:14:04] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[10:14:05] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[10:14:08] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[10:14:09] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[10:14:13] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[10:14:15] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[10:14:18] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[10:14:19] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[10:14:22] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[10:14:24] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/222/ScreenMode/Alphabetical/ \n Default.aspx \n[10:14:26] 'NoneType' object has no attribute 'parent' handlers.py:36\n Committing batch 1 consisting of 34 files base.py:297\n[10:14:27] Finished attempting to scrape: FYL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 300, in get_single_councillor\n party = self.get_party_name(list_page_html)\n File \"scrapers/FYL-fylde/councillors.py\", line 17, in get_party_name\n .parent.get_text(strip=True)\nAttributeError: 'NoneType' object has no attribute 'parent'\n","start":"2024-12-12 10:13:02.911904","end":"2024-12-12 10:14:27.796724","duration":84}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[09:32:24] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n Deleting existing data... base.py:257\n[09:32:25] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n[09:32:26] Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-12-12 09:32:24.079653","end":"2024-12-12 09:32:26.397431","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:06:21] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n[09:06:22] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:06:23] ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[09:06:38] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-12-12 09:06:21.187398","end":"2024-12-12 09:06:38.961683","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:23:29] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n[08:23:30] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:23:31] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[08:23:37] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n[08:23:38] Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-12-12 08:23:29.916586","end":"2024-12-12 08:23:38.277164","duration":8}},{"council_id":"KHL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:27:46] Fetching Scraper for: KHL handlers.py:23\n Begin attempting to scrape: KHL handlers.py:27\n[10:27:47] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:27:48] ...data deleted. base.py:264\n Scraping from base.py:49\n https://cmis.hullcc.gov.uk/cmis/CouncillorsandSeniorOffice \n rs/CouncillorsandSeniorOfficers.aspx \n [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify handlers.py:36\n failed: unable to get local issuer certificate \n (_ssl.c:1007) \n[10:27:49] Finished attempting to scrape: KHL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n","start":"2024-12-12 10:27:46.839324","end":"2024-12-12 10:27:49.146326","duration":2}},{"council_id":"MLN","missing":false,"latest_run":{"status_code":1,"log_text":"[10:35:42] Fetching Scraper for: MLN handlers.py:23\n Begin attempting to scrape: MLN handlers.py:27\n Deleting existing data... base.py:257\n[10:35:43] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 18 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 18 files in Councillors/raw base.py:225\n ...found 37 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 37 files base.py:236\n[10:35:44] ...data deleted. base.py:264\n Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors.aspx \n[10:35:45] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/141/ScreenMode/Alphabetical/ \n Default.aspx \n[10:35:46] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/244/ScreenMode/Alphabetical/ \n Default.aspx \n[10:35:48] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/143/ScreenMode/Alphabetical/ \n Default.aspx \n[10:35:50] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/144/ScreenMode/Alphabetical/ \n Default.aspx \n[10:35:53] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/245/ScreenMode/Alphabetical/ \n Default.aspx \n[10:35:55] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/19/ScreenMode/Alphabetical/D \n efault.aspx \n[10:35:57] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/140/ScreenMode/Alphabetical/ \n Default.aspx \n[10:35:58] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/240/ScreenMode/Alphabetical/ \n Default.aspx \n[10:36:00] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/234/ScreenMode/Alphabetical/ \n Default.aspx \n[10:36:01] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/241/ScreenMode/Alphabetical/ \n Default.aspx \n[10:36:03] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/20/ScreenMode/Alphabetical/D \n efault.aspx \n[10:36:05] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/22/ScreenMode/Alphabetical/D \n efault.aspx \n[10:36:07] 'title' handlers.py:36\n Committing batch 1 consisting of 24 files base.py:297\n[10:36:08] Finished attempting to scrape: MLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 300, in get_single_councillor\n party = self.get_party_name(list_page_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 281, in get_party_name\n list_page_html.find_all(\"img\")[-1][\"title\"]\n File \"/opt/python/bs4/element.py\", line 1930, in __getitem__\n return self.attrs[key]\nKeyError: 'title'\n","start":"2024-12-12 10:35:42.137893","end":"2024-12-12 10:36:08.476166","duration":26}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"STG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:36:40] Fetching Scraper for: STG handlers.py:23\n Begin attempting to scrape: STG handlers.py:27\n Deleting existing data... base.py:257\n[08:36:41] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 19 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 19 files in Councillors/raw base.py:225\n ...found 39 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 39 files base.py:236\n[08:36:42] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-elections/councillors/ \n[08:36:43] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/elaine-watterson/ \n[08:36:45] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gene-maxwell/ \n[08:36:46] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/martin-earl/ \n[08:36:47] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mcgarvey/ \n[08:36:48] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/paul-henke/ \n[08:36:49] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rosemary-fraser/ \n[08:36:51] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/alasdair-tollemache/ \n[08:36:52] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/david-wilson/ \n[08:36:53] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/robin-kleinman/ \n[08:36:54] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/thomas-heald/ \n[08:36:56] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/danny-gibson/ \n[08:36:57] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jim-thomson/ \n[08:36:58] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rachel-nunn/ \n[08:36:59] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/susan-mcgill/ \n[08:37:00] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jen-preston/ \n[08:37:03] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/neil-benny/ \n[08:37:04] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/scott-farmer/ \n[08:37:06] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/bryan-flannagan/ \n[08:37:07] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mclaughlan/ \n[08:37:08] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/willie-ferguson/ \n[08:37:09] 'NoneType' object is not subscriptable handlers.py:36\n Committing batch 1 consisting of 38 files base.py:297\n[08:37:10] Finished attempting to scrape: STG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/STG-stirling/councillors.py\", line 42, in get_single_councillor\n councillor.email = soup.select_one(\"a[href^=mailto]\")[\nTypeError: 'NoneType' object is not subscriptable\n","start":"2024-12-12 08:36:40.179718","end":"2024-12-12 08:37:10.979368","duration":30}},{"council_id":"SWD","missing":false,"latest_run":{"status_code":1,"log_text":"[10:43:18] Fetching Scraper for: SWD handlers.py:23\n Begin attempting to scrape: SWD handlers.py:27\n Deleting existing data... base.py:257\n[10:43:19] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:43:20] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.winchester.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify handlers.py:36\n failed: unable to get local issuer certificate \n (_ssl.c:1007) \n Finished attempting to scrape: SWD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n","start":"2024-12-12 10:43:18.370911","end":"2024-12-12 10:43:20.604967","duration":2}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:45:56] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n[08:45:57] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:45:58] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[08:46:01] 'NoneType' object has no attribute 'findNext' handlers.py:36\n Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-12-12 08:45:56.843418","end":"2024-12-12 08:46:01.339790","duration":4}},{"council_id":"VGL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:51:57] Fetching Scraper for: VGL handlers.py:23\n Begin attempting to scrape: VGL handlers.py:27\n Deleting existing data... base.py:257\n[09:51:58] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 16 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 16 files in Councillors/raw base.py:225\n ...found 33 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 33 files base.py:236\n[09:51:59] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Councillors.aspx \n[09:52:01] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Asbrey-Anne.aspx \n[09:52:03] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Aviet-Julie.aspx \n[09:52:04] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Ball-Gareth.aspx \n[09:52:06] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Birch-Rhiannon.aspx \n[09:52:09] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Brooks-Bronwen.aspx \n[09:52:11] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Bruce-Gillian.aspx \n[09:52:12] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Buckley-Ian.aspx \n[09:52:14] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Burnett-Lis.aspx \n[09:52:16] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Campbell-Samantha.aspx \n[09:52:17] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Carroll-George.aspx \n[09:52:19] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cave-Christine.aspx \n[09:52:21] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Champion-Charles.aspx \n[09:52:22] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Charles-Janice.aspx \n[09:52:23] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Collins-Amelia.aspx \n[09:52:25] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cowpe-Marianne.aspx \n[09:52:27] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Drake-Pamela.aspx \n[09:52:28] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Driscoll-Vincent.aspx \n[09:52:30] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 32 files base.py:297\n[09:52:32] Finished attempting to scrape: VGL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/VGL-the-vale-of-glamorgan/councillors.py\", line 36, in get_single_councillor\n email_text = soup.select_one(\"#S4_EmailPlaceholder\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-12-12 09:51:57.141916","end":"2024-12-12 09:52:32.016642","duration":34}},{"council_id":"WIN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:31:58] Fetching Scraper for: WIN handlers.py:23\n Begin attempting to scrape: WIN handlers.py:27\n Deleting existing data... base.py:257\n[09:31:59] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:32:00] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.winchester.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify handlers.py:36\n failed: unable to get local issuer certificate \n (_ssl.c:1007) \n Finished attempting to scrape: WIN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n","start":"2024-12-12 09:31:58.425972","end":"2024-12-12 09:32:00.562072","duration":2}},{"council_id":"WLN","missing":false,"latest_run":{"status_code":1,"log_text":"[10:27:52] Fetching Scraper for: WLN handlers.py:23\n Begin attempting to scrape: WLN handlers.py:27\n[10:27:53] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:27:54] Getting all files in Councillors/json... base.py:209\n ...found 25 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 25 files in Councillors/raw base.py:225\n ...found 51 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 51 files base.py:236\n[10:27:55] ...data deleted. base.py:264\n Scraping from https://westlothian.gov.uk/councillors base.py:49\n[10:27:56] Scraping from base.py:49\n https://westlothian.gov.uk/article/33888/Linlithgow \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33889/Broxburn-Uphall-a \n nd-Winchburgh \n[10:27:57] Scraping from base.py:49\n https://westlothian.gov.uk/article/33890/Livingston-North \n[10:27:58] Scraping from base.py:49\n https://westlothian.gov.uk/article/33891/Livingston-South \n[10:27:59] Scraping from base.py:49\n https://westlothian.gov.uk/article/33893/East-Livingston-a \n nd-East-Calder \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33892/Fauldhouse-and-th \n e-Breich-Valley \n[10:28:00] Scraping from base.py:49\n https://westlothian.gov.uk/article/33894/Whitburn-and-Blac \n kburn \n[10:28:01] Scraping from base.py:49\n https://westlothian.gov.uk/article/33895/Bathgate \n[10:28:02] Scraping from base.py:49\n https://westlothian.gov.uk/article/33896/Armadale-and-Blac \n kridge \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33897/Councillor-Tom-Co \n nn \n[10:28:03] Scraping from base.py:49\n https://westlothian.gov.uk/article/33898/Councillor-Paulin \n e-Orr \n[10:28:04] Scraping from base.py:49\n https://westlothian.gov.uk/article/33899/Councillor-Sally- \n Pattle \n[10:28:05] Scraping from base.py:49\n https://westlothian.gov.uk/article/34305/Councillor-Tony-B \n oyle \n[10:28:06] Scraping from base.py:49\n https://westlothian.gov.uk/article/34302/Councillor-Diane- \n Calder \n[10:28:07] Scraping from base.py:49\n https://westlothian.gov.uk/article/34303/Councillor-Janet- \n Campbell \n[10:28:10] Scraping from base.py:49\n https://westlothian.gov.uk/article/34304/Councillor-Angela \n -Doran-Timson \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34307/Councillor-Alison \n -Adamson \n[10:28:11] Scraping from base.py:49\n https://westlothian.gov.uk/article/34306/Councillor-Robert \n -De-Bold \n[10:28:12] Scraping from base.py:49\n https://westlothian.gov.uk/article/34308/Councillor-Anne-M \n cMillan \n[10:28:13] Scraping from base.py:49\n https://westlothian.gov.uk/article/34309/Councillor-Andrew \n -Miller \n[10:28:14] Scraping from base.py:49\n https://westlothian.gov.uk/article/34311/Councillor-Lawren \n ce-Fitzpatrick \n[10:28:15] Scraping from base.py:49\n https://westlothian.gov.uk/article/34310/Councillor-Peter- \n Heggie \n[10:28:16] Scraping from base.py:49\n https://westlothian.gov.uk/article/34312/Councillor-Maria- \n MacAulay \n[10:28:17] Scraping from base.py:49\n https://westlothian.gov.uk/article/34313/Councillor-Moira- \n McKee-Shemilt \n[10:28:18] Scraping from base.py:49\n https://westlothian.gov.uk/article/34317/Councillor-Damian \n -Doran-Timson \n[10:28:19] Scraping from base.py:49\n https://westlothian.gov.uk/article/34315/Councillor-Carl-J \n ohn \n[10:28:20] Scraping from base.py:49\n https://westlothian.gov.uk/article/34316/Councillor-Danny- \n Logue \n[10:28:21] Scraping from base.py:49\n https://westlothian.gov.uk/article/34314/Councillor-Veroni \n ca-Smith \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34318/Councillor-Paulin \n e-Clark \n[10:28:22] Scraping from base.py:49\n https://westlothian.gov.uk/article/34319/Councillor-Craig- \n Meek \n[10:28:23] Scraping from base.py:49\n https://westlothian.gov.uk/article/34320/Councillor-Cathy- \n Muldoon \n[10:28:24] Scraping from base.py:49\n https://westlothian.gov.uk/article/34321/Councillor-Jim-Di \n ckson \n[10:28:25] Scraping from base.py:49\n https://westlothian.gov.uk/article/34322/Councillor-Mary-D \n ickson \n[10:28:26] Scraping from base.py:49\n https://westlothian.gov.uk/article/34323/Councillor-George \n -Paul \n[10:28:27] Scraping from base.py:49\n https://westlothian.gov.uk/article/34324/Councillor-David- \n Russell \n[10:28:28] list index out of range handlers.py:36\n Committing batch 1 consisting of 50 files base.py:297\n[10:28:29] Finished attempting to scrape: WLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/WLN-west-lothian/councillors.py\", line 37, in get_single_councillor\n councillor.email = soup.select(\"a[href^=mailto]\")[0].get_text(\nIndexError: list index out of range\n","start":"2024-12-12 10:27:52.881757","end":"2024-12-12 10:28:29.646178","duration":36}}] +[{"council_id":"BDG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:47:17] Fetching Scraper for: BDG handlers.py:23\n Begin attempting to scrape: BDG handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:47:18] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgov.lbbd.gov.uk/internet/mgWebService.asmx/GetC \n ouncillorsByWard \n[08:47:34] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BDG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-12-13 08:47:17.165337","end":"2024-12-13 08:47:34.552032","duration":17}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:06:13] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n Deleting existing data... base.py:257\n[10:06:14] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:06:15] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-12-13 10:06:13.539565","end":"2024-12-13 10:06:15.781154","duration":2}},{"council_id":"BRT","missing":false,"latest_run":{"status_code":1,"log_text":"[09:08:38] Fetching Scraper for: BRT handlers.py:23\n Begin attempting to scrape: BRT handlers.py:27\n[09:08:39] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:08:40] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.broxtowe.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[09:08:56] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BRT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-12-13 09:08:38.869752","end":"2024-12-13 09:08:56.386554","duration":17}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:37:42] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n[08:37:43] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:37:44] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[08:37:46] list index out of range handlers.py:36\n[08:37:47] Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-12-13 08:37:42.867246","end":"2024-12-13 08:37:47.210290","duration":4}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:10:25] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n Deleting existing data... base.py:257\n[09:10:26] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:10:27] ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[09:10:28] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-12-13 09:10:25.431013","end":"2024-12-13 09:10:28.940810","duration":3}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[09:28:37] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n[09:28:38] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:28:39] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[09:28:40] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n 'NoneType' object has no attribute 'find_parent' handlers.py:36\n[09:28:41] Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-12-13 09:28:37.606273","end":"2024-12-13 09:28:41.120151","duration":3}},{"council_id":"EDU","missing":false,"latest_run":{"status_code":1,"log_text":"[09:33:49] Fetching Scraper for: EDU handlers.py:23\n Begin attempting to scrape: EDU handlers.py:27\n Deleting existing data... base.py:257\n[09:33:50] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:33:51] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.eastdunbarton.gov.uk/residents/council-democra \n cy/committees-and-councillors/councillors-2017 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.eastdunbarton.gov.uk/residents/council-de \n mocracy/committees-and-councillors/councillors-2017' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: EDU base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.eastdunbarton.gov.uk/residents/council-democracy/committees-and-councillors/councillors-2017'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-12-13 09:33:49.402719","end":"2024-12-13 09:33:51.555811","duration":2}},{"council_id":"ELS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:39:14] Fetching Scraper for: ELS handlers.py:23\n Begin attempting to scrape: ELS handlers.py:27\n Deleting existing data... base.py:257\n[09:39:15] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://cne-siar.gov.uk/home/your-council/council-members/ \n[09:39:16] Client error '404 Not Found' for url handlers.py:36\n 'https://www.cne-siar.gov.uk/home/your-council/council \n -members/' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: ELS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.cne-siar.gov.uk/home/your-council/council-members/'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-12-13 09:39:14.238873","end":"2024-12-13 09:39:16.809665","duration":2}},{"council_id":"FYL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:59:09] Fetching Scraper for: FYL handlers.py:23\n Begin attempting to scrape: FYL handlers.py:27\n[08:59:10] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 17 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 17 files in Councillors/raw base.py:225\n ...found 35 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 35 files base.py:236\n[08:59:11] ...data deleted. base.py:264\n Scraping from base.py:49\n https://fylde.cmis.uk.com/fylde/CouncillorsandMP.aspx \n[08:59:12] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[08:59:16] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[08:59:18] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:20] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:22] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[08:59:24] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[08:59:27] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:29] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:31] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:33] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:34] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[08:59:37] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[08:59:39] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:41] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:42] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[08:59:45] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[08:59:47] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:51] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:52] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:55] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:56] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[08:59:59] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:00] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[09:00:03] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[09:00:04] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:07] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:08] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:11] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:14] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[09:00:17] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[09:00:18] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:21] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:22] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:25] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:27] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/222/ScreenMode/Alphabetical/ \n Default.aspx \n[09:00:29] 'NoneType' object has no attribute 'parent' handlers.py:36\n Committing batch 1 consisting of 34 files base.py:297\n[09:00:30] Finished attempting to scrape: FYL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 300, in get_single_councillor\n party = self.get_party_name(list_page_html)\n File \"scrapers/FYL-fylde/councillors.py\", line 17, in get_party_name\n .parent.get_text(strip=True)\nAttributeError: 'NoneType' object has no attribute 'parent'\n","start":"2024-12-13 08:59:09.550361","end":"2024-12-13 09:00:30.923904","duration":81}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:32:00] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n Deleting existing data... base.py:257\n[08:32:02] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:32:03] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n[08:32:04] Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-12-13 08:32:00.519257","end":"2024-12-13 08:32:04.136505","duration":3}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:48:45] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n[08:48:46] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:48:47] ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[08:49:02] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-12-13 08:48:45.259071","end":"2024-12-13 08:49:02.891038","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:18:49] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n[09:18:50] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:18:51] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[09:18:56] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-12-13 09:18:49.797633","end":"2024-12-13 09:18:56.501064","duration":6}},{"council_id":"KHL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:18:33] Fetching Scraper for: KHL handlers.py:23\n Begin attempting to scrape: KHL handlers.py:27\n Deleting existing data... base.py:257\n[09:18:34] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:18:35] ...data deleted. base.py:264\n Scraping from base.py:49\n https://cmis.hullcc.gov.uk/cmis/CouncillorsandSeniorOffice \n rs/CouncillorsandSeniorOfficers.aspx \n [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify handlers.py:36\n failed: unable to get local issuer certificate \n (_ssl.c:1007) \n Finished attempting to scrape: KHL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n","start":"2024-12-13 09:18:33.495210","end":"2024-12-13 09:18:35.666144","duration":2}},{"council_id":"MLN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:27:00] Fetching Scraper for: MLN handlers.py:23\n Begin attempting to scrape: MLN handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:27:01] Getting all files in Councillors/json... base.py:209\n ...found 12 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 12 files in Councillors/raw base.py:225\n ...found 25 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 25 files base.py:236\n[09:27:02] ...data deleted. base.py:264\n Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors.aspx \n[09:27:03] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/141/ScreenMode/Alphabetical/ \n Default.aspx \n[09:27:04] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/244/ScreenMode/Alphabetical/ \n Default.aspx \n[09:27:06] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/143/ScreenMode/Alphabetical/ \n Default.aspx \n[09:27:07] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/144/ScreenMode/Alphabetical/ \n Default.aspx \n[09:27:09] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/245/ScreenMode/Alphabetical/ \n Default.aspx \n[09:27:10] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/19/ScreenMode/Alphabetical/D \n efault.aspx \n[09:27:12] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/140/ScreenMode/Alphabetical/ \n Default.aspx \n[09:27:14] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/240/ScreenMode/Alphabetical/ \n Default.aspx \n[09:27:17] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/234/ScreenMode/Alphabetical/ \n Default.aspx \n[09:27:18] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/241/ScreenMode/Alphabetical/ \n Default.aspx \n[09:27:19] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/20/ScreenMode/Alphabetical/D \n efault.aspx \n[09:27:21] Scraping from base.py:49\n https://midlothian.cmis.uk.com/live/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/22/ScreenMode/Alphabetical/D \n efault.aspx \n[09:27:23] 'title' handlers.py:36\n Committing batch 1 consisting of 24 files base.py:297\n[09:27:24] Finished attempting to scrape: MLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 300, in get_single_councillor\n party = self.get_party_name(list_page_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 281, in get_party_name\n list_page_html.find_all(\"img\")[-1][\"title\"]\n File \"/opt/python/bs4/element.py\", line 1930, in __getitem__\n return self.attrs[key]\nKeyError: 'title'\n","start":"2024-12-13 09:27:00.009617","end":"2024-12-13 09:27:24.683312","duration":24}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"STG","missing":false,"latest_run":{"status_code":1,"log_text":"[09:04:45] Fetching Scraper for: STG handlers.py:23\n Begin attempting to scrape: STG handlers.py:27\n Deleting existing data... base.py:257\n[09:04:46] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 19 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 19 files in Councillors/raw base.py:225\n ...found 39 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 39 files base.py:236\n[09:04:47] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-elections/councillors/ \n[09:04:48] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/elaine-watterson/ \n[09:04:49] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gene-maxwell/ \n[09:04:51] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/martin-earl/ \n[09:04:52] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mcgarvey/ \n[09:04:53] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/paul-henke/ \n[09:04:54] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rosemary-fraser/ \n[09:04:55] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/alasdair-tollemache/ \n[09:04:57] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/david-wilson/ \n[09:04:58] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/robin-kleinman/ \n[09:04:59] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/thomas-heald/ \n[09:05:02] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/danny-gibson/ \n[09:05:03] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jim-thomson/ \n[09:05:04] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rachel-nunn/ \n[09:05:05] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/susan-mcgill/ \n[09:05:07] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jen-preston/ \n[09:05:08] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/neil-benny/ \n[09:05:09] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/scott-farmer/ \n[09:05:10] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/bryan-flannagan/ \n[09:05:11] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mclaughlan/ \n[09:05:12] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/willie-ferguson/ \n[09:05:14] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/alasdair-macpherson/ \n[09:05:15] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/brian-hambly/ \n[09:05:16] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/vacant-1/ \n[09:05:17] 'NoneType' object is not subscriptable handlers.py:36\n Committing batch 1 consisting of 44 files base.py:297\n[09:05:18] Finished attempting to scrape: STG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/STG-stirling/councillors.py\", line 42, in get_single_councillor\n councillor.email = soup.select_one(\"a[href^=mailto]\")[\nTypeError: 'NoneType' object is not subscriptable\n","start":"2024-12-13 09:04:45.424210","end":"2024-12-13 09:05:18.942373","duration":33}},{"council_id":"SWD","missing":false,"latest_run":{"status_code":1,"log_text":"[08:27:27] Fetching Scraper for: SWD handlers.py:23\n Begin attempting to scrape: SWD handlers.py:27\n Deleting existing data... base.py:257\n[08:27:28] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:27:29] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.winchester.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify handlers.py:36\n failed: unable to get local issuer certificate \n (_ssl.c:1007) \n Finished attempting to scrape: SWD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n","start":"2024-12-13 08:27:27.246936","end":"2024-12-13 08:27:29.331493","duration":2}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:31:52] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n[08:31:53] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:31:54] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[08:31:57] 'NoneType' object has no attribute 'findNext' handlers.py:36\n Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-12-13 08:31:52.931084","end":"2024-12-13 08:31:57.319868","duration":4}},{"council_id":"VGL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:46:06] Fetching Scraper for: VGL handlers.py:23\n Begin attempting to scrape: VGL handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:46:07] Getting all files in Councillors/json... base.py:209\n ...found 16 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 16 files in Councillors/raw base.py:225\n ...found 33 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 33 files base.py:236\n[08:46:08] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Councillors.aspx \n[08:46:10] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Asbrey-Anne.aspx \n[08:46:11] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Aviet-Julie.aspx \n[08:46:12] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Ball-Gareth.aspx \n[08:46:14] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Birch-Rhiannon.aspx \n[08:46:15] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Brooks-Bronwen.aspx \n[08:46:17] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Bruce-Gillian.aspx \n[08:46:20] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Buckley-Ian.aspx \n[08:46:21] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Burnett-Lis.aspx \n[08:46:23] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Campbell-Samantha.aspx \n[08:46:25] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Carroll-George.aspx \n[08:46:26] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cave-Christine.aspx \n[08:46:28] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Champion-Charles.aspx \n[08:46:29] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Charles-Janice.aspx \n[08:46:31] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Collins-Amelia.aspx \n[08:46:32] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cowpe-Marianne.aspx \n[08:46:34] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Drake-Pamela.aspx \n[08:46:36] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Driscoll-Vincent.aspx \n[08:46:37] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 32 files base.py:297\n[08:46:38] Finished attempting to scrape: VGL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/VGL-the-vale-of-glamorgan/councillors.py\", line 36, in get_single_councillor\n email_text = soup.select_one(\"#S4_EmailPlaceholder\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-12-13 08:46:06.018896","end":"2024-12-13 08:46:38.932789","duration":32}},{"council_id":"WIN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:26:21] Fetching Scraper for: WIN handlers.py:23\n Begin attempting to scrape: WIN handlers.py:27\n[08:26:22] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:26:23] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.winchester.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify handlers.py:36\n failed: unable to get local issuer certificate \n (_ssl.c:1007) \n[08:26:24] Finished attempting to scrape: WIN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n","start":"2024-12-13 08:26:21.895642","end":"2024-12-13 08:26:24.173307","duration":2}},{"council_id":"WLN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:34:46] Fetching Scraper for: WLN handlers.py:23\n Begin attempting to scrape: WLN handlers.py:27\n[09:34:47] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 25 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 25 files in Councillors/raw base.py:225\n ...found 51 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 51 files base.py:236\n[09:34:48] ...data deleted. base.py:264\n Scraping from https://westlothian.gov.uk/councillors base.py:49\n[09:34:50] Scraping from base.py:49\n https://westlothian.gov.uk/article/33888/Linlithgow \n[09:34:51] Scraping from base.py:49\n https://westlothian.gov.uk/article/33889/Broxburn-Uphall-a \n nd-Winchburgh \n[09:34:52] Scraping from base.py:49\n https://westlothian.gov.uk/article/33890/Livingston-North \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33891/Livingston-South \n[09:34:53] Scraping from base.py:49\n https://westlothian.gov.uk/article/33893/East-Livingston-a \n nd-East-Calder \n[09:34:54] Scraping from base.py:49\n https://westlothian.gov.uk/article/33892/Fauldhouse-and-th \n e-Breich-Valley \n[09:34:55] Scraping from base.py:49\n https://westlothian.gov.uk/article/33894/Whitburn-and-Blac \n kburn \n[09:34:56] Scraping from base.py:49\n https://westlothian.gov.uk/article/33895/Bathgate \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33896/Armadale-and-Blac \n kridge \n[09:34:57] Scraping from base.py:49\n https://westlothian.gov.uk/article/33897/Councillor-Tom-Co \n nn \n[09:34:58] Scraping from base.py:49\n https://westlothian.gov.uk/article/33898/Councillor-Paulin \n e-Orr \n[09:34:59] Scraping from base.py:49\n https://westlothian.gov.uk/article/33899/Councillor-Sally- \n Pattle \n[09:35:00] Scraping from base.py:49\n https://westlothian.gov.uk/article/34305/Councillor-Tony-B \n oyle \n[09:35:01] Scraping from base.py:49\n https://westlothian.gov.uk/article/34302/Councillor-Diane- \n Calder \n[09:35:02] Scraping from base.py:49\n https://westlothian.gov.uk/article/34303/Councillor-Janet- \n Campbell \n[09:35:03] Scraping from base.py:49\n https://westlothian.gov.uk/article/34304/Councillor-Angela \n -Doran-Timson \n[09:35:04] Scraping from base.py:49\n https://westlothian.gov.uk/article/34307/Councillor-Alison \n -Adamson \n[09:35:05] Scraping from base.py:49\n https://westlothian.gov.uk/article/34306/Councillor-Robert \n -De-Bold \n[09:35:06] Scraping from base.py:49\n https://westlothian.gov.uk/article/34308/Councillor-Anne-M \n cMillan \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34309/Councillor-Andrew \n -Miller \n[09:35:07] Scraping from base.py:49\n https://westlothian.gov.uk/article/34311/Councillor-Lawren \n ce-Fitzpatrick \n[09:35:08] Scraping from base.py:49\n https://westlothian.gov.uk/article/34310/Councillor-Peter- \n Heggie \n[09:35:09] Scraping from base.py:49\n https://westlothian.gov.uk/article/34312/Councillor-Maria- \n MacAulay \n[09:35:10] Scraping from base.py:49\n https://westlothian.gov.uk/article/34313/Councillor-Moira- \n McKee-Shemilt \n[09:35:11] Scraping from base.py:49\n https://westlothian.gov.uk/article/34317/Councillor-Damian \n -Doran-Timson \n[09:35:12] Scraping from base.py:49\n https://westlothian.gov.uk/article/34315/Councillor-Carl-J \n ohn \n[09:35:13] Scraping from base.py:49\n https://westlothian.gov.uk/article/34316/Councillor-Danny- \n Logue \n[09:35:15] Scraping from base.py:49\n https://westlothian.gov.uk/article/34314/Councillor-Veroni \n ca-Smith \n[09:35:16] Scraping from base.py:49\n https://westlothian.gov.uk/article/34318/Councillor-Paulin \n e-Clark \n[09:35:17] Scraping from base.py:49\n https://westlothian.gov.uk/article/34319/Councillor-Craig- \n Meek \n[09:35:18] Scraping from base.py:49\n https://westlothian.gov.uk/article/34320/Councillor-Cathy- \n Muldoon \n[09:35:19] Scraping from base.py:49\n https://westlothian.gov.uk/article/34321/Councillor-Jim-Di \n ckson \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34322/Councillor-Mary-D \n ickson \n[09:35:20] Scraping from base.py:49\n https://westlothian.gov.uk/article/34323/Councillor-George \n -Paul \n[09:35:21] Scraping from base.py:49\n https://westlothian.gov.uk/article/34324/Councillor-David- \n Russell \n[09:35:22] list index out of range handlers.py:36\n Committing batch 1 consisting of 50 files base.py:297\n[09:35:23] Finished attempting to scrape: WLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/WLN-west-lothian/councillors.py\", line 37, in get_single_councillor\n councillor.email = soup.select(\"a[href^=mailto]\")[0].get_text(\nIndexError: list index out of range\n","start":"2024-12-13 09:34:46.514180","end":"2024-12-13 09:35:23.823556","duration":37}}] diff --git a/index.html b/index.html index a18784b8a5..6fd28d1c5c 100644 --- a/index.html +++ b/index.html @@ -49,10 +49,6 @@
[08:55:18] Fetching Scraper for: ABC handlers.py:23 + Begin attempting to scrape: ABC handlers.py:27 +[08:55:19] Deleting existing data... base.py:257 + Getting all files in Councillors... base.py:209 + Getting all files in Councillors/json... base.py:209 + ...found 16 files in Councillors/json base.py:225 + Getting all files in Councillors/raw... base.py:209 + ...found 16 files in Councillors/raw base.py:225 + ...found 33 files in Councillors base.py:225 + Deleting batch no. 1 consisting of 33 files base.py:236 +[08:55:20] ...data deleted. base.py:264 + Scraping from base.py:49 + https://www.armaghbanbridgecraigavon.gov.uk/councillors/ +[08:55:24] Committing batch 1 consisting of 32 files base.py:297 +[08:55:25] Finished attempting to scrape: ABC base.py:345 ++ + + + +
[09:00:18] Fetching Scraper for: ABC handlers.py:23 - Begin attempting to scrape: ABC handlers.py:27 -[09:00:19] Deleting existing data... base.py:257 - Getting all files in Councillors... base.py:209 -[09:00:20] Getting all files in Councillors/json... base.py:209 - ...found 16 files in Councillors/json base.py:225 - Getting all files in Councillors/raw... base.py:209 - ...found 16 files in Councillors/raw base.py:225 - ...found 33 files in Councillors base.py:225 - Deleting batch no. 1 consisting of 33 files base.py:236 -[09:00:21] ...data deleted. base.py:264 - Scraping from base.py:49 - https://www.armaghbanbridgecraigavon.gov.uk/councillors/ -[09:00:24] Committing batch 1 consisting of 32 files base.py:297 -[09:00:25] Finished attempting to scrape: ABC base.py:345 -- -