diff --git a/api/failing.json b/api/failing.json index 5021151a01..83839bb2a5 100644 --- a/api/failing.json +++ b/api/failing.json @@ -1 +1 @@ -[{"council_id":"AGB","missing":false,"latest_run":{"status_code":1,"log_text":"[09:23:05] Fetching Scraper for: AGB handlers.py:23\n Begin attempting to scrape: AGB handlers.py:27\n Deleting existing data... base.py:257\n[09:23:06] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 33 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 33 files in Councillors/raw base.py:225\n ...found 67 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 67 files base.py:236\n[09:23:07] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list \n[09:23:10] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/john-armour \n[09:23:11] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/gordon-blair \n[09:23:13] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/jan-brown \n[09:23:14] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/math-campbel \n l-sturgess \n[09:23:16] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/garret-corner \n[09:23:17] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/maurice-corry \n[09:23:18] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/audrey-e-forrest \n[09:23:19] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/kieron-green \n[09:23:21] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/daniel-hampsey \n[09:23:22] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/amanda-hampsey \n[09:23:24] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/graham-archibald-hard \n ie \n[09:23:25] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/fiona-howard \n[09:23:28] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list?page=1 \n[09:23:31] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/willie-hume \n[09:23:32] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/mark-irvine \n[09:23:33] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/andrew-kain \n[09:23:35] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/jennifer-kelly \n[09:23:36] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/paul-donald- \n kennedy \n[09:23:38] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/isle-bute/councillor/reeni-kennedy-boyle \n[09:23:39] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/jim-lynch \n[09:23:40] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/tommy-macpherson \n[09:23:42] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/ian-james-macquire \n[09:23:43] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/luna-martin \n[09:23:45] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/isle-bute/councillor/liz-mccabe \n[09:23:46] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/dougie-mcfadzean \n[09:23:48] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list?page=2 \n[09:23:51] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/julie-mckenzie \n[09:23:53] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/yvonne-mcneilly \n[09:23:54] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/ross-moreland \n[09:23:55] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/gary-mulvaney \n[09:23:57] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/shonny-iain-paterson \n[09:23:58] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/gemma-penfol \n d \n[09:23:59] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/douglas-philand \n[09:24:01] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/alastair-redman \n[09:24:02] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/william-sinclair \n[09:24:04] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/vacancy \n[09:24:05] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 66 files base.py:297\n[09:24:06] Finished attempting to scrape: AGB base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/AGB-argyll-and-bute/councillors.py\", line 36, in get_single_councillor\n ).get_text()\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-07-20 09:23:05.180007","end":"2024-07-20 09:24:06.790717","duration":61}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:25:27] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n Deleting existing data... base.py:257\n[08:25:28] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:25:29] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-07-20 08:25:27.353674","end":"2024-07-20 08:25:29.484166","duration":2}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:41:56] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n[09:41:57] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:41:58] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[09:42:00] list index out of range handlers.py:36\n Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-07-20 09:41:56.876809","end":"2024-07-20 09:42:00.908208","duration":4}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[10:24:28] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n Deleting existing data... base.py:257\n[10:24:29] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:24:30] ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[10:24:31] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-07-20 10:24:28.379890","end":"2024-07-20 10:24:31.420130","duration":3}},{"council_id":"CMD","missing":false,"latest_run":{"status_code":1,"log_text":"[08:18:57] Fetching Scraper for: CMD handlers.py:23\n Begin attempting to scrape: CMD handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:18:58] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.camden.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Client error '403 Forbidden' for url handlers.py:36\n 'https://democracy.camden.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/403 \n[08:18:59] Finished attempting to scrape: CMD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://democracy.camden.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403\n","start":"2024-07-20 08:18:57.101814","end":"2024-07-20 08:18:59.162003","duration":2}},{"council_id":"DST","missing":false,"latest_run":{"status_code":1,"log_text":"[10:20:24] Fetching Scraper for: DST handlers.py:23\n Begin attempting to scrape: DST handlers.py:27\n[10:20:25] Deleting existing data... base.py:257\n[10:20:27] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://moderngov.dorsetcouncil.gov.uk/mgWebService.asmx/Ge \n tCouncillorsByWard \n[10:20:43] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: DST base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-20 10:20:24.666314","end":"2024-07-20 10:20:43.602199","duration":18}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[10:30:14] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:30:15] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[10:30:16] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n[10:30:17] 'NoneType' object has no attribute 'find_parent' handlers.py:36\n Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-07-20 10:30:14.150312","end":"2024-07-20 10:30:17.513869","duration":3}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:27:20] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n[08:27:21] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:27:22] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-07-20 08:27:20.567124","end":"2024-07-20 08:27:22.783556","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:32:46] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n[08:32:47] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[08:33:03] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-20 08:32:46.157714","end":"2024-07-20 08:33:03.773095","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:29:06] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n Deleting existing data... base.py:257\n[08:29:07] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:29:08] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[08:29:12] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-07-20 08:29:06.475695","end":"2024-07-20 08:29:12.760158","duration":6}},{"council_id":"HRY","missing":false,"latest_run":{"status_code":1,"log_text":"[08:34:20] Fetching Scraper for: HRY handlers.py:23\n Begin attempting to scrape: HRY handlers.py:27\n Deleting existing data... base.py:257\n[08:34:21] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 57 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 57 files in Councillors/raw base.py:225\n ...found 115 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:34:22] Deleting batch no. 2 consisting of 15 files base.py:236\n[08:34:23] ...data deleted. base.py:264\n Scraping from base.py:49\n http://www.minutes.haringey.gov.uk/mgWebService.asmx/GetCo \n uncillorsByWard \n[08:34:53] The read operation timed out handlers.py:36\n Finished attempting to scrape: HRY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-07-20 08:34:20.096667","end":"2024-07-20 08:34:53.439132","duration":33}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:25:07] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n[09:25:08] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:25:09] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[09:25:11] 'NoneType' object has no attribute 'findNext' handlers.py:36\n Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-07-20 09:25:07.843871","end":"2024-07-20 09:25:11.368540","duration":3}}] +[{"council_id":"AGB","missing":false,"latest_run":{"status_code":1,"log_text":"[09:35:10] Fetching Scraper for: AGB handlers.py:23\n Begin attempting to scrape: AGB handlers.py:27\n[09:35:11] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 33 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 33 files in Councillors/raw base.py:225\n ...found 67 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 67 files base.py:236\n[09:35:12] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list \n[09:35:16] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/john-armour \n[09:35:17] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/gordon-blair \n[09:35:18] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/jan-brown \n[09:35:19] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/math-campbel \n l-sturgess \n[09:35:20] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/garret-corner \n[09:35:21] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/maurice-corry \n[09:35:23] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/audrey-e-forrest \n[09:35:24] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/kieron-green \n[09:35:25] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/daniel-hampsey \n[09:35:27] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/amanda-hampsey \n[09:35:28] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/graham-archibald-hard \n ie \n[09:35:29] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/fiona-howard \n[09:35:31] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list?page=1 \n[09:35:35] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/willie-hume \n[09:35:36] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/mark-irvine \n[09:35:37] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/andrew-kain \n[09:35:39] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/jennifer-kelly \n[09:35:40] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/paul-donald- \n kennedy \n[09:35:41] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/isle-bute/councillor/reeni-kennedy-boyle \n[09:35:42] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/jim-lynch \n[09:35:44] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/tommy-macpherson \n[09:35:45] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/ian-james-macquire \n[09:35:46] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/luna-martin \n[09:35:48] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/isle-bute/councillor/liz-mccabe \n[09:35:49] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/dougie-mcfadzean \n[09:35:50] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list?page=2 \n[09:35:53] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/julie-mckenzie \n[09:35:54] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/yvonne-mcneilly \n[09:35:56] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/ross-moreland \n[09:35:57] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/gary-mulvaney \n[09:35:58] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/shonny-iain-paterson \n[09:35:59] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/gemma-penfol \n d \n[09:36:00] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/douglas-philand \n[09:36:02] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/alastair-redman \n[09:36:03] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/william-sinclair \n[09:36:04] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/vacancy \n[09:36:05] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 66 files base.py:297\n[09:36:07] Finished attempting to scrape: AGB base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/AGB-argyll-and-bute/councillors.py\", line 36, in get_single_councillor\n ).get_text()\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-07-21 09:35:10.761514","end":"2024-07-21 09:36:07.166579","duration":56}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:29:18] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n[10:29:19] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:29:20] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n[10:29:21] Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-07-21 10:29:18.884394","end":"2024-07-21 10:29:21.051868","duration":2}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:34:19] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n[08:34:20] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n[08:34:21] Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[08:34:23] list index out of range handlers.py:36\n Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-07-21 08:34:19.936029","end":"2024-07-21 08:34:23.780464","duration":3}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:30:36] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n[08:30:37] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:30:38] ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[08:30:39] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n[08:30:40] Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-07-21 08:30:36.898439","end":"2024-07-21 08:30:40.022837","duration":3}},{"council_id":"CMD","missing":false,"latest_run":{"status_code":1,"log_text":"[09:22:13] Fetching Scraper for: CMD handlers.py:23\n Begin attempting to scrape: CMD handlers.py:27\n Deleting existing data... base.py:257\n[09:22:14] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:22:15] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.camden.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Client error '403 Forbidden' for url handlers.py:36\n 'https://democracy.camden.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/403 \n Finished attempting to scrape: CMD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://democracy.camden.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403\n","start":"2024-07-21 09:22:13.701358","end":"2024-07-21 09:22:15.807755","duration":2}},{"council_id":"DST","missing":false,"latest_run":{"status_code":1,"log_text":"[09:37:16] Fetching Scraper for: DST handlers.py:23\n Begin attempting to scrape: DST handlers.py:27\n[09:37:17] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:37:18] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:37:19] ...data deleted. base.py:264\n Scraping from base.py:49\n http://moderngov.dorsetcouncil.gov.uk/mgWebService.asmx/Ge \n tCouncillorsByWard \n[09:37:34] [Errno 110] Connection timed out handlers.py:36\n[09:37:35] Finished attempting to scrape: DST base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-21 09:37:16.897214","end":"2024-07-21 09:37:35.206064","duration":18}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[08:29:18] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:29:19] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[08:29:20] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n[08:29:21] 'NoneType' object has no attribute 'find_parent' handlers.py:36\n Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-07-21 08:29:18.144055","end":"2024-07-21 08:29:21.527548","duration":3}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:52:05] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n Deleting existing data... base.py:257\n[08:52:06] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:52:07] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-07-21 08:52:05.561053","end":"2024-07-21 08:52:07.683213","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[10:17:02] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n[10:17:03] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:17:05] ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[10:17:20] [Errno 110] Connection timed out handlers.py:36\n[10:17:21] Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-21 10:17:02.505848","end":"2024-07-21 10:17:21.327231","duration":18}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:34:47] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n[08:34:48] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:34:49] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[08:34:50] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-07-21 08:34:47.952008","end":"2024-07-21 08:34:50.446696","duration":2}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:58:21] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n Deleting existing data... base.py:257\n[09:58:22] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:58:23] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[09:58:25] 'NoneType' object has no attribute 'findNext' handlers.py:36\n[09:58:26] Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-07-21 09:58:21.404512","end":"2024-07-21 09:58:26.023891","duration":4}}] diff --git a/index.html b/index.html index 6a9107f516..077ca7a34f 100644 --- a/index.html +++ b/index.html @@ -49,10 +49,6 @@
[08:33:53] Fetching Scraper for: ABC handlers.py:23 + Begin attempting to scrape: ABC handlers.py:27 + Deleting existing data... base.py:257 +[08:33:54] Getting all files in Councillors... base.py:209 + Getting all files in Councillors/json... base.py:209 + ...found 16 files in Councillors/json base.py:225 + Getting all files in Councillors/raw... base.py:209 + ...found 16 files in Councillors/raw base.py:225 + ...found 33 files in Councillors base.py:225 + Deleting batch no. 1 consisting of 33 files base.py:236 +[08:33:55] ...data deleted. base.py:264 + Scraping from base.py:49 + https://www.armaghbanbridgecraigavon.gov.uk/councillors/ +[08:33:58] Committing batch 1 consisting of 32 files base.py:297 +[08:34:00] Finished attempting to scrape: ABC base.py:345 ++ + + + +
[09:59:12] Fetching Scraper for: ABC handlers.py:23 - Begin attempting to scrape: ABC handlers.py:27 - Deleting existing data... base.py:257 - Getting all files in Councillors... base.py:209 -[09:59:13] Getting all files in Councillors/json... base.py:209 - ...found 16 files in Councillors/json base.py:225 - Getting all files in Councillors/raw... base.py:209 - ...found 16 files in Councillors/raw base.py:225 - ...found 33 files in Councillors base.py:225 - Deleting batch no. 1 consisting of 33 files base.py:236 -[09:59:14] ...data deleted. base.py:264 - Scraping from base.py:49 - https://www.armaghbanbridgecraigavon.gov.uk/councillors/ -[09:59:18] Committing batch 1 consisting of 32 files base.py:297 -[09:59:19] Finished attempting to scrape: ABC base.py:345 -- -