diff --git a/api/failing.json b/api/failing.json index e46a17879c..204401a7f6 100644 --- a/api/failing.json +++ b/api/failing.json @@ -1 +1 @@ -[{"council_id":"AGB","missing":false,"latest_run":{"status_code":1,"log_text":"[10:24:34] Fetching Scraper for: AGB handlers.py:23\n Begin attempting to scrape: AGB handlers.py:27\n Deleting existing data... base.py:257\n[10:24:35] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 33 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 33 files in Councillors/raw base.py:225\n ...found 67 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 67 files base.py:236\n[10:24:36] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list \n[10:24:39] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/john-armour \n[10:24:41] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/gordon-blair \n[10:24:42] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/jan-brown \n[10:24:43] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/math-campbel \n l-sturgess \n[10:24:44] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/garret-corner \n[10:24:46] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/maurice-corry \n[10:24:47] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/audrey-e-forrest \n[10:24:50] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/kieron-green \n[10:24:51] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/daniel-hampsey \n[10:24:53] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/amanda-hampsey \n[10:24:54] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/graham-archibald-hard \n ie \n[10:24:55] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/fiona-howard \n[10:24:57] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list?page=1 \n[10:25:00] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/willie-hume \n[10:25:01] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/mark-irvine \n[10:25:02] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/andrew-kain \n[10:25:03] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/jennifer-kelly \n[10:25:05] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/paul-donald- \n kennedy \n[10:25:06] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/isle-bute/councillor/reeni-kennedy-boyle \n[10:25:08] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/jim-lynch \n[10:25:09] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/tommy-macpherson \n[10:25:11] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/ian-james-macquire \n[10:25:12] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/luna-martin \n[10:25:13] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/isle-bute/councillor/liz-mccabe \n[10:25:15] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/dougie-mcfadzean \n[10:25:16] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list?page=2 \n[10:25:20] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/julie-mckenzie \n[10:25:21] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/yvonne-mcneilly \n[10:25:22] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/ross-moreland \n[10:25:24] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/gary-mulvaney \n[10:25:25] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/shonny-iain-paterson \n[10:25:26] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/gemma-penfol \n d \n[10:25:29] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/douglas-philand \n[10:25:30] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/alastair-redman \n[10:25:32] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/william-sinclair \n[10:25:33] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/vacancy \n[10:25:34] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 66 files base.py:297\n[10:25:36] Finished attempting to scrape: AGB base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/AGB-argyll-and-bute/councillors.py\", line 36, in get_single_councillor\n ).get_text()\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-07-16 10:24:34.241638","end":"2024-07-16 10:25:36.172855","duration":61}},{"council_id":"BLA","missing":false,"latest_run":{"status_code":1,"log_text":"[08:42:30] Fetching Scraper for: BLA handlers.py:23\n Begin attempting to scrape: BLA handlers.py:27\n[08:42:31] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 36 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[08:42:32] ...found 36 files in Councillors/raw base.py:225\n ...found 73 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 73 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://w3.blaby.gov.uk/decision-making/mgWebService.asmx/ \n GetCouncillorsByWard \n[08:42:48] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BLA base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-16 08:42:30.585379","end":"2024-07-16 08:42:48.733742","duration":18}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:50:41] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n Deleting existing data... base.py:257\n[09:50:42] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:50:43] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-07-16 09:50:41.198846","end":"2024-07-16 09:50:43.456682","duration":2}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:32:20] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n[08:32:21] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:32:22] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[08:32:24] list index out of range handlers.py:36\n Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-07-16 08:32:20.652712","end":"2024-07-16 08:32:24.971599","duration":4}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:29:36] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n Deleting existing data... base.py:257\n[08:29:37] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:29:38] ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[08:29:39] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n[08:29:40] Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-07-16 08:29:36.345274","end":"2024-07-16 08:29:40.212997","duration":3}},{"council_id":"CHR","missing":false,"latest_run":{"status_code":1,"log_text":"[10:17:05] Fetching Scraper for: CHR handlers.py:23\n Begin attempting to scrape: CHR handlers.py:27\n[10:17:06] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[10:17:07] ...found 48 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 48 files in Councillors/raw base.py:225\n ...found 97 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 97 files base.py:236\n[10:17:08] ...data deleted. base.py:264\n Scraping from base.py:49\n http://modgov.cherwell.gov.uk/mgWebService.asmx/GetCouncil \n lorsByWard \n[10:17:28] Server error '504 Gateway Time-out' for url handlers.py:36\n 'http://modgov.cherwell.gov.uk/mgWebService.asmx/GetCo \n uncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/504 \n Finished attempting to scrape: CHR base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '504 Gateway Time-out' for url 'http://modgov.cherwell.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/504\n","start":"2024-07-16 10:17:05.918201","end":"2024-07-16 10:17:28.641739","duration":22}},{"council_id":"CMD","missing":false,"latest_run":{"status_code":1,"log_text":"[08:24:06] Fetching Scraper for: CMD handlers.py:23\n Begin attempting to scrape: CMD handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:24:07] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.camden.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n[08:24:08] Client error '403 Forbidden' for url handlers.py:36\n 'https://democracy.camden.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/403 \n[08:24:09] Finished attempting to scrape: CMD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://democracy.camden.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403\n","start":"2024-07-16 08:24:06.023248","end":"2024-07-16 08:24:09.933424","duration":3}},{"council_id":"DST","missing":false,"latest_run":{"status_code":1,"log_text":"[08:43:59] Fetching Scraper for: DST handlers.py:23\n Begin attempting to scrape: DST handlers.py:27\n Deleting existing data... base.py:257\n[08:44:00] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:44:01] ...data deleted. base.py:264\n Scraping from base.py:49\n http://moderngov.dorsetcouncil.gov.uk/mgWebService.asmx/Ge \n tCouncillorsByWard \n[08:44:16] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: DST base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-16 08:43:59.477666","end":"2024-07-16 08:44:16.993737","duration":17}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[10:37:00] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n[10:37:01] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:37:02] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[10:37:03] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n[10:37:04] 'NoneType' object has no attribute 'find_parent' handlers.py:36\n Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-07-16 10:37:00.776713","end":"2024-07-16 10:37:04.696081","duration":3}},{"council_id":"EPS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:37:55] Fetching Scraper for: EPS handlers.py:23\n Begin attempting to scrape: EPS handlers.py:27\n Deleting existing data... base.py:257\n[08:37:56] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:37:57] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.epsom-ewell.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard \n [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify handlers.py:36\n failed: unable to get local issuer certificate \n (_ssl.c:1007) \n Finished attempting to scrape: EPS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)\n","start":"2024-07-16 08:37:55.184398","end":"2024-07-16 08:37:57.494853","duration":2}},{"council_id":"ERW","missing":false,"latest_run":{"status_code":1,"log_text":"[10:14:47] Fetching Scraper for: ERW handlers.py:23\n Begin attempting to scrape: ERW handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:14:48] Getting all files in Councillors/json... base.py:209\n ...found 18 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 18 files in Councillors/raw base.py:225\n ...found 37 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 37 files base.py:236\n[10:14:49] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/Find-my-councillor \n[10:14:50] Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/councillor-angela-conv \n ery \n[10:14:51] Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/councillor-betty-cunni \n ngham \n[10:14:52] Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/councillor-danny-devli \n n \n[10:14:53] Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/Councillor-Chris-Lunda \n y \n[10:14:54] Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/councillor-tony-buchan \n an \n 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 8 files base.py:297\n[10:14:56] Finished attempting to scrape: ERW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/ERW-east-renfrewshire/councillors.py\", line 46, in get_single_councillor\n councillor.email = soup.select_one(\"td a[href^=mailto]\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-07-16 10:14:47.023976","end":"2024-07-16 10:14:56.092222","duration":9}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:37:35] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n Deleting existing data... base.py:257\n[08:37:36] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:37:37] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-07-16 08:37:35.481458","end":"2024-07-16 08:37:37.614370","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:59:33] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n[08:59:34] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:59:35] ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[08:59:51] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-16 08:59:33.709762","end":"2024-07-16 08:59:51.804681","duration":18}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:02:20] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n[09:02:21] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:02:22] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[09:02:27] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-07-16 09:02:20.699141","end":"2024-07-16 09:02:27.396725","duration":6}},{"council_id":"HIN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:18:22] Fetching Scraper for: HIN handlers.py:23\n Begin attempting to scrape: HIN handlers.py:27\n[09:18:24] Deleting existing data... base.py:257\n[09:18:25] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 34 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 34 files in Councillors/raw base.py:225\n ...found 69 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 69 files base.py:236\n[09:18:26] ...data deleted. base.py:264\n Scraping from base.py:49\n http://moderngov.hinckley-bosworth.gov.uk/mgWebService.asm \n x/GetCouncillorsByWard \n[09:18:30] [Errno 113] No route to host handlers.py:36\n Finished attempting to scrape: HIN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 113] No route to host\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 113] No route to host\n","start":"2024-07-16 09:18:22.350327","end":"2024-07-16 09:18:30.415383","duration":8}},{"council_id":"MEL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:34:30] Fetching Scraper for: MEL handlers.py:23\n Begin attempting to scrape: MEL handlers.py:27\n[08:34:31] Deleting existing data... base.py:257\n[08:34:33] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 28 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 28 files in Councillors/raw base.py:225\n ...found 57 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 57 files base.py:236\n[08:34:34] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.melton.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n[08:34:50] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: MEL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-16 08:34:30.908915","end":"2024-07-16 08:34:50.474137","duration":19}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[10:43:03] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n[10:43:04] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:43:05] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[10:43:07] 'NoneType' object has no attribute 'findNext' handlers.py:36\n Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-07-16 10:43:03.536540","end":"2024-07-16 10:43:07.276781","duration":3}}] +[{"council_id":"AGB","missing":false,"latest_run":{"status_code":1,"log_text":"[09:14:42] Fetching Scraper for: AGB handlers.py:23\n Begin attempting to scrape: AGB handlers.py:27\n Deleting existing data... base.py:257\n[09:14:43] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 33 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 33 files in Councillors/raw base.py:225\n ...found 67 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 67 files base.py:236\n[09:14:44] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list \n[09:14:47] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/john-armour \n[09:14:48] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/gordon-blair \n[09:14:49] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/jan-brown \n[09:14:50] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/math-campbel \n l-sturgess \n[09:14:51] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/garret-corner \n[09:14:52] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/maurice-corry \n[09:14:53] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/audrey-e-forrest \n[09:14:55] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/kieron-green \n[09:14:56] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/daniel-hampsey \n[09:14:57] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/amanda-hampsey \n[09:14:58] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/graham-archibald-hard \n ie \n[09:14:59] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/fiona-howard \n[09:15:01] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list?page=1 \n[09:15:05] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/willie-hume \n[09:15:06] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/mark-irvine \n[09:15:08] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/andrew-kain \n[09:15:09] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/jennifer-kelly \n[09:15:10] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/paul-donald- \n kennedy \n[09:15:11] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/isle-bute/councillor/reeni-kennedy-boyle \n[09:15:12] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-south-and-isles/councillor/jim-lynch \n[09:15:13] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/south-kintyre/councillor/tommy-macpherson \n[09:15:15] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/ian-james-macquire \n[09:15:16] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/luna-martin \n[09:15:18] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/isle-bute/councillor/liz-mccabe \n[09:15:19] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/dougie-mcfadzean \n[09:15:21] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/councillor_list?page=2 \n[09:15:24] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/oban-north-and-lorn/councillor/julie-mckenzie \n[09:15:25] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/yvonne-mcneilly \n[09:15:27] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/dunoon/councillor/ross-moreland \n[09:15:28] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-central/councillor/gary-mulvaney \n[09:15:30] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/lomond-north/councillor/shonny-iain-paterson \n[09:15:31] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/helensburgh-and-lomond-south/councillor/gemma-penfol \n d \n[09:15:32] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/mid-argyll/councillor/douglas-philand \n[09:15:33] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/alastair-redman \n[09:15:34] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/cowal/councillor/william-sinclair \n[09:15:36] Scraping from base.py:49\n https://www.argyll-bute.gov.uk/my-council/councillors-dire \n ctory/kintyre-and-islands/councillor/vacancy \n[09:15:37] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 66 files base.py:297\n[09:15:39] Finished attempting to scrape: AGB base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/AGB-argyll-and-bute/councillors.py\", line 36, in get_single_councillor\n ).get_text()\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-07-17 09:14:42.285116","end":"2024-07-17 09:15:39.101636","duration":56}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:55:10] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n[08:55:11] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:55:12] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-07-17 08:55:10.656402","end":"2024-07-17 08:55:12.776243","duration":2}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[11:13:36] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n[11:13:37] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[11:13:38] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[11:13:40] list index out of range handlers.py:36\n[11:13:41] Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-07-17 11:13:36.964293","end":"2024-07-17 11:13:41.198831","duration":4}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:46:17] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n Deleting existing data... base.py:257\n[09:46:18] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:46:19] ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[09:46:20] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-07-17 09:46:17.309483","end":"2024-07-17 09:46:20.616415","duration":3}},{"council_id":"CMD","missing":false,"latest_run":{"status_code":1,"log_text":"[10:08:35] Fetching Scraper for: CMD handlers.py:23\n Begin attempting to scrape: CMD handlers.py:27\n[10:08:36] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:08:37] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.camden.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Client error '403 Forbidden' for url handlers.py:36\n 'https://democracy.camden.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/403 \n Finished attempting to scrape: CMD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://democracy.camden.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403\n","start":"2024-07-17 10:08:35.783814","end":"2024-07-17 10:08:37.841149","duration":2}},{"council_id":"DST","missing":false,"latest_run":{"status_code":1,"log_text":"[09:33:45] Fetching Scraper for: DST handlers.py:23\n Begin attempting to scrape: DST handlers.py:27\n[09:33:46] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:33:47] ...data deleted. base.py:264\n Scraping from base.py:49\n http://moderngov.dorsetcouncil.gov.uk/mgWebService.asmx/Ge \n tCouncillorsByWard \n[09:34:02] [Errno 110] Connection timed out handlers.py:36\n[09:34:03] Finished attempting to scrape: DST base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-17 09:33:45.648120","end":"2024-07-17 09:34:03.287743","duration":17}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[09:01:26] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n[09:01:27] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:01:28] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[09:01:29] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n[09:01:30] 'NoneType' object has no attribute 'find_parent' handlers.py:36\n Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-07-17 09:01:26.615599","end":"2024-07-17 09:01:30.528934","duration":3}},{"council_id":"ERW","missing":false,"latest_run":{"status_code":1,"log_text":"[08:42:47] Fetching Scraper for: ERW handlers.py:23\n Begin attempting to scrape: ERW handlers.py:27\n[08:42:48] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:42:49] ...found 4 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 4 files in Councillors/raw base.py:225\n ...found 9 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 9 files base.py:236\n[08:42:50] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/Find-my-councillor \n[08:42:52] Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/councillor-angela-conv \n ery \n[08:42:53] Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/councillor-betty-cunni \n ngham \n Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/councillor-danny-devli \n n \n[08:42:54] Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/Councillor-Chris-Lunda \n y \n[08:42:55] Scraping from base.py:49\n https://www.eastrenfrewshire.gov.uk/councillor-tony-buchan \n an \n[08:42:56] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 8 files base.py:297\n[08:42:57] Finished attempting to scrape: ERW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/ERW-east-renfrewshire/councillors.py\", line 46, in get_single_councillor\n councillor.email = soup.select_one(\"td a[href^=mailto]\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-07-17 08:42:47.738198","end":"2024-07-17 08:42:57.702615","duration":9}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[09:33:35] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n[09:33:36] Deleting existing data... base.py:257\n[09:33:38] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:33:39] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-07-17 09:33:35.755810","end":"2024-07-17 09:33:39.536600","duration":3}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:40:33] Fetching Scraper for: GLS handlers.py:23\n[08:40:34] Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:40:35] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[08:40:51] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-07-17 08:40:33.994054","end":"2024-07-17 08:40:51.510514","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:48:20] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n[09:48:21] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:48:22] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[09:48:27] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-07-17 09:48:20.685000","end":"2024-07-17 09:48:27.420285","duration":6}},{"council_id":"HAO","missing":false,"latest_run":{"status_code":1,"log_text":"[09:34:49] Fetching Scraper for: HAO handlers.py:23\n Begin attempting to scrape: HAO handlers.py:27\n Deleting existing data... base.py:257\n[09:34:50] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 34 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 34 files in Councillors/raw base.py:225\n ...found 69 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 69 files base.py:236\n[09:34:51] ...data deleted. base.py:264\n Scraping from base.py:49\n https://cmis.harborough.gov.uk/cmis5/Councillors.aspx \n[09:34:53] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/1427/ScreenMode/Ward/Default \n .aspx \n[09:35:12] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2186/ScreenMode/Ward/Default \n .aspx \n[09:35:23] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2054/ScreenMode/Ward/Default \n .aspx \n[09:35:44] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2196/ScreenMode/Ward/Default \n .aspx \n[09:35:52] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2151/ScreenMode/Ward/Default \n .aspx \n[09:36:06] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2149/ScreenMode/Ward/Default \n .aspx \n[09:36:23] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2107/ScreenMode/Ward/Default \n .aspx \n[09:36:44] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2147/ScreenMode/Ward/Default \n .aspx \n[09:36:55] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2199/ScreenMode/Ward/Default \n .aspx \n[09:37:07] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2109/ScreenMode/Ward/Default \n .aspx \n[09:37:25] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2163/ScreenMode/Ward/Default \n .aspx \n[09:37:38] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2161/ScreenMode/Ward/Default \n .aspx \n[09:37:48] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2059/ScreenMode/Ward/Default \n .aspx \n[09:38:10] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2165/ScreenMode/Ward/Default \n .aspx \n[09:38:24] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2216/ScreenMode/Ward/Default \n .aspx \n[09:38:43] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2218/ScreenMode/Ward/Default \n .aspx \n[09:38:59] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2164/ScreenMode/Ward/Default \n .aspx \n[09:39:12] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2158/ScreenMode/Ward/Default \n .aspx \n[09:39:26] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2224/ScreenMode/Ward/Default \n .aspx \n[09:39:41] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2230/ScreenMode/Ward/Default \n .aspx \n[09:39:55] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2162/ScreenMode/Ward/Default \n .aspx \n[09:40:16] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2074/ScreenMode/Ward/Default \n .aspx \n[09:40:46] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2172/ScreenMode/Ward/Default \n .aspx \n[09:40:56] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2253/ScreenMode/Ward/Default \n .aspx \n[09:41:07] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2072/ScreenMode/Ward/Default \n .aspx \n[09:41:33] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2244/ScreenMode/Ward/Default \n .aspx \n[09:41:45] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2251/ScreenMode/Ward/Default \n .aspx \n[09:41:55] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2252/ScreenMode/Ward/Default \n .aspx \n[09:42:14] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2156/ScreenMode/Ward/Default \n .aspx \n[09:42:27] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/2152/ScreenMode/Ward/Default \n .aspx \n[09:42:49] Scraping from base.py:49\n http://cmis.harborough.gov.uk/cmis5/Councillors/tabid/62/c \n tl/ViewCMIS_Person/mid/480/id/1444/ScreenMode/Ward/Default \n .aspx \n[09:43:20] The read operation timed out handlers.py:36\n Committing batch 1 consisting of 60 files base.py:297\n[09:43:21] Finished attempting to scrape: HAO base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 310, in get_single_councillor\n req = self.get(url)\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-07-17 09:34:49.345800","end":"2024-07-17 09:43:21.478175","duration":512}},{"council_id":"NGM","missing":false,"latest_run":{"status_code":1,"log_text":"[09:46:33] Fetching Scraper for: NGM handlers.py:23\n Begin attempting to scrape: NGM handlers.py:27\n Deleting existing data... base.py:257\n[09:46:34] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 55 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 55 files in Councillors/raw base.py:225\n ...found 111 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:46:35] Deleting batch no. 2 consisting of 11 files base.py:236\n[09:46:36] ...data deleted. base.py:264\n Scraping from base.py:49\n http://committee.nottinghamcity.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard \n[09:47:06] The read operation timed out handlers.py:36\n Finished attempting to scrape: NGM base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-07-17 09:46:33.199143","end":"2024-07-17 09:47:06.610012","duration":33}},{"council_id":"NLK","missing":false,"latest_run":{"status_code":1,"log_text":"[08:41:57] Fetching Scraper for: NLK handlers.py:23\n Begin attempting to scrape: NLK handlers.py:27\n Deleting existing data... base.py:257\n[08:41:58] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 77 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 77 files in Councillors/raw base.py:225\n ...found 155 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:41:59] Deleting batch no. 2 consisting of 55 files base.py:236\n[08:42:00] ...data deleted. base.py:264\n Scraping from base.py:49\n https://northlanarkshire.cmis.uk.com/Councillors.aspx \n[08:42:05] Scraping from base.py:49\n http://northlanarkshire.cmis.uk.com/Councillors/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/156/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:35] The read operation timed out handlers.py:36\n Finished attempting to scrape: NLK base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 310, in get_single_councillor\n req = self.get(url)\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-07-17 08:41:57.137622","end":"2024-07-17 08:42:35.801561","duration":38}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:43:16] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n[08:43:17] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:43:18] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[08:43:20] 'NoneType' object has no attribute 'findNext' handlers.py:36\n Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-07-17 08:43:16.829810","end":"2024-07-17 08:43:20.540696","duration":3}},{"council_id":"WDU","missing":false,"latest_run":{"status_code":1,"log_text":"[09:45:08] Fetching Scraper for: WDU handlers.py:23\n Begin attempting to scrape: WDU handlers.py:27\n[09:45:10] Deleting existing data... base.py:257\n[09:45:11] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 22 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 22 files in Councillors/raw base.py:225\n ...found 45 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 45 files base.py:236\n[09:45:12] ...data deleted. base.py:264\n Scraping from base.py:49\n https://wdccmis.west-dunbarton.gov.uk/cmis5/Councillors.as \n px \n[09:45:42] Scraping from base.py:49\n http://wdccmis.west-dunbarton.gov.uk/cmis5/Councillors/tab \n id/62/ctl/ViewCMIS_Person/mid/480/id/1032/ScreenMode/Ward/ \n Default.aspx \n[09:46:12] The read operation timed out handlers.py:36\n[09:46:13] Finished attempting to scrape: WDU base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 310, in get_single_councillor\n req = self.get(url)\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-07-17 09:45:08.444926","end":"2024-07-17 09:46:13.338383","duration":64}}] diff --git a/index.html b/index.html index 724159543b..342229884e 100644 --- a/index.html +++ b/index.html @@ -49,10 +49,6 @@
[10:04:21] Fetching Scraper for: ABC handlers.py:23 + Begin attempting to scrape: ABC handlers.py:27 + Deleting existing data... base.py:257 +[10:04:22] Getting all files in Councillors... base.py:209 + Getting all files in Councillors/json... base.py:209 + ...found 16 files in Councillors/json base.py:225 + Getting all files in Councillors/raw... base.py:209 + ...found 16 files in Councillors/raw base.py:225 + ...found 33 files in Councillors base.py:225 + Deleting batch no. 1 consisting of 33 files base.py:236 +[10:04:23] ...data deleted. base.py:264 + Scraping from base.py:49 + https://www.armaghbanbridgecraigavon.gov.uk/councillors/ +[10:04:28] Committing batch 1 consisting of 32 files base.py:297 +[10:04:30] Finished attempting to scrape: ABC base.py:345 ++ + + + +
[08:47:38] Fetching Scraper for: ABC handlers.py:23 - Begin attempting to scrape: ABC handlers.py:27 - Deleting existing data... base.py:257 - Getting all files in Councillors... base.py:209 -[08:47:39] Getting all files in Councillors/json... base.py:209 - ...found 16 files in Councillors/json base.py:225 - Getting all files in Councillors/raw... base.py:209 - ...found 16 files in Councillors/raw base.py:225 - ...found 33 files in Councillors base.py:225 - Deleting batch no. 1 consisting of 33 files base.py:236 -[08:47:40] ...data deleted. base.py:264 - Scraping from base.py:49 - https://www.armaghbanbridgecraigavon.gov.uk/councillors/ -[08:47:44] Committing batch 1 consisting of 32 files base.py:297 -[08:47:45] Finished attempting to scrape: ABC base.py:345 -- -