diff --git a/api/failing.json b/api/failing.json index f701b841e5..2484a5be38 100644 --- a/api/failing.json +++ b/api/failing.json @@ -1 +1 @@ -[{"council_id":"BDG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:30:06] Fetching Scraper for: BDG handlers.py:23\n Begin attempting to scrape: BDG handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:30:07] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgov.lbbd.gov.uk/internet/mgWebService.asmx/GetC \n ouncillorsByWard \n[08:30:23] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BDG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-07 08:30:06.122732","end":"2024-10-07 08:30:23.823748","duration":17}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:57:38] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n Deleting existing data... base.py:257\n[09:57:39] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:57:40] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-10-07 09:57:38.406403","end":"2024-10-07 09:57:40.639076","duration":2}},{"council_id":"BRT","missing":false,"latest_run":{"status_code":1,"log_text":"[08:24:39] Fetching Scraper for: BRT handlers.py:23\n Begin attempting to scrape: BRT handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:24:40] ...found 44 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 44 files in Councillors/raw base.py:225\n ...found 89 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 89 files base.py:236\n[08:24:41] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.broxtowe.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[08:24:56] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BRT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-07 08:24:39.053925","end":"2024-10-07 08:24:56.895875","duration":17}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:19:08] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:19:09] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[09:19:11] list index out of range handlers.py:36\n Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-10-07 09:19:08.058804","end":"2024-10-07 09:19:11.674907","duration":3}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:49:19] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n[09:49:20] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:49:21] ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[09:49:22] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n[09:49:23] Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-10-07 09:49:19.675611","end":"2024-10-07 09:49:23.070907","duration":3}},{"council_id":"CMD","missing":false,"latest_run":{"status_code":1,"log_text":"[09:54:31] Fetching Scraper for: CMD handlers.py:23\n Begin attempting to scrape: CMD handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:54:32] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.camden.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Client error '403 Forbidden' for url handlers.py:36\n 'https://democracy.camden.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/403 \n Finished attempting to scrape: CMD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://democracy.camden.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403\n","start":"2024-10-07 09:54:31.049979","end":"2024-10-07 09:54:32.931475","duration":1}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[09:29:29] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n[09:29:30] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:29:31] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[09:29:32] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n 'NoneType' object has no attribute 'find_parent' handlers.py:36\n[09:29:33] Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-10-07 09:29:29.813544","end":"2024-10-07 09:29:33.202112","duration":3}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[09:18:25] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n Deleting existing data... base.py:257\n[09:18:26] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:18:27] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-10-07 09:18:25.344522","end":"2024-10-07 09:18:27.398217","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[10:19:56] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n[10:19:57] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:19:58] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[10:20:14] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-07 10:19:56.933810","end":"2024-10-07 10:20:14.725198","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[10:01:10] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n Deleting existing data... base.py:257\n[10:01:11] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n[10:01:12] Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[10:01:17] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-10-07 10:01:10.088352","end":"2024-10-07 10:01:17.580496","duration":7}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"STG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:46:12] Fetching Scraper for: STG handlers.py:23\n Begin attempting to scrape: STG handlers.py:27\n[08:46:13] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 19 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 19 files in Councillors/raw base.py:225\n ...found 39 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 39 files base.py:236\n[08:46:14] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-elections/councillors/ \n[08:46:15] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/elaine-watterson/ \n[08:46:17] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/gene-maxwell/ \n[08:46:18] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/martin-earl/ \n[08:46:19] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/gerry-mcgarvey/ \n[08:46:20] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/paul-henke/ \n[08:46:21] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/rosemary-fraser/ \n[08:46:22] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/alasdair-tollemache/ \n[08:46:23] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/david-wilson/ \n[08:46:24] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/robin-kleinman/ \n[08:46:25] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/thomas-heald/ \n[08:46:26] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/danny-gibson/ \n[08:46:27] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/jim-thomson/ \n[08:46:28] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/rachel-nunn/ \n[08:46:29] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/susan-mcgill/ \n[08:46:30] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/jen-preston/ \n[08:46:31] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/neil-benny/ \n[08:46:32] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/scott-farmer/ \n[08:46:34] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/bryan-flannagan/ \n[08:46:35] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/gerry-mclaughlan/ \n[08:46:36] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/vacant/ \n[08:46:37] 'NoneType' object is not subscriptable handlers.py:36\n Committing batch 1 consisting of 38 files base.py:297\n[08:46:39] Finished attempting to scrape: STG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/STG-stirling/councillors.py\", line 42, in get_single_councillor\n councillor.email = soup.select_one(\"a[href^=mailto]\")[\nTypeError: 'NoneType' object is not subscriptable\n","start":"2024-10-07 08:46:12.656206","end":"2024-10-07 08:46:39.241713","duration":26}},{"council_id":"STS","missing":false,"latest_run":{"status_code":1,"log_text":"[10:01:46] Fetching Scraper for: STS handlers.py:23\n Begin attempting to scrape: STS handlers.py:27\n[10:01:47] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 62 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[10:01:48] ...found 62 files in Councillors/raw base.py:225\n ...found 125 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:01:49] Deleting batch no. 2 consisting of 25 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://moderngov.staffordshire.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard \n [Errno 104] Connection reset by peer handlers.py:36\n Finished attempting to scrape: STS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 104] Connection reset by peer\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 104] Connection reset by peer\n","start":"2024-10-07 10:01:46.606155","end":"2024-10-07 10:01:49.999774","duration":3}},{"council_id":"TEW","missing":false,"latest_run":{"status_code":1,"log_text":"[09:08:18] Fetching Scraper for: TEW handlers.py:23\n Begin attempting to scrape: TEW handlers.py:27\n[09:08:19] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:08:20] ...data deleted. base.py:264\n Scraping from base.py:49\n http://minutes.tewkesbury.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[09:08:50] timed out handlers.py:36\n Finished attempting to scrape: TEW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: timed out\n","start":"2024-10-07 09:08:18.833023","end":"2024-10-07 09:08:50.939637","duration":32}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:29:10] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:29:11] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[09:29:12] 'NoneType' object has no attribute 'findNext' handlers.py:36\n[09:29:13] Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-10-07 09:29:10.038768","end":"2024-10-07 09:29:13.123390","duration":3}},{"council_id":"VGL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:43:40] Fetching Scraper for: VGL handlers.py:23\n Begin attempting to scrape: VGL handlers.py:27\n[10:43:41] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 16 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[10:43:42] ...found 16 files in Councillors/raw base.py:225\n ...found 33 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 33 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Councillors.aspx \n[10:43:44] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Asbrey-Anne.aspx \n[10:43:46] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Aviet-Julie.aspx \n[10:43:47] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Ball-Gareth.aspx \n[10:43:49] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Birch-Rhiannon.aspx \n[10:43:50] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Brooks-Bronwen.aspx \n[10:43:51] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Bruce-Gillian.aspx \n[10:43:53] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Buckley-Ian.aspx \n[10:43:54] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Burnett-Lis.aspx \n[10:43:57] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Campbell-Samantha.aspx \n[10:43:59] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Carroll-George.aspx \n[10:44:00] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cave-Christine.aspx \n[10:44:02] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Champion-Charles.aspx \n[10:44:03] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Charles-Janice.aspx \n[10:44:04] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Collins-Amelia.aspx \n[10:44:06] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cowpe-Marianne.aspx \n[10:44:07] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Drake-Pamela.aspx \n[10:44:09] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Driscoll-Vincent.aspx \n[10:44:10] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 32 files base.py:297\n[10:44:12] Finished attempting to scrape: VGL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/VGL-the-vale-of-glamorgan/councillors.py\", line 36, in get_single_councillor\n email_text = soup.select_one(\"#S4_EmailPlaceholder\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-10-07 10:43:40.628191","end":"2024-10-07 10:44:12.148409","duration":31}},{"council_id":"WLN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:57:26] Fetching Scraper for: WLN handlers.py:23\n Begin attempting to scrape: WLN handlers.py:27\n Deleting existing data... base.py:257\n[08:57:27] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 25 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 25 files in Councillors/raw base.py:225\n ...found 51 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 51 files base.py:236\n[08:57:28] ...data deleted. base.py:264\n Scraping from https://westlothian.gov.uk/councillors base.py:49\n[08:57:30] Scraping from base.py:49\n https://westlothian.gov.uk/article/33888/Linlithgow \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33889/Broxburn-Uphall-a \n nd-Winchburgh \n[08:57:31] Scraping from base.py:49\n https://westlothian.gov.uk/article/33890/Livingston-North \n[08:57:32] Scraping from base.py:49\n https://westlothian.gov.uk/article/33891/Livingston-South \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33893/East-Livingston-a \n nd-East-Calder \n[08:57:33] Scraping from base.py:49\n https://westlothian.gov.uk/article/33892/Fauldhouse-and-th \n e-Breich-Valley \n[08:57:34] Scraping from base.py:49\n https://westlothian.gov.uk/article/33894/Whitburn-and-Blac \n kburn \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33895/Bathgate \n[08:57:35] Scraping from base.py:49\n https://westlothian.gov.uk/article/33896/Armadale-and-Blac \n kridge \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33897/Councillor-Tom-Co \n nn \n[08:57:36] Scraping from base.py:49\n https://westlothian.gov.uk/article/33898/Councillor-Paulin \n e-Orr \n[08:57:37] Scraping from base.py:49\n https://westlothian.gov.uk/article/33899/Councillor-Sally- \n Pattle \n[08:57:38] Scraping from base.py:49\n https://westlothian.gov.uk/article/34305/Councillor-Tony-B \n oyle \n[08:57:39] Scraping from base.py:49\n https://westlothian.gov.uk/article/34302/Councillor-Diane- \n Calder \n[08:57:40] Scraping from base.py:49\n https://westlothian.gov.uk/article/34303/Councillor-Janet- \n Campbell \n[08:57:41] Scraping from base.py:49\n https://westlothian.gov.uk/article/34304/Councillor-Angela \n -Doran-Timson \n[08:57:42] Scraping from base.py:49\n https://westlothian.gov.uk/article/34307/Councillor-Alison \n -Adamson \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34306/Councillor-Robert \n -De-Bold \n[08:57:43] Scraping from base.py:49\n https://westlothian.gov.uk/article/34308/Councillor-Anne-M \n cMillan \n[08:57:44] Scraping from base.py:49\n https://westlothian.gov.uk/article/34309/Councillor-Andrew \n -Miller \n[08:57:45] Scraping from base.py:49\n https://westlothian.gov.uk/article/34311/Councillor-Lawren \n ce-Fitzpatrick \n[08:57:46] Scraping from base.py:49\n https://westlothian.gov.uk/article/34310/Councillor-Peter- \n Heggie \n[08:57:47] Scraping from base.py:49\n https://westlothian.gov.uk/article/34312/Councillor-Maria- \n MacAulay \n[08:57:48] Scraping from base.py:49\n https://westlothian.gov.uk/article/34313/Councillor-Moira- \n McKee-Shemilt \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34317/Councillor-Damian \n -Doran-Timson \n[08:57:49] Scraping from base.py:49\n https://westlothian.gov.uk/article/34315/Councillor-Carl-J \n ohn \n[08:57:50] Scraping from base.py:49\n https://westlothian.gov.uk/article/34316/Councillor-Danny- \n Logue \n[08:57:51] Scraping from base.py:49\n https://westlothian.gov.uk/article/34314/Councillor-Veroni \n ca-Smith \n[08:57:52] Scraping from base.py:49\n https://westlothian.gov.uk/article/34318/Councillor-Paulin \n e-Clark \n[08:57:53] Scraping from base.py:49\n https://westlothian.gov.uk/article/34319/Councillor-Craig- \n Meek \n[08:57:54] Scraping from base.py:49\n https://westlothian.gov.uk/article/34320/Councillor-Cathy- \n Muldoon \n[08:57:55] Scraping from base.py:49\n https://westlothian.gov.uk/article/34321/Councillor-Jim-Di \n ckson \n[08:57:56] Scraping from base.py:49\n https://westlothian.gov.uk/article/34322/Councillor-Mary-D \n ickson \n[08:57:57] Scraping from base.py:49\n https://westlothian.gov.uk/article/34323/Councillor-George \n -Paul \n[08:57:58] Scraping from base.py:49\n https://westlothian.gov.uk/article/34324/Councillor-Vacant \n[08:57:59] list index out of range handlers.py:36\n Committing batch 1 consisting of 50 files base.py:297\n[08:58:00] Finished attempting to scrape: WLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/WLN-west-lothian/councillors.py\", line 37, in get_single_councillor\n councillor.email = soup.select(\"a[href^=mailto]\")[0].get_text(\nIndexError: list index out of range\n","start":"2024-10-07 08:57:26.437410","end":"2024-10-07 08:58:00.574441","duration":34}}] +[{"council_id":"BDG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:18:21] Fetching Scraper for: BDG handlers.py:23\n Begin attempting to scrape: BDG handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:18:22] ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgov.lbbd.gov.uk/internet/mgWebService.asmx/GetC \n ouncillorsByWard \n[08:18:37] [Errno 110] Connection timed out handlers.py:36\n[08:18:38] Finished attempting to scrape: BDG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-08 08:18:21.005056","end":"2024-10-08 08:18:38.266313","duration":17}},{"council_id":"BLA","missing":false,"latest_run":{"status_code":1,"log_text":"[09:24:52] Fetching Scraper for: BLA handlers.py:23\n Begin attempting to scrape: BLA handlers.py:27\n[09:24:53] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 36 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[09:24:54] ...found 36 files in Councillors/raw base.py:225\n ...found 73 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 73 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://w3.blaby.gov.uk/decision-making/mgWebService.asmx/ \n GetCouncillorsByWard \n Client error '403 Forbidden' for url handlers.py:36\n 'https://w3.blaby.gov.uk/decision-making/mgWebService. \n asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/403 \n[09:24:55] Finished attempting to scrape: BLA base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://w3.blaby.gov.uk/decision-making/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403\n","start":"2024-10-08 09:24:52.552597","end":"2024-10-08 09:24:55.090228","duration":2}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:48:30] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n[08:48:31] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:48:32] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n[08:48:33] Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-10-08 08:48:30.931272","end":"2024-10-08 08:48:33.187425","duration":2}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:30:36] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n Deleting existing data... base.py:257\n[08:30:37] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:30:38] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[08:30:40] list index out of range handlers.py:36\n Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-10-08 08:30:36.315056","end":"2024-10-08 08:30:40.607249","duration":4}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:56:42] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:56:43] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[08:56:45] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-10-08 08:56:42.004914","end":"2024-10-08 08:56:45.487831","duration":3}},{"council_id":"CMD","missing":false,"latest_run":{"status_code":1,"log_text":"[10:45:50] Fetching Scraper for: CMD handlers.py:23\n Begin attempting to scrape: CMD handlers.py:27\n[10:45:51] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:45:52] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.camden.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Client error '403 Forbidden' for url handlers.py:36\n 'https://democracy.camden.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/403 \n Finished attempting to scrape: CMD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://democracy.camden.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403\n","start":"2024-10-08 10:45:50.767484","end":"2024-10-08 10:45:52.638003","duration":1}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[08:23:40] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n Deleting existing data... base.py:257\n[08:23:41] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n[08:23:42] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[08:23:43] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n 'NoneType' object has no attribute 'find_parent' handlers.py:36\n[08:23:44] Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-10-08 08:23:40.199336","end":"2024-10-08 08:23:44.066992","duration":3}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[10:15:30] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n[10:15:31] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:15:32] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-10-08 10:15:30.615768","end":"2024-10-08 10:15:32.773199","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:52:09] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:52:10] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[09:52:26] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-08 09:52:09.027791","end":"2024-10-08 09:52:26.617690","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:37:53] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:37:54] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[09:37:58] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n[09:37:59] Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-10-08 09:37:53.100109","end":"2024-10-08 09:37:59.204711","duration":6}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"STG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:42:37] Fetching Scraper for: STG handlers.py:23\n Begin attempting to scrape: STG handlers.py:27\n[08:42:38] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 19 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[08:42:39] ...found 19 files in Councillors/raw base.py:225\n ...found 39 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 39 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-elections/councillors/ \n[08:42:41] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/elaine-watterson/ \n[08:42:43] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/gene-maxwell/ \n[08:42:44] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/martin-earl/ \n[08:42:46] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/gerry-mcgarvey/ \n[08:42:47] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/paul-henke/ \n[08:42:48] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/rosemary-fraser/ \n[08:42:49] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/alasdair-tollemache/ \n[08:42:51] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/david-wilson/ \n[08:42:52] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/robin-kleinman/ \n[08:42:53] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/thomas-heald/ \n[08:42:54] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/danny-gibson/ \n[08:42:55] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/jim-thomson/ \n[08:42:56] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/rachel-nunn/ \n[08:42:58] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/susan-mcgill/ \n[08:42:59] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/jen-preston/ \n[08:43:00] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/neil-benny/ \n[08:43:01] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/scott-farmer/ \n[08:43:02] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/bryan-flannagan/ \n[08:43:04] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/gerry-mclaughlan/ \n[08:43:05] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-councillors/your-councillors/vacant/ \n[08:43:06] 'NoneType' object is not subscriptable handlers.py:36\n Committing batch 1 consisting of 38 files base.py:297\n[08:43:07] Finished attempting to scrape: STG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/STG-stirling/councillors.py\", line 42, in get_single_councillor\n councillor.email = soup.select_one(\"a[href^=mailto]\")[\nTypeError: 'NoneType' object is not subscriptable\n","start":"2024-10-08 08:42:37.611796","end":"2024-10-08 08:43:07.816675","duration":30}},{"council_id":"TEW","missing":false,"latest_run":{"status_code":1,"log_text":"[08:35:17] Fetching Scraper for: TEW handlers.py:23\n[08:35:18] Begin attempting to scrape: TEW handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:35:19] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://minutes.tewkesbury.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[08:35:49] timed out handlers.py:36\n[08:35:50] Finished attempting to scrape: TEW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: timed out\n","start":"2024-10-08 08:35:17.976434","end":"2024-10-08 08:35:50.338163","duration":32}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:35:19] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n[09:35:20] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:35:21] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[09:35:23] 'NoneType' object has no attribute 'findNext' handlers.py:36\n Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-10-08 09:35:19.765619","end":"2024-10-08 09:35:23.381591","duration":3}},{"council_id":"VGL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:26:25] Fetching Scraper for: VGL handlers.py:23\n Begin attempting to scrape: VGL handlers.py:27\n[09:26:26] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[09:26:27] ...found 16 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 16 files in Councillors/raw base.py:225\n ...found 33 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 33 files base.py:236\n[09:26:28] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Councillors.aspx \n[09:26:29] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Asbrey-Anne.aspx \n[09:26:31] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Aviet-Julie.aspx \n[09:26:32] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Ball-Gareth.aspx \n[09:26:34] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Birch-Rhiannon.aspx \n[09:26:35] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Brooks-Bronwen.aspx \n[09:26:37] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Bruce-Gillian.aspx \n[09:26:38] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Buckley-Ian.aspx \n[09:26:40] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Burnett-Lis.aspx \n[09:26:41] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Campbell-Samantha.aspx \n[09:26:44] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Carroll-George.aspx \n[09:26:45] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cave-Christine.aspx \n[09:26:47] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Champion-Charles.aspx \n[09:26:48] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Charles-Janice.aspx \n[09:26:50] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Collins-Amelia.aspx \n[09:26:51] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cowpe-Marianne.aspx \n[09:26:53] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Drake-Pamela.aspx \n[09:26:54] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Driscoll-Vincent.aspx \n[09:26:56] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 32 files base.py:297\n[09:26:57] Finished attempting to scrape: VGL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/VGL-the-vale-of-glamorgan/councillors.py\", line 36, in get_single_councillor\n email_text = soup.select_one(\"#S4_EmailPlaceholder\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-10-08 09:26:25.709323","end":"2024-10-08 09:26:57.773506","duration":32}},{"council_id":"WLN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:55:10] Fetching Scraper for: WLN handlers.py:23\n Begin attempting to scrape: WLN handlers.py:27\n[08:55:11] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 25 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[08:55:12] ...found 25 files in Councillors/raw base.py:225\n ...found 51 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 51 files base.py:236\n ...data deleted. base.py:264\n Scraping from https://westlothian.gov.uk/councillors base.py:49\n[08:55:17] Scraping from base.py:49\n https://westlothian.gov.uk/article/33888/Linlithgow \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33889/Broxburn-Uphall-a \n nd-Winchburgh \n[08:55:18] Scraping from base.py:49\n https://westlothian.gov.uk/article/33890/Livingston-North \n[08:55:19] Scraping from base.py:49\n https://westlothian.gov.uk/article/33891/Livingston-South \n[08:55:20] Scraping from base.py:49\n https://westlothian.gov.uk/article/33893/East-Livingston-a \n nd-East-Calder \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33892/Fauldhouse-and-th \n e-Breich-Valley \n[08:55:21] Scraping from base.py:49\n https://westlothian.gov.uk/article/33894/Whitburn-and-Blac \n kburn \n[08:55:22] Scraping from base.py:49\n https://westlothian.gov.uk/article/33895/Bathgate \n[08:55:23] Scraping from base.py:49\n https://westlothian.gov.uk/article/33896/Armadale-and-Blac \n kridge \n[08:55:24] Scraping from base.py:49\n https://westlothian.gov.uk/article/33897/Councillor-Tom-Co \n nn \n[08:55:25] Scraping from base.py:49\n https://westlothian.gov.uk/article/33898/Councillor-Paulin \n e-Orr \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33899/Councillor-Sally- \n Pattle \n[08:55:26] Scraping from base.py:49\n https://westlothian.gov.uk/article/34305/Councillor-Tony-B \n oyle \n[08:55:27] Scraping from base.py:49\n https://westlothian.gov.uk/article/34302/Councillor-Diane- \n Calder \n[08:55:28] Scraping from base.py:49\n https://westlothian.gov.uk/article/34303/Councillor-Janet- \n Campbell \n[08:55:29] Scraping from base.py:49\n https://westlothian.gov.uk/article/34304/Councillor-Angela \n -Doran-Timson \n[08:55:30] Scraping from base.py:49\n https://westlothian.gov.uk/article/34307/Councillor-Alison \n -Adamson \n[08:55:31] Scraping from base.py:49\n https://westlothian.gov.uk/article/34306/Councillor-Robert \n -De-Bold \n[08:55:32] Scraping from base.py:49\n https://westlothian.gov.uk/article/34308/Councillor-Anne-M \n cMillan \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34309/Councillor-Andrew \n -Miller \n[08:55:33] Scraping from base.py:49\n https://westlothian.gov.uk/article/34311/Councillor-Lawren \n ce-Fitzpatrick \n[08:55:34] Scraping from base.py:49\n https://westlothian.gov.uk/article/34310/Councillor-Peter- \n Heggie \n[08:55:35] Scraping from base.py:49\n https://westlothian.gov.uk/article/34312/Councillor-Maria- \n MacAulay \n[08:55:36] Scraping from base.py:49\n https://westlothian.gov.uk/article/34313/Councillor-Moira- \n McKee-Shemilt \n[08:55:37] Scraping from base.py:49\n https://westlothian.gov.uk/article/34317/Councillor-Damian \n -Doran-Timson \n[08:55:38] Scraping from base.py:49\n https://westlothian.gov.uk/article/34315/Councillor-Carl-J \n ohn \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34316/Councillor-Danny- \n Logue \n[08:55:39] Scraping from base.py:49\n https://westlothian.gov.uk/article/34314/Councillor-Veroni \n ca-Smith \n[08:55:40] Scraping from base.py:49\n https://westlothian.gov.uk/article/34318/Councillor-Paulin \n e-Clark \n[08:55:41] Scraping from base.py:49\n https://westlothian.gov.uk/article/34319/Councillor-Craig- \n Meek \n[08:55:42] Scraping from base.py:49\n https://westlothian.gov.uk/article/34320/Councillor-Cathy- \n Muldoon \n[08:55:43] Scraping from base.py:49\n https://westlothian.gov.uk/article/34321/Councillor-Jim-Di \n ckson \n[08:55:44] Scraping from base.py:49\n https://westlothian.gov.uk/article/34322/Councillor-Mary-D \n ickson \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34323/Councillor-George \n -Paul \n[08:55:46] Scraping from base.py:49\n https://westlothian.gov.uk/article/34324/Councillor-Vacant \n[08:55:47] list index out of range handlers.py:36\n Committing batch 1 consisting of 50 files base.py:297\n[08:55:48] Finished attempting to scrape: WLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/WLN-west-lothian/councillors.py\", line 37, in get_single_councillor\n councillor.email = soup.select(\"a[href^=mailto]\")[0].get_text(\nIndexError: list index out of range\n","start":"2024-10-08 08:55:10.678428","end":"2024-10-08 08:55:48.916187","duration":38}}] diff --git a/index.html b/index.html index aa223d5e48..6f16334bdb 100644 --- a/index.html +++ b/index.html @@ -49,10 +49,6 @@
[10:03:31] Fetching Scraper for: ABC handlers.py:23 + Begin attempting to scrape: ABC handlers.py:27 + Deleting existing data... base.py:257 +[10:03:32] Getting all files in Councillors... base.py:209 + Getting all files in Councillors/json... base.py:209 + ...found 16 files in Councillors/json base.py:225 + Getting all files in Councillors/raw... base.py:209 + ...found 16 files in Councillors/raw base.py:225 + ...found 33 files in Councillors base.py:225 + Deleting batch no. 1 consisting of 33 files base.py:236 +[10:03:33] ...data deleted. base.py:264 + Scraping from base.py:49 + https://www.armaghbanbridgecraigavon.gov.uk/councillors/ +[10:03:36] Committing batch 1 consisting of 32 files base.py:297 +[10:03:38] Finished attempting to scrape: ABC base.py:345 ++ + + + +
[08:51:12] Fetching Scraper for: ABC handlers.py:23 - Begin attempting to scrape: ABC handlers.py:27 - Deleting existing data... base.py:257 -[08:51:13] Getting all files in Councillors... base.py:209 - Getting all files in Councillors/json... base.py:209 - ...found 16 files in Councillors/json base.py:225 - Getting all files in Councillors/raw... base.py:209 - ...found 16 files in Councillors/raw base.py:225 - ...found 33 files in Councillors base.py:225 - Deleting batch no. 1 consisting of 33 files base.py:236 -[08:51:14] ...data deleted. base.py:264 - Scraping from base.py:49 - https://www.armaghbanbridgecraigavon.gov.uk/councillors/ -[08:51:18] Committing batch 1 consisting of 32 files base.py:297 -[08:51:19] Finished attempting to scrape: ABC base.py:345 -- -