diff --git a/api/failing.json b/api/failing.json index b69fc01085..4723a5fd8d 100644 --- a/api/failing.json +++ b/api/failing.json @@ -1 +1 @@ -[{"council_id":"BDG","missing":false,"latest_run":{"status_code":1,"log_text":"[10:23:46] Fetching Scraper for: BDG handlers.py:23\n Begin attempting to scrape: BDG handlers.py:27\n Deleting existing data... base.py:257\n[10:23:47] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:23:48] ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgov.lbbd.gov.uk/internet/mgWebService.asmx/GetC \n ouncillorsByWard \n[10:24:03] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BDG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-13 10:23:46.333131","end":"2024-10-13 10:24:03.730183","duration":17}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:02:27] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n[09:02:28] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:02:29] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-10-13 09:02:27.666433","end":"2024-10-13 09:02:29.821977","duration":2}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:37:22] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n[08:37:23] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:37:24] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[08:37:26] list index out of range handlers.py:36\n[08:37:27] Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-10-13 08:37:22.816622","end":"2024-10-13 08:37:27.028334","duration":4}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:18:32] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:18:33] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[08:18:35] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-10-13 08:18:32.001508","end":"2024-10-13 08:18:35.521450","duration":3}},{"council_id":"CMD","missing":false,"latest_run":{"status_code":1,"log_text":"[09:56:54] Fetching Scraper for: CMD handlers.py:23\n Begin attempting to scrape: CMD handlers.py:27\n Deleting existing data... base.py:257\n[09:56:55] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:56:56] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.camden.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Client error '403 Forbidden' for url handlers.py:36\n 'https://democracy.camden.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/403 \n Finished attempting to scrape: CMD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://democracy.camden.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403\n","start":"2024-10-13 09:56:54.450880","end":"2024-10-13 09:56:56.833236","duration":2}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[09:19:28] Fetching Scraper for: EAY handlers.py:23\n[09:19:29] Begin attempting to scrape: EAY handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:19:30] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[09:19:31] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n[09:19:32] 'NoneType' object has no attribute 'find_parent' handlers.py:36\n Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-10-13 09:19:28.983256","end":"2024-10-13 09:19:32.831749","duration":3}},{"council_id":"FYL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:27:39] Fetching Scraper for: FYL handlers.py:23\n Begin attempting to scrape: FYL handlers.py:27\n[10:27:40] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[10:27:41] ...found 30 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 30 files in Councillors/raw base.py:225\n ...found 61 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 61 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://fylde.cmis.uk.com/fylde/CouncillorsandMP.aspx \n[10:27:42] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[10:27:45] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[10:27:47] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[10:27:50] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[10:27:51] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[10:27:54] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[10:27:55] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[10:27:59] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:01] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:03] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:05] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[10:28:08] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[10:28:09] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:11] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:13] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[10:28:16] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[10:28:18] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:20] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:25] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:27] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:29] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:31] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:32] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[10:28:36] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[10:28:37] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:39] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:41] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:43] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:45] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[10:28:49] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[10:28:51] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:54] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:55] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:58] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[10:28:59] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/21/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:02] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/21/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:03] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/35/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:06] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/35/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:08] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/37/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:12] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/37/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:13] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/153/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:16] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/153/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:18] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/212/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:20] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/212/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:21] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/187/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:24] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/187/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:25] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/24/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:28] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/24/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:29] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/177/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:32] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/177/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:35] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/16/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:38] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/16/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:39] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/167/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:42] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/167/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:44] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/99/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:47] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/99/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:48] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/34/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:51] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/34/ScreenMode/Alphabetical/D \n efault.aspx \n[10:29:53] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/210/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:55] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/210/ScreenMode/Alphabetical/ \n Default.aspx \n[10:29:58] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/221/ScreenMode/Alphabetical/ \n Default.aspx \n[10:30:00] 'NoneType' object has no attribute 'parent' handlers.py:36\n Committing batch 1 consisting of 60 files base.py:297\n[10:30:02] Finished attempting to scrape: FYL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 300, in get_single_councillor\n party = self.get_party_name(list_page_html)\n File \"scrapers/FYL-fylde/councillors.py\", line 17, in get_party_name\n .parent.get_text(strip=True)\nAttributeError: 'NoneType' object has no attribute 'parent'\n","start":"2024-10-13 10:27:39.899384","end":"2024-10-13 10:30:02.164437","duration":142}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[09:40:51] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n Deleting existing data... base.py:257\n[09:40:52] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n[09:40:53] Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-10-13 09:40:51.183352","end":"2024-10-13 09:40:53.271746","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[10:39:22] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n[10:39:23] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:39:24] ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[10:39:39] [Errno 110] Connection timed out handlers.py:36\n[10:39:40] Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-13 10:39:22.503927","end":"2024-10-13 10:39:40.145185","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:19:08] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n Deleting existing data... base.py:257\n[09:19:09] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:19:10] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[09:19:14] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-10-13 09:19:08.078596","end":"2024-10-13 09:19:15.018280","duration":6}},{"council_id":"MDB","missing":false,"latest_run":{"status_code":1,"log_text":"[09:44:37] Fetching Scraper for: MDB handlers.py:23\n Begin attempting to scrape: MDB handlers.py:27\n[09:44:38] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 46 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[09:44:39] ...found 46 files in Councillors/raw base.py:225\n ...found 93 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 93 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://moderngov.middlesbrough.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard \n[09:45:10] The read operation timed out handlers.py:36\n Finished attempting to scrape: MDB base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-10-13 09:44:37.574015","end":"2024-10-13 09:45:10.350473","duration":32}},{"council_id":"NGM","missing":false,"latest_run":{"status_code":1,"log_text":"[09:52:30] Fetching Scraper for: NGM handlers.py:23\n Begin attempting to scrape: NGM handlers.py:27\n[09:52:32] Deleting existing data... base.py:257\n[09:52:33] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 55 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 55 files in Councillors/raw base.py:225\n ...found 111 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:52:34] Deleting batch no. 2 consisting of 11 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://committee.nottinghamcity.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard \n[09:53:05] The read operation timed out handlers.py:36\n Finished attempting to scrape: NGM base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-10-13 09:52:30.416294","end":"2024-10-13 09:53:05.510980","duration":35}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"STG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:46:11] Fetching Scraper for: STG handlers.py:23\n Begin attempting to scrape: STG handlers.py:27\n[08:46:12] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:46:13] ...found 19 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 19 files in Councillors/raw base.py:225\n ...found 39 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 39 files base.py:236\n[08:46:14] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-elections/councillors/ \n[08:46:15] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/elaine-watterson/ \n[08:46:16] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gene-maxwell/ \n[08:46:17] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/martin-earl/ \n[08:46:19] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mcgarvey/ \n[08:46:20] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/paul-henke/ \n[08:46:21] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rosemary-fraser/ \n[08:46:22] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/alasdair-tollemache/ \n[08:46:24] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/david-wilson/ \n[08:46:25] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/robin-kleinman/ \n[08:46:26] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/thomas-heald/ \n[08:46:27] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/danny-gibson/ \n[08:46:29] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jim-thomson/ \n[08:46:30] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rachel-nunn/ \n[08:46:31] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/susan-mcgill/ \n[08:46:33] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jen-preston/ \n[08:46:34] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/neil-benny/ \n[08:46:35] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/scott-farmer/ \n[08:46:38] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/bryan-flannagan/ \n[08:46:39] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mclaughlan/ \n[08:46:40] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/vacant/ \n[08:46:41] 'NoneType' object is not subscriptable handlers.py:36\n[08:46:42] Committing batch 1 consisting of 38 files base.py:297\n[08:46:43] Finished attempting to scrape: STG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/STG-stirling/councillors.py\", line 42, in get_single_councillor\n councillor.email = soup.select_one(\"a[href^=mailto]\")[\nTypeError: 'NoneType' object is not subscriptable\n","start":"2024-10-13 08:46:11.873240","end":"2024-10-13 08:46:43.325981","duration":31}},{"council_id":"TEW","missing":false,"latest_run":{"status_code":1,"log_text":"[09:42:05] Fetching Scraper for: TEW handlers.py:23\n Begin attempting to scrape: TEW handlers.py:27\n[09:42:06] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:42:07] ...data deleted. base.py:264\n Scraping from base.py:49\n http://minutes.tewkesbury.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[09:42:37] timed out handlers.py:36\n Finished attempting to scrape: TEW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: timed out\n","start":"2024-10-13 09:42:05.553204","end":"2024-10-13 09:42:37.913675","duration":32}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:49:12] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n[09:49:13] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:49:14] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[09:49:17] 'NoneType' object has no attribute 'findNext' handlers.py:36\n Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-10-13 09:49:12.672602","end":"2024-10-13 09:49:17.791506","duration":5}},{"council_id":"VGL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:33:08] Fetching Scraper for: VGL handlers.py:23\n Begin attempting to scrape: VGL handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:33:09] Getting all files in Councillors/json... base.py:209\n ...found 16 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 16 files in Councillors/raw base.py:225\n ...found 33 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 33 files base.py:236\n[09:33:10] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Councillors.aspx \n[09:33:12] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Asbrey-Anne.aspx \n[09:33:15] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Aviet-Julie.aspx \n[09:33:17] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Ball-Gareth.aspx \n[09:33:18] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Birch-Rhiannon.aspx \n[09:33:20] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Brooks-Bronwen.aspx \n[09:33:21] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Bruce-Gillian.aspx \n[09:33:23] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Buckley-Ian.aspx \n[09:33:24] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Burnett-Lis.aspx \n[09:33:26] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Campbell-Samantha.aspx \n[09:33:27] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Carroll-George.aspx \n[09:33:29] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cave-Christine.aspx \n[09:33:31] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Champion-Charles.aspx \n[09:33:32] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Charles-Janice.aspx \n[09:33:34] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Collins-Amelia.aspx \n[09:33:35] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cowpe-Marianne.aspx \n[09:33:37] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Drake-Pamela.aspx \n[09:33:39] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Driscoll-Vincent.aspx \n[09:33:40] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 32 files base.py:297\n[09:33:42] Finished attempting to scrape: VGL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/VGL-the-vale-of-glamorgan/councillors.py\", line 36, in get_single_councillor\n email_text = soup.select_one(\"#S4_EmailPlaceholder\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-10-13 09:33:08.029842","end":"2024-10-13 09:33:42.290453","duration":34}},{"council_id":"WLN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:40:57] Fetching Scraper for: WLN handlers.py:23\n Begin attempting to scrape: WLN handlers.py:27\n Deleting existing data... base.py:257\n[09:40:58] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 25 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 25 files in Councillors/raw base.py:225\n ...found 51 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 51 files base.py:236\n[09:40:59] ...data deleted. base.py:264\n Scraping from https://westlothian.gov.uk/councillors base.py:49\n[09:41:01] Scraping from base.py:49\n https://westlothian.gov.uk/article/33888/Linlithgow \n[09:41:02] Scraping from base.py:49\n https://westlothian.gov.uk/article/33889/Broxburn-Uphall-a \n nd-Winchburgh \n[09:41:03] Scraping from base.py:49\n https://westlothian.gov.uk/article/33890/Livingston-North \n[09:41:04] Scraping from base.py:49\n https://westlothian.gov.uk/article/33891/Livingston-South \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33893/East-Livingston-a \n nd-East-Calder \n[09:41:05] Scraping from base.py:49\n https://westlothian.gov.uk/article/33892/Fauldhouse-and-th \n e-Breich-Valley \n[09:41:06] Scraping from base.py:49\n https://westlothian.gov.uk/article/33894/Whitburn-and-Blac \n kburn \n[09:41:07] Scraping from base.py:49\n https://westlothian.gov.uk/article/33895/Bathgate \n[09:41:08] Scraping from base.py:49\n https://westlothian.gov.uk/article/33896/Armadale-and-Blac \n kridge \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33897/Councillor-Tom-Co \n nn \n[09:41:09] Scraping from base.py:49\n https://westlothian.gov.uk/article/33898/Councillor-Paulin \n e-Orr \n[09:41:10] Scraping from base.py:49\n https://westlothian.gov.uk/article/33899/Councillor-Sally- \n Pattle \n[09:41:11] Scraping from base.py:49\n https://westlothian.gov.uk/article/34305/Councillor-Tony-B \n oyle \n[09:41:12] Scraping from base.py:49\n https://westlothian.gov.uk/article/34302/Councillor-Diane- \n Calder \n[09:41:13] Scraping from base.py:49\n https://westlothian.gov.uk/article/34303/Councillor-Janet- \n Campbell \n[09:41:14] Scraping from base.py:49\n https://westlothian.gov.uk/article/34304/Councillor-Angela \n -Doran-Timson \n[09:41:15] Scraping from base.py:49\n https://westlothian.gov.uk/article/34307/Councillor-Alison \n -Adamson \n[09:41:16] Scraping from base.py:49\n https://westlothian.gov.uk/article/34306/Councillor-Robert \n -De-Bold \n[09:41:17] Scraping from base.py:49\n https://westlothian.gov.uk/article/34308/Councillor-Anne-M \n cMillan \n[09:41:18] Scraping from base.py:49\n https://westlothian.gov.uk/article/34309/Councillor-Andrew \n -Miller \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34311/Councillor-Lawren \n ce-Fitzpatrick \n[09:41:19] Scraping from base.py:49\n https://westlothian.gov.uk/article/34310/Councillor-Peter- \n Heggie \n[09:41:20] Scraping from base.py:49\n https://westlothian.gov.uk/article/34312/Councillor-Maria- \n MacAulay \n[09:41:21] Scraping from base.py:49\n https://westlothian.gov.uk/article/34313/Councillor-Moira- \n McKee-Shemilt \n[09:41:23] Scraping from base.py:49\n https://westlothian.gov.uk/article/34317/Councillor-Damian \n -Doran-Timson \n[09:41:24] Scraping from base.py:49\n https://westlothian.gov.uk/article/34315/Councillor-Carl-J \n ohn \n[09:41:25] Scraping from base.py:49\n https://westlothian.gov.uk/article/34316/Councillor-Danny- \n Logue \n[09:41:26] Scraping from base.py:49\n https://westlothian.gov.uk/article/34314/Councillor-Veroni \n ca-Smith \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34318/Councillor-Paulin \n e-Clark \n[09:41:27] Scraping from base.py:49\n https://westlothian.gov.uk/article/34319/Councillor-Craig- \n Meek \n[09:41:28] Scraping from base.py:49\n https://westlothian.gov.uk/article/34320/Councillor-Cathy- \n Muldoon \n[09:41:29] Scraping from base.py:49\n https://westlothian.gov.uk/article/34321/Councillor-Jim-Di \n ckson \n[09:41:30] Scraping from base.py:49\n https://westlothian.gov.uk/article/34322/Councillor-Mary-D \n ickson \n[09:41:31] Scraping from base.py:49\n https://westlothian.gov.uk/article/34323/Councillor-George \n -Paul \n[09:41:32] Scraping from base.py:49\n https://westlothian.gov.uk/article/34324/Councillor-Vacant \n[09:41:33] list index out of range handlers.py:36\n Committing batch 1 consisting of 50 files base.py:297\n[09:41:34] Finished attempting to scrape: WLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/WLN-west-lothian/councillors.py\", line 37, in get_single_councillor\n councillor.email = soup.select(\"a[href^=mailto]\")[0].get_text(\nIndexError: list index out of range\n","start":"2024-10-13 09:40:57.179810","end":"2024-10-13 09:41:34.613399","duration":37}}] +[{"council_id":"BDG","missing":false,"latest_run":{"status_code":1,"log_text":"[10:07:54] Fetching Scraper for: BDG handlers.py:23\n Begin attempting to scrape: BDG handlers.py:27\n[10:07:55] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:07:56] ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgov.lbbd.gov.uk/internet/mgWebService.asmx/GetC \n ouncillorsByWard \n[10:08:11] [Errno 110] Connection timed out handlers.py:36\n[10:08:12] Finished attempting to scrape: BDG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-14 10:07:54.673928","end":"2024-10-14 10:08:12.190946","duration":17}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:42:15] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:42:16] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n[10:42:17] Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-10-14 10:42:15.024020","end":"2024-10-14 10:42:17.031221","duration":2}},{"council_id":"BRT","missing":false,"latest_run":{"status_code":1,"log_text":"[10:51:51] Fetching Scraper for: BRT handlers.py:23\n Begin attempting to scrape: BRT handlers.py:27\n[10:51:53] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[10:51:54] ...found 44 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 44 files in Councillors/raw base.py:225\n ...found 89 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 89 files base.py:236\n[10:51:55] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.broxtowe.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[10:52:10] [Errno 110] Connection timed out handlers.py:36\n[10:52:11] Finished attempting to scrape: BRT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-14 10:51:51.360561","end":"2024-10-14 10:52:11.088513","duration":19}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:24:47] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n Deleting existing data... base.py:257\n[09:24:48] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:24:49] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[09:24:51] list index out of range handlers.py:36\n Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-10-14 09:24:47.487051","end":"2024-10-14 09:24:51.835127","duration":4}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:52:38] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n Deleting existing data... base.py:257\n[09:52:39] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:52:40] ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[09:52:41] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-10-14 09:52:38.202087","end":"2024-10-14 09:52:41.846619","duration":3}},{"council_id":"CMD","missing":false,"latest_run":{"status_code":1,"log_text":"[09:50:07] Fetching Scraper for: CMD handlers.py:23\n Begin attempting to scrape: CMD handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:50:08] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.camden.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Client error '403 Forbidden' for url handlers.py:36\n 'https://democracy.camden.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/403 \n[09:50:09] Finished attempting to scrape: CMD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '403 Forbidden' for url 'https://democracy.camden.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403\n","start":"2024-10-14 09:50:07.121564","end":"2024-10-14 09:50:09.026090","duration":1}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[10:45:32] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n[10:45:33] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:45:34] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[10:45:35] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n[10:45:36] 'NoneType' object has no attribute 'find_parent' handlers.py:36\n Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-10-14 10:45:32.645824","end":"2024-10-14 10:45:36.528283","duration":3}},{"council_id":"FYL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:33:35] Fetching Scraper for: FYL handlers.py:23\n Begin attempting to scrape: FYL handlers.py:27\n Deleting existing data... base.py:257\n[09:33:36] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 30 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 30 files in Councillors/raw base.py:225\n ...found 61 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 61 files base.py:236\n[09:33:37] ...data deleted. base.py:264\n Scraping from base.py:49\n https://fylde.cmis.uk.com/fylde/CouncillorsandMP.aspx \n[09:33:38] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[09:33:42] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[09:33:43] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[09:33:46] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[09:33:47] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[09:33:50] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[09:33:52] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[09:33:54] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[09:33:56] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:00] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:01] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[09:34:04] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[09:34:06] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:08] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:09] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[09:34:13] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[09:34:15] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:18] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:20] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:24] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:25] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:28] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:29] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[09:34:32] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[09:34:34] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:36] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:38] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:40] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:42] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[09:34:44] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[09:34:50] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:54] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[09:34:55] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:01] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:03] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/21/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:05] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/21/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:06] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/35/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:10] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/35/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:12] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/37/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:15] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/37/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:16] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/153/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:21] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/153/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:23] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/212/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:25] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/212/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:27] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/187/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:29] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/187/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:31] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/24/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:34] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/24/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:36] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/177/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:39] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/177/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:40] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/16/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:44] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/16/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:46] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/167/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:49] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/167/ScreenMode/Alphabetical/ \n Default.aspx \n[09:35:51] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/99/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:54] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/99/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:55] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/34/ScreenMode/Alphabetical/D \n efault.aspx \n[09:35:58] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/34/ScreenMode/Alphabetical/D \n efault.aspx \n[09:36:00] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/210/ScreenMode/Alphabetical/ \n Default.aspx \n[09:36:02] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/210/ScreenMode/Alphabetical/ \n Default.aspx \n[09:36:04] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/221/ScreenMode/Alphabetical/ \n Default.aspx \n[09:36:06] 'NoneType' object has no attribute 'parent' handlers.py:36\n Committing batch 1 consisting of 60 files base.py:297\n[09:36:07] Finished attempting to scrape: FYL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 300, in get_single_councillor\n party = self.get_party_name(list_page_html)\n File \"scrapers/FYL-fylde/councillors.py\", line 17, in get_party_name\n .parent.get_text(strip=True)\nAttributeError: 'NoneType' object has no attribute 'parent'\n","start":"2024-10-14 09:33:35.387569","end":"2024-10-14 09:36:07.700098","duration":152}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[10:38:03] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n[10:38:04] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:38:05] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n[10:38:06] Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-10-14 10:38:03.924649","end":"2024-10-14 10:38:06.164819","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:49:04] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n[09:49:05] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:49:06] ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[09:49:21] [Errno 110] Connection timed out handlers.py:36\n[09:49:22] Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-10-14 09:49:04.331425","end":"2024-10-14 09:49:22.302976","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[10:27:01] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n Deleting existing data... base.py:257\n[10:27:02] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:27:03] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[10:27:05] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-10-14 10:27:01.573681","end":"2024-10-14 10:27:05.939628","duration":4}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"STG","missing":false,"latest_run":{"status_code":1,"log_text":"[10:36:41] Fetching Scraper for: STG handlers.py:23\n Begin attempting to scrape: STG handlers.py:27\n[10:36:42] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 19 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[10:36:43] ...found 19 files in Councillors/raw base.py:225\n ...found 39 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 39 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-elections/councillors/ \n[10:36:45] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/elaine-watterson/ \n[10:36:46] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gene-maxwell/ \n[10:36:47] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/martin-earl/ \n[10:36:49] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mcgarvey/ \n[10:36:50] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/paul-henke/ \n[10:36:51] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rosemary-fraser/ \n[10:36:53] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/alasdair-tollemache/ \n[10:36:54] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/david-wilson/ \n[10:36:55] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/robin-kleinman/ \n[10:36:56] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/thomas-heald/ \n[10:36:58] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/danny-gibson/ \n[10:36:59] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jim-thomson/ \n[10:37:02] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rachel-nunn/ \n[10:37:03] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/susan-mcgill/ \n[10:37:05] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jen-preston/ \n[10:37:06] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/neil-benny/ \n[10:37:07] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/scott-farmer/ \n[10:37:08] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/bryan-flannagan/ \n[10:37:10] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mclaughlan/ \n[10:37:11] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/vacant/ \n[10:37:12] 'NoneType' object is not subscriptable handlers.py:36\n Committing batch 1 consisting of 38 files base.py:297\n[10:37:13] Finished attempting to scrape: STG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/STG-stirling/councillors.py\", line 42, in get_single_councillor\n councillor.email = soup.select_one(\"a[href^=mailto]\")[\nTypeError: 'NoneType' object is not subscriptable\n","start":"2024-10-14 10:36:41.524142","end":"2024-10-14 10:37:13.864649","duration":32}},{"council_id":"TEW","missing":false,"latest_run":{"status_code":1,"log_text":"[08:30:54] Fetching Scraper for: TEW handlers.py:23\n Begin attempting to scrape: TEW handlers.py:27\n Deleting existing data... base.py:257\n[08:30:55] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:30:56] ...data deleted. base.py:264\n Scraping from base.py:49\n http://minutes.tewkesbury.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[08:31:26] timed out handlers.py:36\n Finished attempting to scrape: TEW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: timed out\n","start":"2024-10-14 08:30:54.528162","end":"2024-10-14 08:31:26.919340","duration":32}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[10:26:40] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n Deleting existing data... base.py:257\n[10:26:41] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:26:42] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[10:26:44] 'NoneType' object has no attribute 'findNext' handlers.py:36\n Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-10-14 10:26:40.482363","end":"2024-10-14 10:26:44.182807","duration":3}},{"council_id":"TRF","missing":false,"latest_run":{"status_code":1,"log_text":"[10:16:46] Fetching Scraper for: TRF handlers.py:23\n Begin attempting to scrape: TRF handlers.py:27\n[10:16:47] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[10:16:48] ...found 63 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 63 files in Councillors/raw base.py:225\n ...found 127 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:16:49] Deleting batch no. 2 consisting of 27 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democratic.trafford.gov.uk/mgWebService.asmx/GetCo \n uncillorsByWard \n[10:17:19] The read operation timed out handlers.py:36\n[10:17:20] Finished attempting to scrape: TRF base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-10-14 10:16:46.748519","end":"2024-10-14 10:17:20.347433","duration":33}},{"council_id":"VGL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:21:01] Fetching Scraper for: VGL handlers.py:23\n Begin attempting to scrape: VGL handlers.py:27\n Deleting existing data... base.py:257\n[10:21:02] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 16 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 16 files in Councillors/raw base.py:225\n ...found 33 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 33 files base.py:236\n[10:21:03] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Councillors.aspx \n[10:21:05] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Asbrey-Anne.aspx \n[10:21:07] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Aviet-Julie.aspx \n[10:21:09] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Ball-Gareth.aspx \n[10:21:10] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Birch-Rhiannon.aspx \n[10:21:12] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Brooks-Bronwen.aspx \n[10:21:14] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Bruce-Gillian.aspx \n[10:21:16] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Buckley-Ian.aspx \n[10:21:17] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Burnett-Lis.aspx \n[10:21:19] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Campbell-Samantha.aspx \n[10:21:22] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Carroll-George.aspx \n[10:21:23] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cave-Christine.aspx \n[10:21:25] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Champion-Charles.aspx \n[10:21:26] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Charles-Janice.aspx \n[10:21:28] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Collins-Amelia.aspx \n[10:21:29] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cowpe-Marianne.aspx \n[10:21:31] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Drake-Pamela.aspx \n[10:21:32] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Driscoll-Vincent.aspx \n[10:21:34] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 32 files base.py:297\n[10:21:36] Finished attempting to scrape: VGL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/VGL-the-vale-of-glamorgan/councillors.py\", line 36, in get_single_councillor\n email_text = soup.select_one(\"#S4_EmailPlaceholder\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-10-14 10:21:01.361150","end":"2024-10-14 10:21:36.028093","duration":34}},{"council_id":"WLN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:55:24] Fetching Scraper for: WLN handlers.py:23\n Begin attempting to scrape: WLN handlers.py:27\n[09:55:25] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[09:55:26] ...found 25 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 25 files in Councillors/raw base.py:225\n ...found 51 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 51 files base.py:236\n[09:55:27] ...data deleted. base.py:264\n Scraping from https://westlothian.gov.uk/councillors base.py:49\n[09:55:29] Scraping from base.py:49\n https://westlothian.gov.uk/article/33888/Linlithgow \n[09:55:30] Scraping from base.py:49\n https://westlothian.gov.uk/article/33889/Broxburn-Uphall-a \n nd-Winchburgh \n[09:55:31] Scraping from base.py:49\n https://westlothian.gov.uk/article/33890/Livingston-North \n[09:55:32] Scraping from base.py:49\n https://westlothian.gov.uk/article/33891/Livingston-South \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33893/East-Livingston-a \n nd-East-Calder \n[09:55:33] Scraping from base.py:49\n https://westlothian.gov.uk/article/33892/Fauldhouse-and-th \n e-Breich-Valley \n[09:55:34] Scraping from base.py:49\n https://westlothian.gov.uk/article/33894/Whitburn-and-Blac \n kburn \n[09:55:35] Scraping from base.py:49\n https://westlothian.gov.uk/article/33895/Bathgate \n[09:55:36] Scraping from base.py:49\n https://westlothian.gov.uk/article/33896/Armadale-and-Blac \n kridge \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33897/Councillor-Tom-Co \n nn \n[09:55:37] Scraping from base.py:49\n https://westlothian.gov.uk/article/33898/Councillor-Paulin \n e-Orr \n[09:55:38] Scraping from base.py:49\n https://westlothian.gov.uk/article/33899/Councillor-Sally- \n Pattle \n[09:55:40] Scraping from base.py:49\n https://westlothian.gov.uk/article/34305/Councillor-Tony-B \n oyle \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34302/Councillor-Diane- \n Calder \n[09:55:41] Scraping from base.py:49\n https://westlothian.gov.uk/article/34303/Councillor-Janet- \n Campbell \n[09:55:42] Scraping from base.py:49\n https://westlothian.gov.uk/article/34304/Councillor-Angela \n -Doran-Timson \n[09:55:44] Scraping from base.py:49\n https://westlothian.gov.uk/article/34307/Councillor-Alison \n -Adamson \n[09:55:45] Scraping from base.py:49\n https://westlothian.gov.uk/article/34306/Councillor-Robert \n -De-Bold \n[09:55:46] Scraping from base.py:49\n https://westlothian.gov.uk/article/34308/Councillor-Anne-M \n cMillan \n[09:55:47] Scraping from base.py:49\n https://westlothian.gov.uk/article/34309/Councillor-Andrew \n -Miller \n[09:55:48] Scraping from base.py:49\n https://westlothian.gov.uk/article/34311/Councillor-Lawren \n ce-Fitzpatrick \n[09:55:49] Scraping from base.py:49\n https://westlothian.gov.uk/article/34310/Councillor-Peter- \n Heggie \n[09:55:50] Scraping from base.py:49\n https://westlothian.gov.uk/article/34312/Councillor-Maria- \n MacAulay \n[09:55:51] Scraping from base.py:49\n https://westlothian.gov.uk/article/34313/Councillor-Moira- \n McKee-Shemilt \n[09:55:52] Scraping from base.py:49\n https://westlothian.gov.uk/article/34317/Councillor-Damian \n -Doran-Timson \n[09:55:53] Scraping from base.py:49\n https://westlothian.gov.uk/article/34315/Councillor-Carl-J \n ohn \n[09:55:54] Scraping from base.py:49\n https://westlothian.gov.uk/article/34316/Councillor-Danny- \n Logue \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34314/Councillor-Veroni \n ca-Smith \n[09:55:56] Scraping from base.py:49\n https://westlothian.gov.uk/article/34318/Councillor-Paulin \n e-Clark \n[09:55:57] Scraping from base.py:49\n https://westlothian.gov.uk/article/34319/Councillor-Craig- \n Meek \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34320/Councillor-Cathy- \n Muldoon \n[09:55:58] Scraping from base.py:49\n https://westlothian.gov.uk/article/34321/Councillor-Jim-Di \n ckson \n[09:56:00] Scraping from base.py:49\n https://westlothian.gov.uk/article/34322/Councillor-Mary-D \n ickson \n[09:56:01] Scraping from base.py:49\n https://westlothian.gov.uk/article/34323/Councillor-George \n -Paul \n[09:56:03] Scraping from base.py:49\n https://westlothian.gov.uk/article/34324/Councillor-Vacant \n[09:56:04] list index out of range handlers.py:36\n Committing batch 1 consisting of 50 files base.py:297\n[09:56:05] Finished attempting to scrape: WLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/WLN-west-lothian/councillors.py\", line 37, in get_single_councillor\n councillor.email = soup.select(\"a[href^=mailto]\")[0].get_text(\nIndexError: list index out of range\n","start":"2024-10-14 09:55:24.804490","end":"2024-10-14 09:56:05.646100","duration":40}}] diff --git a/index.html b/index.html index 03f42bc4ef..a193531b1b 100644 --- a/index.html +++ b/index.html @@ -49,10 +49,6 @@
[08:19:27] Fetching Scraper for: ABC handlers.py:23 + Begin attempting to scrape: ABC handlers.py:27 + Deleting existing data... base.py:257 +[08:19:28] Getting all files in Councillors... base.py:209 + Getting all files in Councillors/json... base.py:209 + ...found 16 files in Councillors/json base.py:225 + Getting all files in Councillors/raw... base.py:209 + ...found 16 files in Councillors/raw base.py:225 + ...found 33 files in Councillors base.py:225 + Deleting batch no. 1 consisting of 33 files base.py:236 +[08:19:29] ...data deleted. base.py:264 + Scraping from base.py:49 + https://www.armaghbanbridgecraigavon.gov.uk/councillors/ +[08:19:34] Committing batch 1 consisting of 32 files base.py:297 +[08:19:35] Finished attempting to scrape: ABC base.py:345 ++ + + + +
[10:02:43] Fetching Scraper for: ABC handlers.py:23 - Begin attempting to scrape: ABC handlers.py:27 - Deleting existing data... base.py:257 -[10:02:44] Getting all files in Councillors... base.py:209 - Getting all files in Councillors/json... base.py:209 - ...found 16 files in Councillors/json base.py:225 - Getting all files in Councillors/raw... base.py:209 - ...found 16 files in Councillors/raw base.py:225 - ...found 33 files in Councillors base.py:225 - Deleting batch no. 1 consisting of 33 files base.py:236 -[10:02:45] ...data deleted. base.py:264 - Scraping from base.py:49 - https://www.armaghbanbridgecraigavon.gov.uk/councillors/ -[10:02:49] Committing batch 1 consisting of 32 files base.py:297 -[10:02:50] Finished attempting to scrape: ABC base.py:345 -- -