diff --git a/api/failing.json b/api/failing.json index 9d754ac487..39a0656695 100644 --- a/api/failing.json +++ b/api/failing.json @@ -1 +1 @@ -[{"council_id":"BAN","missing":false,"latest_run":{"status_code":1,"log_text":"[10:12:42] Fetching Scraper for: BAN handlers.py:23\n Begin attempting to scrape: BAN handlers.py:27\n Deleting existing data... base.py:257\n[10:12:43] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 54 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 54 files in Councillors/raw base.py:225\n ...found 109 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:12:44] Deleting batch no. 2 consisting of 9 files base.py:236\n[10:12:45] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.basingstoke.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard \n[10:13:00] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 10:12:42.270718","end":"2024-11-07 10:13:00.900120","duration":18}},{"council_id":"BDF","missing":false,"latest_run":{"status_code":1,"log_text":"[08:18:30] Fetching Scraper for: BDF handlers.py:23\n Begin attempting to scrape: BDF handlers.py:27\n Deleting existing data... base.py:257\n[08:18:31] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 47 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 47 files in Councillors/raw base.py:225\n ...found 95 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 95 files base.py:236\n[08:18:32] ...data deleted. base.py:264\n Scraping from base.py:49\n https://councillorsupport.bedford.gov.uk/mgWebService.asmx \n /GetCouncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://councillorsupport.bedford.gov.uk/mgWebService \n .asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: BDF base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://councillorsupport.bedford.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-07 08:18:30.302431","end":"2024-11-07 08:18:32.921287","duration":2}},{"council_id":"BDG","missing":false,"latest_run":{"status_code":1,"log_text":"[10:18:53] Fetching Scraper for: BDG handlers.py:23\n Begin attempting to scrape: BDG handlers.py:27\n[10:18:54] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:18:55] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgov.lbbd.gov.uk/internet/mgWebService.asmx/GetC \n ouncillorsByWard \n[10:19:11] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BDG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 10:18:53.876902","end":"2024-11-07 10:19:11.881274","duration":18}},{"council_id":"BEN","missing":false,"latest_run":{"status_code":1,"log_text":"[10:44:35] Fetching Scraper for: BEN handlers.py:23\n Begin attempting to scrape: BEN handlers.py:27\n Deleting existing data... base.py:257\n[10:44:36] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 57 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 57 files in Councillors/raw base.py:225\n ...found 115 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:44:37] Deleting batch no. 2 consisting of 15 files base.py:236\n[10:44:38] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.brent.gov.uk/mgWebService.asmx/GetCouncil \n lorsByWard \n[10:44:53] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BEN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 10:44:35.141578","end":"2024-11-07 10:44:53.996733","duration":18}},{"council_id":"BGE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:51:53] Fetching Scraper for: BGE handlers.py:23\n Begin attempting to scrape: BGE handlers.py:27\n Deleting existing data... base.py:257\n[09:51:54] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 51 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 51 files in Councillors/raw base.py:225\n ...found 103 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:51:55] Deleting batch no. 2 consisting of 3 files base.py:236\n[09:51:56] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democratic.bridgend.gov.uk/mgWebService.asmx/GetCo \n uncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://democratic.bridgend.gov.uk/mgWebService.asmx/ \n GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: BGE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://democratic.bridgend.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-07 09:51:53.194566","end":"2024-11-07 09:51:56.588568","duration":3}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:50:13] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n[09:50:14] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:50:15] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-11-07 09:50:13.884273","end":"2024-11-07 09:50:15.969907","duration":2}},{"council_id":"BOS","missing":false,"latest_run":{"status_code":1,"log_text":"[10:24:19] Fetching Scraper for: BOS handlers.py:23\n Begin attempting to scrape: BOS handlers.py:27\n Deleting existing data... base.py:257\n[10:24:20] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 37 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 37 files in Councillors/raw base.py:225\n ...found 75 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 75 files base.py:236\n[10:24:21] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.bolsover.gov.uk/mgWebService.asmx/GetCo \n uncillorsByWard \n Server error '502 Bad Gateway' for url handlers.py:36\n 'https://committees.bolsover.gov.uk/mgWebService.asmx/ \n GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n Finished attempting to scrape: BOS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://committees.bolsover.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-07 10:24:19.161383","end":"2024-11-07 10:24:21.623709","duration":2}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[10:18:10] Fetching Scraper for: CAN handlers.py:23\n Begin attempting to scrape: CAN handlers.py:27\n Deleting existing data... base.py:257\n[10:18:11] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:18:12] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[10:18:15] list index out of range handlers.py:36\n Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-11-07 10:18:10.484169","end":"2024-11-07 10:18:15.451147","duration":4}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:43:27] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n[08:43:28] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:43:29] ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[08:43:30] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n[08:43:31] Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-11-07 08:43:27.650154","end":"2024-11-07 08:43:31.250359","duration":3}},{"council_id":"CAT","missing":false,"latest_run":{"status_code":1,"log_text":"[08:25:56] Fetching Scraper for: CAT handlers.py:23\n Begin attempting to scrape: CAT handlers.py:27\n[08:25:57] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 39 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[08:25:58] ...found 39 files in Councillors/raw base.py:225\n ...found 79 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 79 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.canterbury.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n[08:26:14] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: CAT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 08:25:56.596200","end":"2024-11-07 08:26:14.566700","duration":17}},{"council_id":"CAY","missing":false,"latest_run":{"status_code":1,"log_text":"[10:37:44] Fetching Scraper for: CAY handlers.py:23\n Begin attempting to scrape: CAY handlers.py:27\n[10:37:45] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[10:37:46] ...found 69 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 69 files in Councillors/raw base.py:225\n ...found 139 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:37:47] Deleting batch no. 2 consisting of 39 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.caerphilly.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n[10:37:51] [Errno 113] No route to host handlers.py:36\n Finished attempting to scrape: CAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 113] No route to host\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 113] No route to host\n","start":"2024-11-07 10:37:44.889657","end":"2024-11-07 10:37:51.186136","duration":6}},{"council_id":"CHW","missing":false,"latest_run":{"status_code":1,"log_text":"[09:46:39] Fetching Scraper for: CHW handlers.py:23\n Begin attempting to scrape: CHW handlers.py:27\n[09:46:40] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[09:46:41] ...found 71 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 71 files in Councillors/raw base.py:225\n ...found 143 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n Deleting batch no. 2 consisting of 43 files base.py:236\n[09:46:42] ...data deleted. base.py:264\n Scraping from base.py:49\n http://cmttpublic.cheshirewestandchester.gov.uk/mgWebServi \n ce.asmx/GetCouncillorsByWard \n[09:46:58] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: CHW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 09:46:39.792010","end":"2024-11-07 09:46:58.329495","duration":18}},{"council_id":"COT","missing":false,"latest_run":{"status_code":1,"log_text":"[09:53:29] Fetching Scraper for: COT handlers.py:23\n Begin attempting to scrape: COT handlers.py:27\n[09:53:30] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:53:31] Getting all files in Councillors/json... base.py:209\n ...found 34 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 34 files in Councillors/raw base.py:225\n ...found 69 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 69 files base.py:236\n[09:53:32] ...data deleted. base.py:264\n Scraping from base.py:49\n https://meetings.cotswold.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[09:54:02] The read operation timed out handlers.py:36\n[09:54:03] Finished attempting to scrape: COT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-11-07 09:53:29.857895","end":"2024-11-07 09:54:03.030936","duration":33}},{"council_id":"CWY","missing":false,"latest_run":{"status_code":1,"log_text":"[09:47:02] Fetching Scraper for: CWY handlers.py:23\n Begin attempting to scrape: CWY handlers.py:27\n Deleting existing data... base.py:257\n[09:47:03] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:47:04] ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgovpub-eng.secure.conwy.gov.uk/mgWebService.asm \n x/GetCouncillorsByWard \n[09:47:34] The read operation timed out handlers.py:36\n Finished attempting to scrape: CWY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-11-07 09:47:02.063314","end":"2024-11-07 09:47:34.512398","duration":32}},{"council_id":"DND","missing":false,"latest_run":{"status_code":1,"log_text":"[08:36:11] Fetching Scraper for: DND handlers.py:23\n Begin attempting to scrape: DND handlers.py:27\n Deleting existing data... base.py:257\n[08:36:12] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 42 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 42 files in Councillors/raw base.py:225\n ...found 85 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 85 files base.py:236\n[08:36:13] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.northdevon.gov.uk//mgWebService.asmx/Get \n CouncillorsByWard \n[08:36:44] [Errno 110] Connection timed out handlers.py:36\n[08:36:45] Finished attempting to scrape: DND base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 08:36:11.448343","end":"2024-11-07 08:36:45.085387","duration":33}},{"council_id":"DUR","missing":false,"latest_run":{"status_code":1,"log_text":"[10:11:36] Fetching Scraper for: DUR handlers.py:23\n Begin attempting to scrape: DUR handlers.py:27\n[10:11:37] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 126 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 126 files in Councillors/raw base.py:225\n ...found 253 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:11:38] Deleting batch no. 2 consisting of 100 files base.py:236\n[10:11:39] Deleting batch no. 3 consisting of 53 files base.py:236\n[10:11:40] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.durham.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n[10:11:50] [Errno 104] Connection reset by peer handlers.py:36\n Finished attempting to scrape: DUR base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 104] Connection reset by peer\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 104] Connection reset by peer\n","start":"2024-11-07 10:11:36.567294","end":"2024-11-07 10:11:50.899418","duration":14}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[08:33:58] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n Deleting existing data... base.py:257\n[08:33:59] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:34:00] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[08:34:01] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n 'NoneType' object has no attribute 'find_parent' handlers.py:36\n[08:34:02] Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-11-07 08:33:58.421089","end":"2024-11-07 08:34:02.144656","duration":3}},{"council_id":"EDE","missing":false,"latest_run":{"status_code":1,"log_text":"[10:48:16] Fetching Scraper for: EDE handlers.py:23\n Begin attempting to scrape: EDE handlers.py:27\n[10:48:17] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:48:18] Getting all files in Councillors/json... base.py:209\n ...found 60 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 60 files in Councillors/raw base.py:225\n ...found 121 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:48:19] Deleting batch no. 2 consisting of 21 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.eastdevon.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n[10:48:22] [Errno 113] No route to host handlers.py:36\n Finished attempting to scrape: EDE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 113] No route to host\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 113] No route to host\n","start":"2024-11-07 10:48:16.914608","end":"2024-11-07 10:48:22.289173","duration":5}},{"council_id":"EDU","missing":false,"latest_run":{"status_code":1,"log_text":"[08:47:29] Fetching Scraper for: EDU handlers.py:23\n Begin attempting to scrape: EDU handlers.py:27\n[08:47:30] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n[08:47:31] Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.eastdunbarton.gov.uk/residents/council-democra \n cy/committees-and-councillors/councillors-2017 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.eastdunbarton.gov.uk/residents/council-de \n mocracy/committees-and-councillors/councillors-2017' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n[08:47:32] Finished attempting to scrape: EDU base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.eastdunbarton.gov.uk/residents/council-democracy/committees-and-councillors/councillors-2017'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-07 08:47:29.931584","end":"2024-11-07 08:47:32.118472","duration":2}},{"council_id":"EHE","missing":false,"latest_run":{"status_code":1,"log_text":"[11:09:18] Fetching Scraper for: EHE handlers.py:23\n Begin attempting to scrape: EHE handlers.py:27\n[11:09:19] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[11:09:21] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.eastherts.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n[11:09:36] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: EHE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 11:09:18.786323","end":"2024-11-07 11:09:36.851742","duration":18}},{"council_id":"FEN","missing":false,"latest_run":{"status_code":1,"log_text":"[10:42:26] Fetching Scraper for: FEN handlers.py:23\n Begin attempting to scrape: FEN handlers.py:27\n Deleting existing data... base.py:257\n[10:42:27] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 43 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 43 files in Councillors/raw base.py:225\n ...found 87 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 87 files base.py:236\n[10:42:28] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.fenland.gov.uk/localgov//mgWebService.asmx/Get \n CouncillorsByWard \n[10:42:31] Server error '500 Internal Server Error' for url handlers.py:36\n 'https://www.fenland.gov.uk/localgov//mgWebService.asm \n x/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/500 \n Finished attempting to scrape: FEN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '500 Internal Server Error' for url 'https://www.fenland.gov.uk/localgov//mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/500\n","start":"2024-11-07 10:42:26.171664","end":"2024-11-07 10:42:31.887434","duration":5}},{"council_id":"FYL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:50:05] Fetching Scraper for: FYL handlers.py:23\n Begin attempting to scrape: FYL handlers.py:27\n[10:50:06] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 30 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[10:50:07] ...found 30 files in Councillors/raw base.py:225\n ...found 61 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 61 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://fylde.cmis.uk.com/fylde/CouncillorsandMP.aspx \n[10:50:09] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[10:50:12] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[10:50:14] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:17] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:19] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[10:50:22] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[10:50:23] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:26] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:27] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:29] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:31] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[10:50:34] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[10:50:35] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:40] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:41] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[10:50:44] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[10:50:46] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:49] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:50] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:53] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:55] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:58] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[10:50:59] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:04] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:06] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:08] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:10] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:12] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:14] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:17] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:18] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:21] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:23] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:26] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:29] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/21/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:32] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/21/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:33] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/35/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:37] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/35/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:38] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/37/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:41] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/37/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:42] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/153/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:45] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/153/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:47] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/212/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:49] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/212/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:52] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/187/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:55] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/187/ScreenMode/Alphabetical/ \n Default.aspx \n[10:51:56] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/24/ScreenMode/Alphabetical/D \n efault.aspx \n[10:51:59] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/24/ScreenMode/Alphabetical/D \n efault.aspx \n[10:52:01] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/177/ScreenMode/Alphabetical/ \n Default.aspx \n[10:52:04] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/177/ScreenMode/Alphabetical/ \n Default.aspx \n[10:52:05] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/16/ScreenMode/Alphabetical/D \n efault.aspx \n[10:52:09] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/16/ScreenMode/Alphabetical/D \n efault.aspx \n[10:52:10] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/167/ScreenMode/Alphabetical/ \n Default.aspx \n[10:52:14] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/167/ScreenMode/Alphabetical/ \n Default.aspx \n[10:52:16] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/99/ScreenMode/Alphabetical/D \n efault.aspx \n[10:52:21] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/99/ScreenMode/Alphabetical/D \n efault.aspx \n[10:52:22] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/34/ScreenMode/Alphabetical/D \n efault.aspx \n[10:52:26] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/34/ScreenMode/Alphabetical/D \n efault.aspx \n[10:52:27] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/210/ScreenMode/Alphabetical/ \n Default.aspx \n[10:52:30] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/210/ScreenMode/Alphabetical/ \n Default.aspx \n[10:52:31] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/221/ScreenMode/Alphabetical/ \n Default.aspx \n[10:52:33] 'NoneType' object has no attribute 'parent' handlers.py:36\n Committing batch 1 consisting of 60 files base.py:297\n[10:52:35] Finished attempting to scrape: FYL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 300, in get_single_councillor\n party = self.get_party_name(list_page_html)\n File \"scrapers/FYL-fylde/councillors.py\", line 17, in get_party_name\n .parent.get_text(strip=True)\nAttributeError: 'NoneType' object has no attribute 'parent'\n","start":"2024-11-07 10:50:05.762732","end":"2024-11-07 10:52:35.363454","duration":149}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:40:53] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n Deleting existing data... base.py:257\n[08:40:54] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:40:55] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-07 08:40:53.371172","end":"2024-11-07 08:40:55.745606","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:18:45] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n[08:18:46] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:18:47] ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[08:19:02] [Errno 110] Connection timed out handlers.py:36\n[08:19:03] Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 08:18:45.399678","end":"2024-11-07 08:19:03.072429","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[10:55:41] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n Deleting existing data... base.py:257\n[10:55:42] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:55:43] ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[10:55:45] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-07 10:55:41.215740","end":"2024-11-07 10:55:45.727529","duration":4}},{"council_id":"GWN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:13:49] Fetching Scraper for: GWN handlers.py:23\n Begin attempting to scrape: GWN handlers.py:27\n Deleting existing data... base.py:257\n[09:13:50] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 69 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 69 files in Councillors/raw base.py:225\n ...found 139 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:13:51] Deleting batch no. 2 consisting of 39 files base.py:236\n[09:13:52] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.cyngor.gwynedd.gov.uk/mgWebService.asmx/ \n GetCouncillorsByWard \n[09:14:02] [Errno 104] Connection reset by peer handlers.py:36\n Finished attempting to scrape: GWN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 104] Connection reset by peer\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 104] Connection reset by peer\n","start":"2024-11-07 09:13:49.439850","end":"2024-11-07 09:14:02.808053","duration":13}},{"council_id":"HIN","missing":false,"latest_run":{"status_code":1,"log_text":"[10:10:48] Fetching Scraper for: HIN handlers.py:23\n Begin attempting to scrape: HIN handlers.py:27\n[10:10:49] Deleting existing data... base.py:257\n[10:10:50] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 34 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 34 files in Councillors/raw base.py:225\n ...found 69 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 69 files base.py:236\n[10:10:51] ...data deleted. base.py:264\n Scraping from base.py:49\n http://moderngov.hinckley-bosworth.gov.uk/mgWebService.asm \n x/GetCouncillorsByWard \n[10:11:21] The read operation timed out handlers.py:36\n[10:11:22] Finished attempting to scrape: HIN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-11-07 10:10:48.801076","end":"2024-11-07 10:11:22.080491","duration":33}},{"council_id":"IOS","missing":false,"latest_run":{"status_code":1,"log_text":"[10:27:19] Fetching Scraper for: IOS handlers.py:23\n Begin attempting to scrape: IOS handlers.py:27\n[10:27:20] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 16 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 16 files in Councillors/raw base.py:225\n ...found 33 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 33 files base.py:236\n[10:27:21] ...data deleted. base.py:264\n Scraping from base.py:49\n http://committees.scilly.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://committees.scilly.gov.uk/mgWebService.asmx/Ge \n tCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n[10:27:23] Finished attempting to scrape: IOS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://committees.scilly.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-07 10:27:19.578678","end":"2024-11-07 10:27:23.686005","duration":4}},{"council_id":"KTT","missing":false,"latest_run":{"status_code":1,"log_text":"[09:34:24] Fetching Scraper for: KTT handlers.py:23\n Begin attempting to scrape: KTT handlers.py:27\n[09:34:25] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:34:26] Getting all files in Councillors/json... base.py:209\n ...found 48 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 48 files in Councillors/raw base.py:225\n ...found 97 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 97 files base.py:236\n[09:34:27] ...data deleted. base.py:264\n Scraping from base.py:49\n https://moderngov.kingston.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n[09:34:30] [Errno 113] No route to host handlers.py:36\n Finished attempting to scrape: KTT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 113] No route to host\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 113] No route to host\n","start":"2024-11-07 09:34:24.979463","end":"2024-11-07 09:34:30.392325","duration":5}},{"council_id":"LAC","missing":false,"latest_run":{"status_code":1,"log_text":"[09:27:20] Fetching Scraper for: LAC handlers.py:23\n Begin attempting to scrape: LAC handlers.py:27\n Deleting existing data... base.py:257\n[09:27:21] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 61 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 61 files in Councillors/raw base.py:225\n ...found 123 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:27:22] Deleting batch no. 2 consisting of 23 files base.py:236\n[09:27:23] ...data deleted. base.py:264\n Scraping from base.py:49\n http://committeeadmin.lancaster.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard \n[09:27:25] Server error '502 Bad Gateway' for url handlers.py:36\n 'https://committeeadmin.lancaster.gov.uk/mgWebService. \n asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n[09:27:26] Finished attempting to scrape: LAC base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://committeeadmin.lancaster.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-07 09:27:20.594516","end":"2024-11-07 09:27:26.174986","duration":5}},{"council_id":"MAI","missing":false,"latest_run":{"status_code":1,"log_text":"[09:05:50] Fetching Scraper for: MAI handlers.py:23\n Begin attempting to scrape: MAI handlers.py:27\n Deleting existing data... base.py:257\n[09:05:51] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 49 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 49 files in Councillors/raw base.py:225\n ...found 99 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 99 files base.py:236\n[09:05:52] ...data deleted. base.py:264\n Scraping from base.py:49\n https://meetings.maidstone.gov.uk//mgWebService.asmx/GetCo \n uncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://meetings.maidstone.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: MAI base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://meetings.maidstone.gov.uk//mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-07 09:05:50.123918","end":"2024-11-07 09:05:52.847481","duration":2}},{"council_id":"MDE","missing":false,"latest_run":{"status_code":1,"log_text":"[10:11:26] Fetching Scraper for: MDE handlers.py:23\n Begin attempting to scrape: MDE handlers.py:27\n Deleting existing data... base.py:257\n[10:11:27] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 42 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[10:11:28] ...found 42 files in Councillors/raw base.py:225\n ...found 85 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 85 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.middevon.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n[10:11:31] [Errno 113] No route to host handlers.py:36\n Finished attempting to scrape: MDE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 113] No route to host\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 113] No route to host\n","start":"2024-11-07 10:11:26.512213","end":"2024-11-07 10:11:31.839384","duration":5}},{"council_id":"MEL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:20:38] Fetching Scraper for: MEL handlers.py:23\n Begin attempting to scrape: MEL handlers.py:27\n Deleting existing data... base.py:257\n[10:20:39] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 28 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 28 files in Councillors/raw base.py:225\n ...found 57 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 57 files base.py:236\n[10:20:40] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.melton.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n[10:21:10] The read operation timed out handlers.py:36\n[10:21:11] Finished attempting to scrape: MEL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-11-07 10:20:38.180508","end":"2024-11-07 10:21:11.121695","duration":32}},{"council_id":"NDE","missing":false,"latest_run":{"status_code":1,"log_text":"[10:26:33] Fetching Scraper for: NDE handlers.py:23\n Begin attempting to scrape: NDE handlers.py:27\n[10:26:34] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 42 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[10:26:35] ...found 42 files in Councillors/raw base.py:225\n ...found 85 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 85 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.northdevon.gov.uk//mgWebService.asmx/Get \n CouncillorsByWard \n[10:27:06] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: NDE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 10:26:33.646000","end":"2024-11-07 10:27:07.004330","duration":33}},{"council_id":"NED","missing":false,"latest_run":{"status_code":1,"log_text":"[10:21:55] Fetching Scraper for: NED handlers.py:23\n Begin attempting to scrape: NED handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:21:56] Getting all files in Councillors/json... base.py:209\n ...found 53 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 53 files in Councillors/raw base.py:225\n ...found 107 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:21:57] Deleting batch no. 2 consisting of 7 files base.py:236\n[10:21:58] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.ne-derbyshire.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard \n Server error '502 Bad Gateway' for url handlers.py:36\n 'https://democracy.ne-derbyshire.gov.uk/mgWebService.a \n smx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n Finished attempting to scrape: NED base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://democracy.ne-derbyshire.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-07 10:21:55.078035","end":"2024-11-07 10:21:58.522723","duration":3}},{"council_id":"NET","missing":false,"latest_run":{"status_code":1,"log_text":"[10:44:00] Fetching Scraper for: NET handlers.py:23\n Begin attempting to scrape: NET handlers.py:27\n[10:44:01] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[10:44:02] ...found 78 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 78 files in Councillors/raw base.py:225\n ...found 157 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:44:03] Deleting batch no. 2 consisting of 57 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.newcastle.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n[10:44:19] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: NET base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 10:44:00.860200","end":"2024-11-07 10:44:19.447759","duration":18}},{"council_id":"OXF","missing":false,"latest_run":{"status_code":1,"log_text":"[10:26:11] Fetching Scraper for: OXF handlers.py:23\n Begin attempting to scrape: OXF handlers.py:27\n[10:26:12] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:26:13] Getting all files in Councillors/json... base.py:209\n ...found 63 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 63 files in Councillors/raw base.py:225\n ...found 127 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:26:14] Deleting batch no. 2 consisting of 27 files base.py:236\n[10:26:15] ...data deleted. base.py:264\n Scraping from base.py:49\n https://mycouncil.oxfordshire.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard \n Server disconnected without sending a response. handlers.py:36\n Finished attempting to scrape: OXF base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 238, in _receive_event\n raise RemoteProtocolError(msg)\nhttpcore.RemoteProtocolError: Server disconnected without sending a response.\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.RemoteProtocolError: Server disconnected without sending a response.\n","start":"2024-11-07 10:26:11.916490","end":"2024-11-07 10:26:15.408929","duration":3}},{"council_id":"RIC","missing":false,"latest_run":{"status_code":1,"log_text":"[09:40:19] Fetching Scraper for: RIC handlers.py:23\n Begin attempting to scrape: RIC handlers.py:27\n[09:40:20] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 54 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[09:40:21] ...found 54 files in Councillors/raw base.py:225\n ...found 109 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n Deleting batch no. 2 consisting of 9 files base.py:236\n[09:40:22] ...data deleted. base.py:264\n Scraping from base.py:49\n https://cabnet.richmond.gov.uk/mgWebService.asmx/GetCounci \n llorsByWard \n Server error '502 Bad Gateway' for url handlers.py:36\n 'https://cabnet.richmond.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n[09:40:23] Finished attempting to scrape: RIC base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://cabnet.richmond.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-07 09:40:19.570054","end":"2024-11-07 09:40:23.055150","duration":3}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"SPE","missing":false,"latest_run":{"status_code":1,"log_text":"[10:24:48] Fetching Scraper for: SPE handlers.py:23\n Begin attempting to scrape: SPE handlers.py:27\n Deleting existing data... base.py:257\n[10:24:49] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 39 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 39 files in Councillors/raw base.py:225\n ...found 79 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 79 files base.py:236\n[10:24:50] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.spelthorne.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n[10:25:05] [Errno 110] Connection timed out handlers.py:36\n[10:25:06] Finished attempting to scrape: SPE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 10:24:48.143863","end":"2024-11-07 10:25:06.180424","duration":18}},{"council_id":"STG","missing":false,"latest_run":{"status_code":1,"log_text":"[09:51:16] Fetching Scraper for: STG handlers.py:23\n Begin attempting to scrape: STG handlers.py:27\n Deleting existing data... base.py:257\n[09:51:17] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 19 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 19 files in Councillors/raw base.py:225\n ...found 39 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 39 files base.py:236\n[09:51:18] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-elections/councillors/ \n[09:51:21] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/elaine-watterson/ \n[09:51:22] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gene-maxwell/ \n[09:51:23] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/martin-earl/ \n[09:51:25] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mcgarvey/ \n[09:51:26] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/paul-henke/ \n[09:51:27] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rosemary-fraser/ \n[09:51:28] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/alasdair-tollemache/ \n[09:51:30] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/david-wilson/ \n[09:51:31] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/robin-kleinman/ \n[09:51:32] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/thomas-heald/ \n[09:51:33] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/danny-gibson/ \n[09:51:35] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jim-thomson/ \n[09:51:36] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rachel-nunn/ \n[09:51:37] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/susan-mcgill/ \n[09:51:38] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jen-preston/ \n[09:51:40] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/neil-benny/ \n[09:51:41] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/scott-farmer/ \n[09:51:42] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/bryan-flannagan/ \n[09:51:44] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mclaughlan/ \n[09:51:46] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/vacant/ \n[09:51:47] 'NoneType' object is not subscriptable handlers.py:36\n Committing batch 1 consisting of 38 files base.py:297\n[09:51:49] Finished attempting to scrape: STG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/STG-stirling/councillors.py\", line 42, in get_single_councillor\n councillor.email = soup.select_one(\"a[href^=mailto]\")[\nTypeError: 'NoneType' object is not subscriptable\n","start":"2024-11-07 09:51:16.256906","end":"2024-11-07 09:51:49.270126","duration":33}},{"council_id":"STH","missing":false,"latest_run":{"status_code":1,"log_text":"[09:49:40] Fetching Scraper for: STH handlers.py:23\n Begin attempting to scrape: STH handlers.py:27\n Deleting existing data... base.py:257\n[09:49:41] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 51 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 51 files in Councillors/raw base.py:225\n ...found 103 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:49:42] Deleting batch no. 2 consisting of 3 files base.py:236\n[09:49:43] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.southampton.gov.uk/modernGov//mgWebService.asm \n x/GetCouncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://www.southampton.gov.uk/modernGov//mgWebServic \n e.asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: STH base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://www.southampton.gov.uk/modernGov//mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-07 09:49:40.108966","end":"2024-11-07 09:49:43.570572","duration":3}},{"council_id":"STV","missing":false,"latest_run":{"status_code":1,"log_text":"[10:38:29] Fetching Scraper for: STV handlers.py:23\n Begin attempting to scrape: STV handlers.py:27\n[10:38:30] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[10:38:31] ...found 39 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 39 files in Councillors/raw base.py:225\n ...found 79 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 79 files base.py:236\n[10:38:32] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.stevenage.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n[10:39:02] The read operation timed out handlers.py:36\n Finished attempting to scrape: STV base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-11-07 10:38:29.783858","end":"2024-11-07 10:39:02.601522","duration":32}},{"council_id":"SWL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:53:14] Fetching Scraper for: SWL handlers.py:23\n Begin attempting to scrape: SWL handlers.py:27\n Deleting existing data... base.py:257\n[08:53:15] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 45 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 45 files in Councillors/raw base.py:225\n ...found 91 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 91 files base.py:236\n[08:53:16] ...data deleted. base.py:264\n Scraping from base.py:49\n http://services.swale.gov.uk/mgWebService.asmx/GetCouncill \n orsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://ws.swale.gov.uk/meetings/mgWebService.asmx/Ge \n tCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: SWL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://ws.swale.gov.uk/meetings/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-07 08:53:14.036835","end":"2024-11-07 08:53:16.767758","duration":2}},{"council_id":"TEW","missing":false,"latest_run":{"status_code":1,"log_text":"[09:52:48] Fetching Scraper for: TEW handlers.py:23\n Begin attempting to scrape: TEW handlers.py:27\n[09:52:49] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:52:50] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://minutes.tewkesbury.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[09:53:20] timed out handlers.py:36\n[09:53:21] Finished attempting to scrape: TEW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: timed out\n","start":"2024-11-07 09:52:48.893042","end":"2024-11-07 09:53:21.311055","duration":32}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:35:16] Fetching Scraper for: THE handlers.py:23\n[09:35:17] Begin attempting to scrape: THE handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:35:18] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[09:35:21] 'NoneType' object has no attribute 'findNext' handlers.py:36\n Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-11-07 09:35:16.996497","end":"2024-11-07 09:35:21.663484","duration":4}},{"council_id":"TOB","missing":false,"latest_run":{"status_code":1,"log_text":"[08:34:05] Fetching Scraper for: TOB handlers.py:23\n Begin attempting to scrape: TOB handlers.py:27\n Deleting existing data... base.py:257\n[08:34:06] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 36 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 36 files in Councillors/raw base.py:225\n ...found 73 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 73 files base.py:236\n[08:34:07] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.torbay.gov.uk/DemocraticServices//mgWebService \n .asmx/GetCouncillorsByWard \n[08:34:28] Server error '503 Service Unavailable' for url handlers.py:36\n 'https://www.torbay.gov.uk/DemocraticServices//mgWebSe \n rvice.asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: TOB base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://www.torbay.gov.uk/DemocraticServices//mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-07 08:34:05.194198","end":"2024-11-07 08:34:28.906810","duration":23}},{"council_id":"TRF","missing":false,"latest_run":{"status_code":1,"log_text":"[10:16:03] Fetching Scraper for: TRF handlers.py:23\n Begin attempting to scrape: TRF handlers.py:27\n[10:16:04] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:16:05] Getting all files in Councillors/json... base.py:209\n ...found 63 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 63 files in Councillors/raw base.py:225\n ...found 127 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:16:06] Deleting batch no. 2 consisting of 27 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democratic.trafford.gov.uk/mgWebService.asmx/GetCo \n uncillorsByWard \n[10:16:09] Server disconnected without sending a response. handlers.py:36\n Finished attempting to scrape: TRF base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 238, in _receive_event\n raise RemoteProtocolError(msg)\nhttpcore.RemoteProtocolError: Server disconnected without sending a response.\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.RemoteProtocolError: Server disconnected without sending a response.\n","start":"2024-11-07 10:16:03.882181","end":"2024-11-07 10:16:09.598243","duration":5}},{"council_id":"TUN","missing":false,"latest_run":{"status_code":1,"log_text":"[10:58:00] Fetching Scraper for: TUN handlers.py:23\n Begin attempting to scrape: TUN handlers.py:27\n[10:58:01] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 39 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 39 files in Councillors/raw base.py:225\n ...found 79 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 79 files base.py:236\n[10:58:02] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.tunbridgewells.gov.uk/mgWebService.asmx/ \n GetCouncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://democracy.tunbridgewells.gov.uk/mgWebService. \n asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n[10:58:03] Finished attempting to scrape: TUN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://democracy.tunbridgewells.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-07 10:58:00.553873","end":"2024-11-07 10:58:03.141078","duration":2}},{"council_id":"VGL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:47:55] Fetching Scraper for: VGL handlers.py:23\n Begin attempting to scrape: VGL handlers.py:27\n Deleting existing data... base.py:257\n[09:47:56] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 16 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 16 files in Councillors/raw base.py:225\n ...found 33 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 33 files base.py:236\n[09:47:57] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Councillors.aspx \n[09:47:59] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Asbrey-Anne.aspx \n[09:48:01] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Aviet-Julie.aspx \n[09:48:02] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Ball-Gareth.aspx \n[09:48:04] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Birch-Rhiannon.aspx \n[09:48:05] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Brooks-Bronwen.aspx \n[09:48:07] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Bruce-Gillian.aspx \n[09:48:09] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Buckley-Ian.aspx \n[09:48:10] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Burnett-Lis.aspx \n[09:48:12] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Campbell-Samantha.aspx \n[09:48:13] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Carroll-George.aspx \n[09:48:15] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cave-Christine.aspx \n[09:48:16] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Champion-Charles.aspx \n[09:48:19] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Charles-Janice.aspx \n[09:48:21] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Collins-Amelia.aspx \n[09:48:22] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cowpe-Marianne.aspx \n[09:48:24] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Drake-Pamela.aspx \n[09:48:26] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Driscoll-Vincent.aspx \n[09:48:28] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 32 files base.py:297\n[09:48:29] Finished attempting to scrape: VGL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/VGL-the-vale-of-glamorgan/councillors.py\", line 36, in get_single_councillor\n email_text = soup.select_one(\"#S4_EmailPlaceholder\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-11-07 09:47:55.204038","end":"2024-11-07 09:48:29.489834","duration":34}},{"council_id":"WAE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:59:51] Fetching Scraper for: WAE handlers.py:23\n Begin attempting to scrape: WAE handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:59:52] ...found 50 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 50 files in Councillors/raw base.py:225\n ...found 101 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:59:53] Deleting batch no. 2 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgov.waverley.gov.uk/mgWebService.asmx/GetCounci \n llorsByWard \n[09:00:13] Server disconnected without sending a response. handlers.py:36\n[09:00:14] Finished attempting to scrape: WAE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 238, in _receive_event\n raise RemoteProtocolError(msg)\nhttpcore.RemoteProtocolError: Server disconnected without sending a response.\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.RemoteProtocolError: Server disconnected without sending a response.\n","start":"2024-11-07 08:59:51.029835","end":"2024-11-07 09:00:14.256420","duration":23}},{"council_id":"WEA","missing":false,"latest_run":{"status_code":1,"log_text":"[10:37:06] Fetching Scraper for: WEA handlers.py:23\n Begin attempting to scrape: WEA handlers.py:27\n Deleting existing data... base.py:257\n[10:37:07] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 45 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 45 files in Councillors/raw base.py:225\n ...found 91 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 91 files base.py:236\n[10:37:08] ...data deleted. base.py:264\n Scraping from base.py:49\n http://council.wealden.gov.uk/mgWebService.asmx/GetCouncil \n lorsByWard \n[10:37:24] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: WEA base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 10:37:06.111113","end":"2024-11-07 10:37:24.498936","duration":18}},{"council_id":"WEW","missing":false,"latest_run":{"status_code":1,"log_text":"[10:12:17] Fetching Scraper for: WEW handlers.py:23\n Begin attempting to scrape: WEW handlers.py:27\n[10:12:18] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:12:19] Getting all files in Councillors/json... base.py:209\n ...found 48 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 48 files in Councillors/raw base.py:225\n ...found 97 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 97 files base.py:236\n[10:12:20] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.welhat.gov.uk/mgWebService.asmx/GetCounci \n llorsByWard \n[10:12:35] [Errno 110] Connection timed out handlers.py:36\n[10:12:36] Finished attempting to scrape: WEW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-07 10:12:17.919958","end":"2024-11-07 10:12:36.073433","duration":18}},{"council_id":"WLN","missing":false,"latest_run":{"status_code":1,"log_text":"[10:54:56] Fetching Scraper for: WLN handlers.py:23\n Begin attempting to scrape: WLN handlers.py:27\n Deleting existing data... base.py:257\n[10:54:57] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 25 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 25 files in Councillors/raw base.py:225\n ...found 51 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 51 files base.py:236\n[10:54:58] ...data deleted. base.py:264\n Scraping from https://westlothian.gov.uk/councillors base.py:49\n[10:55:01] Scraping from base.py:49\n https://westlothian.gov.uk/article/33888/Linlithgow \n[10:55:02] Scraping from base.py:49\n https://westlothian.gov.uk/article/33889/Broxburn-Uphall-a \n nd-Winchburgh \n[10:55:03] Scraping from base.py:49\n https://westlothian.gov.uk/article/33890/Livingston-North \n[10:55:04] Scraping from base.py:49\n https://westlothian.gov.uk/article/33891/Livingston-South \n[10:55:05] Scraping from base.py:49\n https://westlothian.gov.uk/article/33893/East-Livingston-a \n nd-East-Calder \n[10:55:06] Scraping from base.py:49\n https://westlothian.gov.uk/article/33892/Fauldhouse-and-th \n e-Breich-Valley \n[10:55:07] Scraping from base.py:49\n https://westlothian.gov.uk/article/33894/Whitburn-and-Blac \n kburn \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33895/Bathgate \n[10:55:08] Scraping from base.py:49\n https://westlothian.gov.uk/article/33896/Armadale-and-Blac \n kridge \n[10:55:09] Scraping from base.py:49\n https://westlothian.gov.uk/article/33897/Councillor-Tom-Co \n nn \n[10:55:12] Scraping from base.py:49\n https://westlothian.gov.uk/article/33898/Councillor-Paulin \n e-Orr \n[10:55:13] Scraping from base.py:49\n https://westlothian.gov.uk/article/33899/Councillor-Sally- \n Pattle \n[10:55:14] Scraping from base.py:49\n https://westlothian.gov.uk/article/34305/Councillor-Tony-B \n oyle \n[10:55:15] Scraping from base.py:49\n https://westlothian.gov.uk/article/34302/Councillor-Diane- \n Calder \n[10:55:16] Scraping from base.py:49\n https://westlothian.gov.uk/article/34303/Councillor-Janet- \n Campbell \n[10:55:17] Scraping from base.py:49\n https://westlothian.gov.uk/article/34304/Councillor-Angela \n -Doran-Timson \n[10:55:18] Scraping from base.py:49\n https://westlothian.gov.uk/article/34307/Councillor-Alison \n -Adamson \n[10:55:19] Scraping from base.py:49\n https://westlothian.gov.uk/article/34306/Councillor-Robert \n -De-Bold \n[10:55:20] Scraping from base.py:49\n https://westlothian.gov.uk/article/34308/Councillor-Anne-M \n cMillan \n[10:55:21] Scraping from base.py:49\n https://westlothian.gov.uk/article/34309/Councillor-Andrew \n -Miller \n[10:55:22] Scraping from base.py:49\n https://westlothian.gov.uk/article/34311/Councillor-Lawren \n ce-Fitzpatrick \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34310/Councillor-Peter- \n Heggie \n[10:55:23] Scraping from base.py:49\n https://westlothian.gov.uk/article/34312/Councillor-Maria- \n MacAulay \n[10:55:24] Scraping from base.py:49\n https://westlothian.gov.uk/article/34313/Councillor-Moira- \n McKee-Shemilt \n[10:55:25] Scraping from base.py:49\n https://westlothian.gov.uk/article/34317/Councillor-Damian \n -Doran-Timson \n[10:55:26] Scraping from base.py:49\n https://westlothian.gov.uk/article/34315/Councillor-Carl-J \n ohn \n[10:55:27] Scraping from base.py:49\n https://westlothian.gov.uk/article/34316/Councillor-Danny- \n Logue \n[10:55:28] Scraping from base.py:49\n https://westlothian.gov.uk/article/34314/Councillor-Veroni \n ca-Smith \n[10:55:29] Scraping from base.py:49\n https://westlothian.gov.uk/article/34318/Councillor-Paulin \n e-Clark \n[10:55:30] Scraping from base.py:49\n https://westlothian.gov.uk/article/34319/Councillor-Craig- \n Meek \n[10:55:31] Scraping from base.py:49\n https://westlothian.gov.uk/article/34320/Councillor-Cathy- \n Muldoon \n[10:55:32] Scraping from base.py:49\n https://westlothian.gov.uk/article/34321/Councillor-Jim-Di \n ckson \n[10:55:33] Scraping from base.py:49\n https://westlothian.gov.uk/article/34322/Councillor-Mary-D \n ickson \n[10:55:34] Scraping from base.py:49\n https://westlothian.gov.uk/article/34323/Councillor-George \n -Paul \n[10:55:35] Scraping from base.py:49\n https://westlothian.gov.uk/article/34324/Councillor-Vacant \n[10:55:36] list index out of range handlers.py:36\n Committing batch 1 consisting of 50 files base.py:297\n[10:55:37] Finished attempting to scrape: WLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/WLN-west-lothian/councillors.py\", line 37, in get_single_councillor\n councillor.email = soup.select(\"a[href^=mailto]\")[0].get_text(\nIndexError: list index out of range\n","start":"2024-11-07 10:54:56.335731","end":"2024-11-07 10:55:37.509856","duration":41}}] +[{"council_id":"AGY","missing":false,"latest_run":{"status_code":1,"log_text":"[08:27:57] Fetching Scraper for: AGY handlers.py:23\n Begin attempting to scrape: AGY handlers.py:27\n[08:27:58] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 35 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 35 files in Councillors/raw base.py:225\n ...found 71 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 71 files base.py:236\n[08:27:59] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.anglesey.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[08:28:14] [Errno 110] Connection timed out handlers.py:36\n[08:28:15] Finished attempting to scrape: AGY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 08:27:57.635235","end":"2024-11-08 08:28:15.346346","duration":17}},{"council_id":"BAN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:53:05] Fetching Scraper for: BAN handlers.py:23\n Begin attempting to scrape: BAN handlers.py:27\n Deleting existing data... base.py:257\n[08:53:06] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:53:07] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.basingstoke.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard \n[08:53:22] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 08:53:05.513606","end":"2024-11-08 08:53:22.900182","duration":17}},{"council_id":"BDF","missing":false,"latest_run":{"status_code":1,"log_text":"[09:01:24] Fetching Scraper for: BDF handlers.py:23\n Begin attempting to scrape: BDF handlers.py:27\n[09:01:25] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:01:26] ...data deleted. base.py:264\n Scraping from base.py:49\n https://councillorsupport.bedford.gov.uk/mgWebService.asmx \n /GetCouncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://councillorsupport.bedford.gov.uk/mgWebService \n .asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: BDF base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://councillorsupport.bedford.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 09:01:24.696329","end":"2024-11-08 09:01:26.935107","duration":2}},{"council_id":"BDG","missing":false,"latest_run":{"status_code":1,"log_text":"[08:51:10] Fetching Scraper for: BDG handlers.py:23\n Begin attempting to scrape: BDG handlers.py:27\n Deleting existing data... base.py:257\n[08:51:11] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n[08:51:12] Scraping from base.py:49\n https://modgov.lbbd.gov.uk/internet/mgWebService.asmx/GetC \n ouncillorsByWard \n[08:51:27] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: BDG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 08:51:10.228059","end":"2024-11-08 08:51:27.739228","duration":17}},{"council_id":"BEN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:50:00] Fetching Scraper for: BEN handlers.py:23\n Begin attempting to scrape: BEN handlers.py:27\n Deleting existing data... base.py:257\n[08:50:01] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:50:02] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.brent.gov.uk/mgWebService.asmx/GetCouncil \n lorsByWard \n[08:50:17] [Errno 110] Connection timed out handlers.py:36\n[08:50:18] Finished attempting to scrape: BEN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 08:50:00.395849","end":"2024-11-08 08:50:18.089329","duration":17}},{"council_id":"BFS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:18:35] Fetching Scraper for: BFS handlers.py:23\n Begin attempting to scrape: BFS handlers.py:27\n Deleting existing data... base.py:257\n[09:18:36] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 60 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 60 files in Councillors/raw base.py:225\n ...found 121 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:18:37] Deleting batch no. 2 consisting of 21 files base.py:236\n[09:18:38] ...data deleted. base.py:264\n Scraping from base.py:49\n https://minutes3.belfastcity.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n Client error '404 Not Found' for url handlers.py:36\n 'https://minutes3.belfastcity.gov.uk/mgWebService.asmx \n /GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: BFS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://minutes3.belfastcity.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-08 09:18:35.258452","end":"2024-11-08 09:18:38.954594","duration":3}},{"council_id":"BOL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:14:02] Fetching Scraper for: BOL handlers.py:23\n Begin attempting to scrape: BOL handlers.py:27\n[09:14:03] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:14:04] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.democracy.bolton.gov.uk/cmis5/People.aspx \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: BOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-11-08 09:14:02.647951","end":"2024-11-08 09:14:04.776463","duration":2}},{"council_id":"BOT","missing":false,"latest_run":{"status_code":1,"log_text":"[08:26:19] Fetching Scraper for: BOT handlers.py:23\n Begin attempting to scrape: BOT handlers.py:27\n[08:26:20] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 30 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[08:26:21] ...found 30 files in Councillors/raw base.py:225\n ...found 61 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 61 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.boston.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Client error '404 Not Found' for url handlers.py:36\n 'https://democracy.boston.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n[08:26:22] Finished attempting to scrape: BOT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://democracy.boston.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-08 08:26:19.748075","end":"2024-11-08 08:26:22.157257","duration":2}},{"council_id":"BRT","missing":false,"latest_run":{"status_code":1,"log_text":"[09:11:45] Fetching Scraper for: BRT handlers.py:23\n Begin attempting to scrape: BRT handlers.py:27\n Deleting existing data... base.py:257\n[09:11:46] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 44 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 44 files in Councillors/raw base.py:225\n ...found 89 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 89 files base.py:236\n[09:11:47] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.broxtowe.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[09:12:02] [Errno 110] Connection timed out handlers.py:36\n[09:12:03] Finished attempting to scrape: BRT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 09:11:45.307021","end":"2024-11-08 09:12:03.157267","duration":17}},{"council_id":"CAN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:26:43] Fetching Scraper for: CAN handlers.py:23\n[08:26:44] Begin attempting to scrape: CAN handlers.py:27\n Deleting existing data... base.py:257\n[08:26:45] Getting all files in Councillors... base.py:209\n[08:26:46] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.cannockchasedc.gov.uk/council/about-council/yo \n ur-councillors \n[08:26:49] list index out of range handlers.py:36\n Finished attempting to scrape: CAN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 158, in get_list_container\n return selected[0]\nIndexError: list index out of range\n","start":"2024-11-08 08:26:43.980536","end":"2024-11-08 08:26:49.306912","duration":5}},{"council_id":"CAS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:53:47] Fetching Scraper for: CAS handlers.py:23\n Begin attempting to scrape: CAS handlers.py:27\n Deleting existing data... base.py:257\n[08:53:48] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from https://www.castlepoint.gov.uk/councillors base.py:49\n[08:53:50] Scraping from base.py:49\n https://www.castlepoint.gov.ukhttps://castlepoint.cmis.uk. \n com/castlepoint/Committees/CurrentCommittees.aspx?a=1 \n [Errno -2] Name or service not known handlers.py:36\n Finished attempting to scrape: CAS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/CAS-castle-point/councillors.py\", line 15, in get_single_councillor\n soup = self.get_page(url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno -2] Name or service not known\n","start":"2024-11-08 08:53:47.086988","end":"2024-11-08 08:53:50.511631","duration":3}},{"council_id":"CAT","missing":false,"latest_run":{"status_code":1,"log_text":"[08:52:57] Fetching Scraper for: CAT handlers.py:23\n Begin attempting to scrape: CAT handlers.py:27\n Deleting existing data... base.py:257\n[08:52:58] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:52:59] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.canterbury.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n [SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handlers.py:36\n handshake failure (_ssl.c:1007) \n Finished attempting to scrape: CAT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:1007)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:1007)\n","start":"2024-11-08 08:52:57.284791","end":"2024-11-08 08:52:59.348770","duration":2}},{"council_id":"CAY","missing":false,"latest_run":{"status_code":1,"log_text":"[08:46:54] Fetching Scraper for: CAY handlers.py:23\n Begin attempting to scrape: CAY handlers.py:27\n[08:46:55] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:46:56] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.caerphilly.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n[08:46:59] [Errno 113] No route to host handlers.py:36\n[08:47:00] Finished attempting to scrape: CAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 113] No route to host\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 113] No route to host\n","start":"2024-11-08 08:46:54.912517","end":"2024-11-08 08:47:00.135448","duration":5}},{"council_id":"CHT","missing":false,"latest_run":{"status_code":1,"log_text":"[08:18:48] Fetching Scraper for: CHT handlers.py:23\n Begin attempting to scrape: CHT handlers.py:27\n Deleting existing data... base.py:257\n[08:18:49] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 40 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 40 files in Councillors/raw base.py:225\n ...found 81 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 81 files base.py:236\n[08:18:50] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.cheltenham.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n[08:19:20] The read operation timed out handlers.py:36\n Finished attempting to scrape: CHT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-11-08 08:18:48.194074","end":"2024-11-08 08:19:20.936608","duration":32}},{"council_id":"CHW","missing":false,"latest_run":{"status_code":1,"log_text":"[08:32:47] Fetching Scraper for: CHW handlers.py:23\n Begin attempting to scrape: CHW handlers.py:27\n[08:32:48] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:32:49] ...data deleted. base.py:264\n Scraping from base.py:49\n http://cmttpublic.cheshirewestandchester.gov.uk/mgWebServi \n ce.asmx/GetCouncillorsByWard \n[08:33:05] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: CHW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 08:32:47.886164","end":"2024-11-08 08:33:05.357550","duration":17}},{"council_id":"CON","missing":false,"latest_run":{"status_code":1,"log_text":"[10:03:29] Fetching Scraper for: CON handlers.py:23\n Begin attempting to scrape: CON handlers.py:27\n[10:03:30] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[10:03:31] ...found 87 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 87 files in Councillors/raw base.py:225\n ...found 175 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[10:03:32] Deleting batch no. 2 consisting of 75 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.cornwall.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n Client error '404 Not Found' for url handlers.py:36\n 'https://democracy.cornwall.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n[10:03:33] Finished attempting to scrape: CON base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://democracy.cornwall.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-08 10:03:29.855579","end":"2024-11-08 10:03:33.256301","duration":3}},{"council_id":"CWY","missing":false,"latest_run":{"status_code":1,"log_text":"[10:33:34] Fetching Scraper for: CWY handlers.py:23\n Begin attempting to scrape: CWY handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:33:35] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgovpub-eng.secure.conwy.gov.uk/mgWebService.asm \n x/GetCouncillorsByWard \n[10:33:36] Client error '404 Not Found' for url handlers.py:36\n 'https://modgovpub-eng.secure.conwy.gov.uk/mgWebServic \n e.asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: CWY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://modgovpub-eng.secure.conwy.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-08 10:33:34.043235","end":"2024-11-08 10:33:36.270576","duration":2}},{"council_id":"DAL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:47:03] Fetching Scraper for: DAL handlers.py:23\n Begin attempting to scrape: DAL handlers.py:27\n[08:47:04] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 50 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[08:47:05] ...found 50 files in Councillors/raw base.py:225\n ...found 101 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:47:06] Deleting batch no. 2 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.darlington.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n Server error '520 ' for url handlers.py:36\n 'https://democracy.darlington.gov.uk/mgWebService.asmx \n /GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/520 \n[08:47:07] Finished attempting to scrape: DAL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '520 ' for url 'https://democracy.darlington.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/520\n","start":"2024-11-08 08:47:03.653380","end":"2024-11-08 08:47:07.177874","duration":3}},{"council_id":"DEN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:55:54] Fetching Scraper for: DEN handlers.py:23\n Begin attempting to scrape: DEN handlers.py:27\n Deleting existing data... base.py:257\n[08:55:55] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 48 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 48 files in Councillors/raw base.py:225\n ...found 97 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 97 files base.py:236\n[08:55:56] ...data deleted. base.py:264\n Scraping from base.py:49\n https://moderngov.denbighshire.gov.uk/mgWebService.asmx/Ge \n tCouncillorsByWard \n Client error '404 Not Found' for url handlers.py:36\n 'https://moderngov.denbighshire.gov.uk/mgWebService.as \n mx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: DEN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://moderngov.denbighshire.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-08 08:55:54.279141","end":"2024-11-08 08:55:56.694430","duration":2}},{"council_id":"DRS","missing":false,"latest_run":{"status_code":1,"log_text":"[08:33:08] Fetching Scraper for: DRS handlers.py:23\n Begin attempting to scrape: DRS handlers.py:27\n[08:33:09] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:33:10] ...found 40 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 40 files in Councillors/raw base.py:225\n ...found 81 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 81 files base.py:236\n[08:33:11] ...data deleted. base.py:264\n Scraping from base.py:49\n https://meetings.derrycityandstrabanedistrict.com/mgWebSer \n vice.asmx/GetCouncillorsByWard \n[08:33:26] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: DRS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 08:33:08.756438","end":"2024-11-08 08:33:26.888147","duration":18}},{"council_id":"DUR","missing":false,"latest_run":{"status_code":1,"log_text":"[09:11:20] Fetching Scraper for: DUR handlers.py:23\n Begin attempting to scrape: DUR handlers.py:27\n[09:11:21] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:11:22] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.durham.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n[09:11:38] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: DUR base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 09:11:20.950756","end":"2024-11-08 09:11:38.545682","duration":17}},{"council_id":"EAY","missing":false,"latest_run":{"status_code":1,"log_text":"[09:34:38] Fetching Scraper for: EAY handlers.py:23\n Begin attempting to scrape: EAY handlers.py:27\n[09:34:39] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:34:40] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx \n[09:34:41] Scraping from base.py:49\n https://www.east-ayrshire.gov.uk/CouncilAndGovernment/Abou \n t-the-Council/Councillors-and-Provost/YourCouncillor.aspx? \n 9 \n[09:34:42] 'NoneType' object has no attribute 'find_parent' handlers.py:36\n Finished attempting to scrape: EAY base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/EAY-east-ayrshire/councillors.py\", line 22, in get_single_councillor\n .find_parent(\"div\")\nAttributeError: 'NoneType' object has no attribute 'find_parent'\n","start":"2024-11-08 09:34:38.667574","end":"2024-11-08 09:34:42.553445","duration":3}},{"council_id":"EDE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:46:29] Fetching Scraper for: EDE handlers.py:23\n Begin attempting to scrape: EDE handlers.py:27\n[08:46:30] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:46:31] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.eastdevon.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n[08:46:34] [Errno 113] No route to host handlers.py:36\n Finished attempting to scrape: EDE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 113] No route to host\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 113] No route to host\n","start":"2024-11-08 08:46:29.622944","end":"2024-11-08 08:46:34.869257","duration":5}},{"council_id":"EDU","missing":false,"latest_run":{"status_code":1,"log_text":"[10:32:32] Fetching Scraper for: EDU handlers.py:23\n Begin attempting to scrape: EDU handlers.py:27\n[10:32:33] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[10:32:34] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.eastdunbarton.gov.uk/residents/council-democra \n cy/committees-and-councillors/councillors-2017 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.eastdunbarton.gov.uk/residents/council-de \n mocracy/committees-and-councillors/councillors-2017' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n[10:32:35] Finished attempting to scrape: EDU base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.eastdunbarton.gov.uk/residents/council-democracy/committees-and-councillors/councillors-2017'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-08 10:32:32.701572","end":"2024-11-08 10:32:35.104305","duration":2}},{"council_id":"EHE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:23:30] Fetching Scraper for: EHE handlers.py:23\n Begin attempting to scrape: EHE handlers.py:27\n[09:23:31] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:23:32] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.eastherts.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n[09:23:48] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: EHE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 09:23:30.714459","end":"2024-11-08 09:23:48.444958","duration":17}},{"council_id":"ELI","missing":false,"latest_run":{"status_code":1,"log_text":"[09:37:31] Fetching Scraper for: ELI handlers.py:23\n Begin attempting to scrape: ELI handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:37:32] Getting all files in Councillors/json... base.py:209\n ...found 55 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 55 files in Councillors/raw base.py:225\n ...found 111 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:37:33] Deleting batch no. 2 consisting of 11 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.e-lindsey.gov.uk/mgWebService.asmx/GetCo \n uncillorsByWard \n[09:37:34] Client error '404 Not Found' for url handlers.py:36\n 'https://democracy.e-lindsey.gov.uk/mgWebService.asmx/ \n GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: ELI base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://democracy.e-lindsey.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-08 09:37:31.099582","end":"2024-11-08 09:37:34.252953","duration":3}},{"council_id":"EXE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:27:35] Fetching Scraper for: EXE handlers.py:23\n Begin attempting to scrape: EXE handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:27:36] Getting all files in Councillors/json... base.py:209\n ...found 39 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 39 files in Councillors/raw base.py:225\n ...found 79 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 79 files base.py:236\n[09:27:37] ...data deleted. base.py:264\n Scraping from base.py:49\n http://committees.exeter.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n[09:27:38] [Errno 113] No route to host handlers.py:36\n Finished attempting to scrape: EXE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 113] No route to host\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 113] No route to host\n","start":"2024-11-08 09:27:35.003300","end":"2024-11-08 09:27:38.173611","duration":3}},{"council_id":"FEN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:58:24] Fetching Scraper for: FEN handlers.py:23\n Begin attempting to scrape: FEN handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:58:25] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.fenland.gov.uk/localgov//mgWebService.asmx/Get \n CouncillorsByWard \n[08:58:27] Server error '502 Bad Gateway' for url handlers.py:36\n 'https://www.fenland.gov.uk/localgov//mgWebService.asm \n x/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n[08:58:28] Finished attempting to scrape: FEN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://www.fenland.gov.uk/localgov//mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-08 08:58:24.030575","end":"2024-11-08 08:58:28.128638","duration":4}},{"council_id":"FLN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:21:47] Fetching Scraper for: FLN handlers.py:23\n Begin attempting to scrape: FLN handlers.py:27\n Deleting existing data... base.py:257\n[08:21:48] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 67 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 67 files in Councillors/raw base.py:225\n ...found 135 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:21:49] Deleting batch no. 2 consisting of 35 files base.py:236\n[08:21:50] ...data deleted. base.py:264\n Scraping from base.py:49\n https://cyfarfodyddpwyllgor.siryfflint.gov.uk/mgWebService \n .asmx/GetCouncillorsByWard \n[08:22:06] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: FLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 08:21:47.323748","end":"2024-11-08 08:22:06.645901","duration":19}},{"council_id":"FYL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:40:27] Fetching Scraper for: FYL handlers.py:23\n Begin attempting to scrape: FYL handlers.py:27\n Deleting existing data... base.py:257\n[08:40:28] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 30 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 30 files in Councillors/raw base.py:225\n ...found 61 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 61 files base.py:236\n[08:40:29] ...data deleted. base.py:264\n Scraping from base.py:49\n https://fylde.cmis.uk.com/fylde/CouncillorsandMP.aspx \n[08:40:30] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[08:40:34] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/48/ScreenMode/Alphabetical/D \n efault.aspx \n[08:40:35] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[08:40:38] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/148/ScreenMode/Alphabetical/ \n Default.aspx \n[08:40:39] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[08:40:42] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/84/ScreenMode/Alphabetical/D \n efault.aspx \n[08:40:45] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[08:40:47] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/211/ScreenMode/Alphabetical/ \n Default.aspx \n[08:40:49] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[08:40:51] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/207/ScreenMode/Alphabetical/ \n Default.aspx \n[08:40:52] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[08:40:56] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/52/ScreenMode/Alphabetical/D \n efault.aspx \n[08:40:57] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:00] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/208/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:01] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:04] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/43/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:05] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:10] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/172/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:11] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:14] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/213/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:15] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:18] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/209/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:20] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:23] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/15/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:24] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:27] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/206/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:28] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:30] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/174/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:33] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:36] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/14/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:37] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:41] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/186/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:42] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:45] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/176/ScreenMode/Alphabetical/ \n Default.aspx \n[08:41:46] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/21/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:49] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/21/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:50] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/35/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:53] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/35/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:56] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/37/ScreenMode/Alphabetical/D \n efault.aspx \n[08:41:59] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/37/ScreenMode/Alphabetical/D \n efault.aspx \n[08:42:00] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/153/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:03] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/153/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:05] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/212/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:07] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/212/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:08] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/187/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:11] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/187/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:12] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/24/ScreenMode/Alphabetical/D \n efault.aspx \n[08:42:15] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/24/ScreenMode/Alphabetical/D \n efault.aspx \n[08:42:17] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/177/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:21] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/177/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:22] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/16/ScreenMode/Alphabetical/D \n efault.aspx \n[08:42:25] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/16/ScreenMode/Alphabetical/D \n efault.aspx \n[08:42:26] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/167/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:29] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/167/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:31] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/99/ScreenMode/Alphabetical/D \n efault.aspx \n[08:42:34] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/99/ScreenMode/Alphabetical/D \n efault.aspx \n[08:42:35] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/34/ScreenMode/Alphabetical/D \n efault.aspx \n[08:42:38] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/34/ScreenMode/Alphabetical/D \n efault.aspx \n[08:42:40] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/210/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:44] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/210/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:45] Scraping from base.py:49\n http://fylde.cmis.uk.com/fylde/CouncillorsandMP/tabid/63/c \n tl/ViewCMIS_Person/mid/383/id/221/ScreenMode/Alphabetical/ \n Default.aspx \n[08:42:47] 'NoneType' object has no attribute 'parent' handlers.py:36\n Committing batch 1 consisting of 60 files base.py:297\n[08:42:49] Finished attempting to scrape: FYL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 300, in get_single_councillor\n party = self.get_party_name(list_page_html)\n File \"scrapers/FYL-fylde/councillors.py\", line 17, in get_party_name\n .parent.get_text(strip=True)\nAttributeError: 'NoneType' object has no attribute 'parent'\n","start":"2024-11-08 08:40:27.332404","end":"2024-11-08 08:42:49.048010","duration":141}},{"council_id":"GED","missing":false,"latest_run":{"status_code":1,"log_text":"[08:50:52] Fetching Scraper for: GED handlers.py:23\n Begin attempting to scrape: GED handlers.py:27\n Deleting existing data... base.py:257\n[08:50:53] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 41 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 41 files in Councillors/raw base.py:225\n ...found 83 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 83 files base.py:236\n[08:50:54] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.gedling.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[08:50:55] Server error '503 Service Unavailable' for url handlers.py:36\n 'https://democracy.gedling.gov.uk/mgWebService.asmx/Ge \n tCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: GED base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://democracy.gedling.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 08:50:52.431285","end":"2024-11-08 08:50:55.728996","duration":3}},{"council_id":"GLG","missing":false,"latest_run":{"status_code":1,"log_text":"[09:34:32] Fetching Scraper for: GLG handlers.py:23\n Begin attempting to scrape: GLG handlers.py:27\n Deleting existing data... base.py:257\n[09:34:33] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:34:34] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.glasgow.gov.uk/councillorsandcommittees/allMem \n bers.asp?sort=0&page=0&rec=100 \n Client error '404 Not Found' for url handlers.py:36\n 'https://www.glasgow.gov.uk/councillorsandcommittees/a \n llMembers.asp?sort=0&page=0&rec=100' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GLG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 152, in get_list_container\n self.base_url_soup = self.get_page(self.base_url)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 141, in get_page\n page = self.get(url, extra_headers=self.extra_headers).text\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://www.glasgow.gov.uk/councillorsandcommittees/allMembers.asp?sort=0&page=0&rec=100'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-08 09:34:32.451455","end":"2024-11-08 09:34:34.878129","duration":2}},{"council_id":"GLS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:19:20] Fetching Scraper for: GLS handlers.py:23\n Begin attempting to scrape: GLS handlers.py:27\n Deleting existing data... base.py:257\n[09:19:21] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:19:22] ...data deleted. base.py:264\n Scraping from base.py:49\n http://glostext.gloucestershire.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard \n[09:19:37] [Errno 110] Connection timed out handlers.py:36\n[09:19:38] Finished attempting to scrape: GLS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 09:19:20.440702","end":"2024-11-08 09:19:38.091767","duration":17}},{"council_id":"GRE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:57:27] Fetching Scraper for: GRE handlers.py:23\n Begin attempting to scrape: GRE handlers.py:27\n Deleting existing data... base.py:257\n[08:57:28] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://committees.royalgreenwich.gov.uk/Councillors/tabid \n /63/ScreenMode/Alphabetical/Default.aspx \n[08:57:34] Client error '404 Not Found' for url handlers.py:36\n 'https://committees.royalgreenwich.gov.uk/mgError.aspx \n ' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/404 \n Finished attempting to scrape: GRE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 272, in get_councillors\n req = self.get(\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Client error '404 Not Found' for url 'https://committees.royalgreenwich.gov.uk/mgError.aspx'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n","start":"2024-11-08 08:57:27.254121","end":"2024-11-08 08:57:34.184650","duration":6}},{"council_id":"GWN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:50:22] Fetching Scraper for: GWN handlers.py:23\n Begin attempting to scrape: GWN handlers.py:27\n[08:50:23] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:50:24] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.cyngor.gwynedd.gov.uk/mgWebService.asmx/ \n GetCouncillorsByWard \n[08:50:34] [Errno 104] Connection reset by peer handlers.py:36\n[08:50:35] Finished attempting to scrape: GWN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 154, in _connect\n stream = stream.start_tls(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 152, in start_tls\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 104] Connection reset by peer\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 104] Connection reset by peer\n","start":"2024-11-08 08:50:22.922044","end":"2024-11-08 08:50:35.288985","duration":12}},{"council_id":"HAV","missing":false,"latest_run":{"status_code":1,"log_text":"[08:27:16] Fetching Scraper for: HAV handlers.py:23\n Begin attempting to scrape: HAV handlers.py:27\n Deleting existing data... base.py:257\n[08:27:17] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 55 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 55 files in Councillors/raw base.py:225\n ...found 111 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:27:18] Deleting batch no. 2 consisting of 11 files base.py:236\n[08:27:19] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.havering.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n Server error '502 Bad Gateway' for url handlers.py:36\n 'https://democracy.havering.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n Finished attempting to scrape: HAV base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://democracy.havering.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-08 08:27:16.141067","end":"2024-11-08 08:27:19.406898","duration":3}},{"council_id":"HIG","missing":false,"latest_run":{"status_code":1,"log_text":"[09:52:45] Fetching Scraper for: HIG handlers.py:23\n Begin attempting to scrape: HIG handlers.py:27\n[09:52:46] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[09:52:47] ...found 99 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 99 files in Councillors/raw base.py:225\n ...found 199 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:52:48] Deleting batch no. 2 consisting of 99 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.highpeak.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n[09:53:04] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: HIG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 09:52:45.843079","end":"2024-11-08 09:53:04.635187","duration":18}},{"council_id":"IOS","missing":false,"latest_run":{"status_code":1,"log_text":"[09:53:36] Fetching Scraper for: IOS handlers.py:23\n Begin attempting to scrape: IOS handlers.py:27\n Deleting existing data... base.py:257\n[09:53:37] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:53:38] ...data deleted. base.py:264\n Scraping from base.py:49\n http://committees.scilly.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://committees.scilly.gov.uk/mgWebService.asmx/Ge \n tCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: IOS base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://committees.scilly.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 09:53:36.146826","end":"2024-11-08 09:53:38.395383","duration":2}},{"council_id":"KIN","missing":false,"latest_run":{"status_code":1,"log_text":"[08:19:55] Fetching Scraper for: KIN handlers.py:23\n Begin attempting to scrape: KIN handlers.py:27\n Deleting existing data... base.py:257\n[08:19:56] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 55 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 55 files in Councillors/raw base.py:225\n ...found 111 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:19:57] Deleting batch no. 2 consisting of 11 files base.py:236\n[08:19:58] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.west-norfolk.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard \n Server disconnected without sending a response. handlers.py:36\n Finished attempting to scrape: KIN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 238, in _receive_event\n raise RemoteProtocolError(msg)\nhttpcore.RemoteProtocolError: Server disconnected without sending a response.\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.RemoteProtocolError: Server disconnected without sending a response.\n","start":"2024-11-08 08:19:55.345563","end":"2024-11-08 08:19:58.680730","duration":3}},{"council_id":"KWL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:53:38] Fetching Scraper for: KWL handlers.py:23\n Begin attempting to scrape: KWL handlers.py:27\n[08:53:39] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:53:40] ...found 45 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 45 files in Councillors/raw base.py:225\n ...found 91 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 91 files base.py:236\n[08:53:41] ...data deleted. base.py:264\n Scraping from base.py:49\n http://councillors.knowsley.gov.uk/mgWebService.asmx/GetCo \n uncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'http://councillors.knowsley.gov.uk/mgWebService.asmx/ \n GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: KWL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'http://councillors.knowsley.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 08:53:38.902456","end":"2024-11-08 08:53:41.550291","duration":2}},{"council_id":"MAI","missing":false,"latest_run":{"status_code":1,"log_text":"[09:41:21] Fetching Scraper for: MAI handlers.py:23\n Begin attempting to scrape: MAI handlers.py:27\n Deleting existing data... base.py:257\n[09:41:22] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:41:23] ...data deleted. base.py:264\n Scraping from base.py:49\n https://meetings.maidstone.gov.uk//mgWebService.asmx/GetCo \n uncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://meetings.maidstone.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: MAI base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://meetings.maidstone.gov.uk//mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 09:41:21.269918","end":"2024-11-08 09:41:23.406897","duration":2}},{"council_id":"MAV","missing":false,"latest_run":{"status_code":1,"log_text":"[08:19:34] Fetching Scraper for: MAV handlers.py:23\n Begin attempting to scrape: MAV handlers.py:27\n[08:19:35] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:19:36] ...found 31 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 31 files in Councillors/raw base.py:225\n ...found 63 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 63 files base.py:236\n[08:19:37] ...data deleted. base.py:264\n Scraping from base.py:49\n http://moderngov.malvernhills.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'http://moderngov.malvernhills.gov.uk/mgWebService.asm \n x/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: MAV base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'http://moderngov.malvernhills.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 08:19:34.810475","end":"2024-11-08 08:19:37.295953","duration":2}},{"council_id":"MEL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:56:13] Fetching Scraper for: MEL handlers.py:23\n Begin attempting to scrape: MEL handlers.py:27\n Deleting existing data... base.py:257\n[08:56:14] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.melton.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n[08:56:45] The read operation timed out handlers.py:36\n Finished attempting to scrape: MEL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-11-08 08:56:13.159662","end":"2024-11-08 08:56:45.414682","duration":32}},{"council_id":"NEA","missing":false,"latest_run":{"status_code":1,"log_text":"[10:02:39] Fetching Scraper for: NEA handlers.py:23\n Begin attempting to scrape: NEA handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:02:40] Getting all files in Councillors/json... base.py:209\n ...found 39 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 39 files in Councillors/raw base.py:225\n ...found 79 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 79 files base.py:236\n[10:02:41] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.newark-sherwooddc.gov.uk/mgWebService.as \n mx/GetCouncillorsByWard \n[10:02:56] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: NEA base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 10:02:39.027205","end":"2024-11-08 10:02:56.978570","duration":17}},{"council_id":"NED","missing":false,"latest_run":{"status_code":1,"log_text":"[08:18:09] Fetching Scraper for: NED handlers.py:23\n[08:18:18] Begin attempting to scrape: NED handlers.py:27\n[08:18:20] Deleting existing data... base.py:257\n[08:18:21] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:18:22] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.ne-derbyshire.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard \n Server error '502 Bad Gateway' for url handlers.py:36\n 'https://democracy.ne-derbyshire.gov.uk/mgWebService.a \n smx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n Finished attempting to scrape: NED base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://democracy.ne-derbyshire.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-08 08:18:09.158068","end":"2024-11-08 08:18:22.780983","duration":13}},{"council_id":"NGM","missing":false,"latest_run":{"status_code":1,"log_text":"[09:22:14] Fetching Scraper for: NGM handlers.py:23\n Begin attempting to scrape: NGM handlers.py:27\n Deleting existing data... base.py:257\n[09:22:15] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 55 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 55 files in Councillors/raw base.py:225\n ...found 111 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:22:16] Deleting batch no. 2 consisting of 11 files base.py:236\n[09:22:17] ...data deleted. base.py:264\n Scraping from base.py:49\n http://committee.nottinghamcity.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard \n[09:22:47] The read operation timed out handlers.py:36\n Finished attempting to scrape: NGM base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-11-08 09:22:14.277037","end":"2024-11-08 09:22:47.751283","duration":33}},{"council_id":"NKE","missing":false,"latest_run":{"status_code":1,"log_text":"[09:17:58] Fetching Scraper for: NKE handlers.py:23\n[09:17:59] Begin attempting to scrape: NKE handlers.py:27\n Deleting existing data... base.py:257\n[09:18:00] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 43 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 43 files in Councillors/raw base.py:225\n ...found 87 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 87 files base.py:236\n[09:18:01] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.n-kesteven.gov.uk//mgWebService.asmx/Get \n CouncillorsByWard \n Server disconnected without sending a response. handlers.py:36\n Finished attempting to scrape: NKE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 238, in _receive_event\n raise RemoteProtocolError(msg)\nhttpcore.RemoteProtocolError: Server disconnected without sending a response.\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.RemoteProtocolError: Server disconnected without sending a response.\n","start":"2024-11-08 09:17:58.999091","end":"2024-11-08 09:18:01.751748","duration":2}},{"council_id":"NTL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:01:42] Fetching Scraper for: NTL handlers.py:23\n Begin attempting to scrape: NTL handlers.py:27\n Deleting existing data... base.py:257\n[09:01:43] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 59 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 59 files in Councillors/raw base.py:225\n ...found 119 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:01:44] Deleting batch no. 2 consisting of 19 files base.py:236\n[09:01:45] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.npt.gov.uk/mgWebService.asmx/GetCouncill \n orsByWard \n Server error '503 no server was available to handle handlers.py:36\n the request' for url \n 'https://democracy.npt.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: NTL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 no server was available to handle the request' for url 'https://democracy.npt.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 09:01:42.500916","end":"2024-11-08 09:01:45.810559","duration":3}},{"council_id":"OLD","missing":false,"latest_run":{"status_code":1,"log_text":"[08:57:10] Fetching Scraper for: OLD handlers.py:23\n Begin attempting to scrape: OLD handlers.py:27\n Deleting existing data... base.py:257\n[08:57:11] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 60 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 60 files in Councillors/raw base.py:225\n ...found 121 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:57:12] Deleting batch no. 2 consisting of 21 files base.py:236\n[08:57:13] ...data deleted. base.py:264\n Scraping from base.py:49\n http://committees.oldham.gov.uk/mgWebService.asmx/GetCounc \n illorsByWard \n[08:57:23] [Errno 104] Connection reset by peer handlers.py:36\n Finished attempting to scrape: OLD base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadError: [Errno 104] Connection reset by peer\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadError: [Errno 104] Connection reset by peer\n","start":"2024-11-08 08:57:10.149653","end":"2024-11-08 08:57:23.635407","duration":13}},{"council_id":"RIC","missing":false,"latest_run":{"status_code":1,"log_text":"[09:35:27] Fetching Scraper for: RIC handlers.py:23\n Begin attempting to scrape: RIC handlers.py:27\n Deleting existing data... base.py:257\n[09:35:28] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:35:29] ...data deleted. base.py:264\n Scraping from base.py:49\n https://cabnet.richmond.gov.uk/mgWebService.asmx/GetCounci \n llorsByWard \n Server error '502 Bad Gateway' for url handlers.py:36\n 'https://cabnet.richmond.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n Finished attempting to scrape: RIC base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://cabnet.richmond.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-08 09:35:27.557274","end":"2024-11-08 09:35:29.688072","duration":2}},{"council_id":"RUH","missing":false,"latest_run":{"status_code":1,"log_text":"[08:47:11] Fetching Scraper for: RUH handlers.py:23\n Begin attempting to scrape: RUH handlers.py:27\n Deleting existing data... base.py:257\n[08:47:12] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 39 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 39 files in Councillors/raw base.py:225\n ...found 79 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 79 files base.py:236\n[08:47:13] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.rushmoor.gov.uk//mgWebService.asmx/GetCo \n uncillorsByWard \n[08:47:16] Server error '503 Service Unavailable' for url handlers.py:36\n 'https://democracy.rushmoor.gov.uk//mgWebService.asmx/ \n GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: RUH base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://democracy.rushmoor.gov.uk//mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 08:47:11.261087","end":"2024-11-08 08:47:16.851025","duration":5}},{"council_id":"SFT","missing":false,"latest_run":{"status_code":1,"log_text":"[08:25:39] Fetching Scraper for: SFT handlers.py:23\n Begin attempting to scrape: SFT handlers.py:27\n Deleting existing data... base.py:257\n[08:25:40] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 65 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 65 files in Councillors/raw base.py:225\n ...found 131 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:25:41] Deleting batch no. 2 consisting of 31 files base.py:236\n[08:25:42] ...data deleted. base.py:264\n Scraping from base.py:49\n https://modgov.sefton.gov.uk/mgWebService.asmx/GetCouncill \n orsByWard \n Server error '502 Bad Gateway' for url handlers.py:36\n 'https://modgov.sefton.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n Finished attempting to scrape: SFT base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://modgov.sefton.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-08 08:25:39.018753","end":"2024-11-08 08:25:42.523667","duration":3}},{"council_id":"SHE","missing":false,"latest_run":{"status_code":null,"log_text":"[11:28:20] Fetching Scraper for: SHE handlers.py:22\n Begin attempting to scrape: SHE handlers.py:25\n Deleting existing data... base.py:234\n Getting all files in SHE... base.py:186\n[11:28:21] Getting all files in SHE/json... base.py:186\n ...found 30 files in SHE/json base.py:202\n Getting all files in SHE/raw... base.py:186\n ...found 30 files in SHE/raw base.py:202\n ...found 61 files in SHE base.py:202\n Deleting batch no. 1 consisting of 61 files base.py:211\n[11:28:32] An error occurred (ThrottlingException) when calling handlers.py:34\n the CreateCommit operation (reached max retries: 4): \n Rate exceeded \n Finished attempting to scrape: SHE base.py:319\n","errors":"An error occurred (ThrottlingException) when calling the CreateCommit operation (reached max retries: 4): Rate exceeded","start":"2022-04-04 11:28:20.509898","end":"2022-04-04 11:28:32.871624","duration":12}},{"council_id":"SOL","missing":false,"latest_run":{"status_code":1,"log_text":"[08:57:03] Fetching Scraper for: SOL handlers.py:23\n Begin attempting to scrape: SOL handlers.py:27\n[08:57:04] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:57:05] ...found 51 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 51 files in Councillors/raw base.py:225\n ...found 103 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:57:06] Deleting batch no. 2 consisting of 3 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.solihull.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n [Errno 111] Connection refused handlers.py:36\n Finished attempting to scrape: SOL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 111] Connection refused\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 111] Connection refused\n","start":"2024-11-08 08:57:03.837196","end":"2024-11-08 08:57:06.986729","duration":3}},{"council_id":"SPE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:26:53] Fetching Scraper for: SPE handlers.py:23\n Begin attempting to scrape: SPE handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[08:26:54] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:26:55] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.spelthorne.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n[08:27:10] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: SPE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 08:26:53.044071","end":"2024-11-08 08:27:10.827379","duration":17}},{"council_id":"STF","missing":false,"latest_run":{"status_code":1,"log_text":"[09:05:19] Fetching Scraper for: STF handlers.py:23\n Begin attempting to scrape: STF handlers.py:27\n[09:05:20] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[09:05:21] ...found 99 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 99 files in Councillors/raw base.py:225\n ...found 199 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[09:05:22] Deleting batch no. 2 consisting of 99 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.staffsmoorlands.gov.uk/mgWebService.asmx \n /GetCouncillorsByWard \n[09:05:38] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: STF base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 09:05:19.847154","end":"2024-11-08 09:05:38.630373","duration":18}},{"council_id":"STG","missing":false,"latest_run":{"status_code":1,"log_text":"[09:14:50] Fetching Scraper for: STG handlers.py:23\n Begin attempting to scrape: STG handlers.py:27\n[09:14:51] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[09:14:52] ...found 19 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 19 files in Councillors/raw base.py:225\n ...found 39 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 39 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/politic \n ians-and-elections/councillors/ \n[09:14:54] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/elaine-watterson/ \n[09:14:55] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gene-maxwell/ \n[09:14:56] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/martin-earl/ \n[09:14:58] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mcgarvey/ \n[09:14:59] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/paul-henke/ \n[09:15:00] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rosemary-fraser/ \n[09:15:01] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/alasdair-tollemache/ \n[09:15:03] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/david-wilson/ \n[09:15:04] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/robin-kleinman/ \n[09:15:05] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/thomas-heald/ \n[09:15:06] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/danny-gibson/ \n[09:15:08] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jim-thomson/ \n[09:15:09] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/rachel-nunn/ \n[09:15:12] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/susan-mcgill/ \n[09:15:13] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/jen-preston/ \n[09:15:14] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/neil-benny/ \n[09:15:15] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/scott-farmer/ \n[09:15:16] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/bryan-flannagan/ \n[09:15:18] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/gerry-mclaughlan/ \n[09:15:19] Scraping from base.py:49\n https://www.stirling.gov.uk/council-and-committees/council \n lors/your-councillors/vacant/ \n[09:15:20] 'NoneType' object is not subscriptable handlers.py:36\n Committing batch 1 consisting of 38 files base.py:297\n[09:15:21] Finished attempting to scrape: STG base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/STG-stirling/councillors.py\", line 42, in get_single_councillor\n councillor.email = soup.select_one(\"a[href^=mailto]\")[\nTypeError: 'NoneType' object is not subscriptable\n","start":"2024-11-08 09:14:50.853415","end":"2024-11-08 09:15:21.890676","duration":31}},{"council_id":"STH","missing":false,"latest_run":{"status_code":1,"log_text":"[08:32:13] Fetching Scraper for: STH handlers.py:23\n Begin attempting to scrape: STH handlers.py:27\n Deleting existing data... base.py:257\n[08:32:14] Getting all files in Councillors... base.py:209\n[08:32:15] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:32:16] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.southampton.gov.uk/modernGov//mgWebService.asm \n x/GetCouncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://www.southampton.gov.uk/modernGov//mgWebServic \n e.asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: STH base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://www.southampton.gov.uk/modernGov//mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 08:32:13.392420","end":"2024-11-08 08:32:16.692676","duration":3}},{"council_id":"SWL","missing":false,"latest_run":{"status_code":1,"log_text":"[10:32:58] Fetching Scraper for: SWL handlers.py:23\n[10:32:59] Begin attempting to scrape: SWL handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[10:33:00] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://services.swale.gov.uk/mgWebService.asmx/GetCouncill \n orsByWard \n[10:33:01] Server error '503 Service Unavailable' for url handlers.py:36\n 'https://ws.swale.gov.uk/meetings/mgWebService.asmx/Ge \n tCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: SWL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://ws.swale.gov.uk/meetings/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 10:32:58.975462","end":"2024-11-08 10:33:01.384125","duration":2}},{"council_id":"TEI","missing":false,"latest_run":{"status_code":1,"log_text":"[09:01:49] Fetching Scraper for: TEI handlers.py:23\n Begin attempting to scrape: TEI handlers.py:27\n[09:01:50] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 47 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[09:01:51] ...found 47 files in Councillors/raw base.py:225\n ...found 95 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 95 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.teignbridge.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n[09:01:55] [Errno 113] No route to host handlers.py:36\n Finished attempting to scrape: TEI base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectError: [Errno 113] No route to host\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectError: [Errno 113] No route to host\n","start":"2024-11-08 09:01:49.625057","end":"2024-11-08 09:01:55.151745","duration":5}},{"council_id":"TEW","missing":false,"latest_run":{"status_code":1,"log_text":"[08:43:14] Fetching Scraper for: TEW handlers.py:23\n Begin attempting to scrape: TEW handlers.py:27\n Deleting existing data... base.py:257\n[08:43:15] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:43:16] ...data deleted. base.py:264\n Scraping from base.py:49\n http://minutes.tewkesbury.gov.uk/mgWebService.asmx/GetCoun \n cillorsByWard \n[08:43:46] timed out handlers.py:36\n Finished attempting to scrape: TEW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: timed out\n","start":"2024-11-08 08:43:14.212549","end":"2024-11-08 08:43:46.628878","duration":32}},{"council_id":"THE","missing":false,"latest_run":{"status_code":1,"log_text":"[08:33:30] Fetching Scraper for: THE handlers.py:23\n Begin attempting to scrape: THE handlers.py:27\n Deleting existing data... base.py:257\n[08:33:31] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:33:32] ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.threerivers.gov.uk/listing/councillors \n[08:33:33] 'NoneType' object has no attribute 'findNext' handlers.py:36\n[08:33:34] Finished attempting to scrape: THE base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 55, in run\n for councillor_html in self.get_councillors():\n File \"/var/task/lgsf/councillors/scrapers.py\", line 161, in get_councillors\n container = self.get_list_container()\n File \"scrapers/THE-three-rivers/councillors.py\", line 15, in get_list_container\n return soup.find(\"h3\", text=\"District Councillor\").findNext(\"ul\")\nAttributeError: 'NoneType' object has no attribute 'findNext'\n","start":"2024-11-08 08:33:30.439604","end":"2024-11-08 08:33:34.150049","duration":3}},{"council_id":"TRF","missing":false,"latest_run":{"status_code":1,"log_text":"[08:29:42] Fetching Scraper for: TRF handlers.py:23\n Begin attempting to scrape: TRF handlers.py:27\n[08:29:43] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[08:29:44] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democratic.trafford.gov.uk/mgWebService.asmx/GetCo \n uncillorsByWard \n[08:30:14] The read operation timed out handlers.py:36\n Finished attempting to scrape: TRF base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 224, in _receive_event\n data = self._network_stream.read(\n File \"/opt/python/httpcore/_backends/sync.py\", line 124, in read\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ReadTimeout: The read operation timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ReadTimeout: The read operation timed out\n","start":"2024-11-08 08:29:42.592272","end":"2024-11-08 08:30:14.849172","duration":32}},{"council_id":"TUN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:36:21] Fetching Scraper for: TUN handlers.py:23\n[09:36:22] Begin attempting to scrape: TUN handlers.py:27\n Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:36:23] ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.tunbridgewells.gov.uk/mgWebService.asmx/ \n GetCouncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://democracy.tunbridgewells.gov.uk/mgWebService. \n asmx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n[09:36:24] Finished attempting to scrape: TUN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://democracy.tunbridgewells.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 09:36:21.964371","end":"2024-11-08 09:36:24.154088","duration":2}},{"council_id":"TWH","missing":false,"latest_run":{"status_code":1,"log_text":"[08:18:38] Fetching Scraper for: TWH handlers.py:23\n Begin attempting to scrape: TWH handlers.py:27\n[08:18:39] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:18:40] ...found 46 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 46 files in Councillors/raw base.py:225\n ...found 93 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 93 files base.py:236\n[08:18:41] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.towerhamlets.gov.uk/mgWebService.asmx/Get \n CouncillorsByWard \n Server error '502 Bad Gateway' for url handlers.py:36\n 'https://democracy.towerhamlets.gov.uk/mgWebService.as \n mx/GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n Finished attempting to scrape: TWH base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://democracy.towerhamlets.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-08 08:18:38.819638","end":"2024-11-08 08:18:41.479941","duration":2}},{"council_id":"VGL","missing":false,"latest_run":{"status_code":1,"log_text":"[09:50:31] Fetching Scraper for: VGL handlers.py:23\n Begin attempting to scrape: VGL handlers.py:27\n[09:50:32] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 16 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[09:50:33] ...found 16 files in Councillors/raw base.py:225\n ...found 33 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 33 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Councillors.aspx \n[09:50:36] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Asbrey-Anne.aspx \n[09:50:37] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Aviet-Julie.aspx \n[09:50:39] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Ball-Gareth.aspx \n[09:50:40] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Birch-Rhiannon.aspx \n[09:50:42] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Brooks-Bronwen.aspx \n[09:50:43] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Bruce-Gillian.aspx \n[09:50:47] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Buckley-Ian.aspx \n[09:50:48] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Burnett-Lis.aspx \n[09:50:50] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Campbell-Samantha.aspx \n[09:50:52] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Carroll-George.aspx \n[09:50:54] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cave-Christine.aspx \n[09:50:56] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Champion-Charles.aspx \n[09:50:57] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Charles-Janice.aspx \n[09:50:59] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Collins-Amelia.aspx \n[09:51:01] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Cowpe-Marianne.aspx \n[09:51:02] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Drake-Pamela.aspx \n[09:51:04] Scraping from base.py:49\n https://www.valeofglamorgan.gov.uk/en/our_council/Council- \n Structure/councillors/Driscoll-Vincent.aspx \n[09:51:06] 'NoneType' object has no attribute 'get_text' handlers.py:36\n Committing batch 1 consisting of 32 files base.py:297\n[09:51:07] Finished attempting to scrape: VGL base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/VGL-the-vale-of-glamorgan/councillors.py\", line 36, in get_single_councillor\n email_text = soup.select_one(\"#S4_EmailPlaceholder\").get_text(\nAttributeError: 'NoneType' object has no attribute 'get_text'\n","start":"2024-11-08 09:50:31.554083","end":"2024-11-08 09:51:07.587616","duration":36}},{"council_id":"WBK","missing":false,"latest_run":{"status_code":1,"log_text":"[09:42:22] Fetching Scraper for: WBK handlers.py:23\n Begin attempting to scrape: WBK handlers.py:27\n Deleting existing data... base.py:257\n[09:42:23] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 43 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 43 files in Councillors/raw base.py:225\n ...found 87 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 87 files base.py:236\n[09:42:24] ...data deleted. base.py:264\n Scraping from base.py:49\n http://decisionmaking.westberks.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard \n[09:42:39] [Errno 110] Connection timed out handlers.py:36\n[09:42:40] Finished attempting to scrape: WBK base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 09:42:22.169226","end":"2024-11-08 09:42:40.288610","duration":18}},{"council_id":"WEW","missing":false,"latest_run":{"status_code":1,"log_text":"[09:04:24] Fetching Scraper for: WEW handlers.py:23\n Begin attempting to scrape: WEW handlers.py:27\n Deleting existing data... base.py:257\n[09:04:25] Getting all files in Councillors... base.py:209\n ...found 1 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 1 files base.py:236\n[09:04:26] ...data deleted. base.py:264\n Scraping from base.py:49\n http://democracy.welhat.gov.uk/mgWebService.asmx/GetCounci \n llorsByWard \n[09:04:41] [Errno 110] Connection timed out handlers.py:36\n Finished attempting to scrape: WEW base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 99, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/connection.py\", line 76, in handle_request\n stream = self._connect(request)\n File \"/opt/python/httpcore/_sync/connection.py\", line 122, in _connect\n stream = self._network_backend.connect_tcp(**kwargs)\n File \"/opt/python/httpcore/_backends/sync.py\", line 205, in connect_tcp\n with map_exceptions(exc_map):\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpcore/_exceptions.py\", line 14, in map_exceptions\n raise to_exc(exc) from exc\nhttpcore.ConnectTimeout: [Errno 110] Connection timed out\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.ConnectTimeout: [Errno 110] Connection timed out\n","start":"2024-11-08 09:04:24.317643","end":"2024-11-08 09:04:41.869343","duration":17}},{"council_id":"WLI","missing":false,"latest_run":{"status_code":1,"log_text":"[09:32:48] Fetching Scraper for: WLI handlers.py:23\n Begin attempting to scrape: WLI handlers.py:27\n[09:32:49] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n[09:32:50] Getting all files in Councillors/json... base.py:209\n ...found 36 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 36 files in Councillors/raw base.py:225\n ...found 73 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 73 files base.py:236\n[09:32:51] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.west-lindsey.gov.uk/mgWebService.asmx/Ge \n tCouncillorsByWard \n Server disconnected without sending a response. handlers.py:36\n Finished attempting to scrape: WLI base.py:345\n","errors":"Traceback (most recent call last):\n File \"/opt/python/httpx/_transports/default.py\", line 69, in map_httpcore_exceptions\n yield\n File \"/opt/python/httpx/_transports/default.py\", line 233, in handle_request\n resp = self._pool.handle_request(req)\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 216, in handle_request\n raise exc from None\n File \"/opt/python/httpcore/_sync/connection_pool.py\", line 196, in handle_request\n response = connection.handle_request(\n File \"/opt/python/httpcore/_sync/connection.py\", line 101, in handle_request\n return self._connection.handle_request(request)\n File \"/opt/python/httpcore/_sync/http11.py\", line 143, in handle_request\n raise exc\n File \"/opt/python/httpcore/_sync/http11.py\", line 113, in handle_request\n ) = self._receive_response_headers(**kwargs)\n File \"/opt/python/httpcore/_sync/http11.py\", line 186, in _receive_response_headers\n event = self._receive_event(timeout=timeout)\n File \"/opt/python/httpcore/_sync/http11.py\", line 238, in _receive_event\n raise RemoteProtocolError(msg)\nhttpcore.RemoteProtocolError: Server disconnected without sending a response.\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 55, in get\n response = self.http_client.get(url, headers=headers, timeout=30)\n File \"/opt/python/httpx/_client.py\", line 1054, in get\n return self.request(\n File \"/opt/python/httpx/_client.py\", line 827, in request\n return self.send(request, auth=auth, follow_redirects=follow_redirects)\n File \"/opt/python/httpx/_client.py\", line 914, in send\n response = self._send_handling_auth(\n File \"/opt/python/httpx/_client.py\", line 942, in _send_handling_auth\n response = self._send_handling_redirects(\n File \"/opt/python/httpx/_client.py\", line 979, in _send_handling_redirects\n response = self._send_single_request(request)\n File \"/opt/python/httpx/_client.py\", line 1015, in _send_single_request\n response = transport.handle_request(request)\n File \"/opt/python/httpx/_transports/default.py\", line 232, in handle_request\n with map_httpcore_exceptions():\n File \"/var/lang/lib/python3.10/contextlib.py\", line 153, in __exit__\n self.gen.throw(typ, value, traceback)\n File \"/opt/python/httpx/_transports/default.py\", line 86, in map_httpcore_exceptions\n raise mapped_exc(message) from exc\nhttpx.RemoteProtocolError: Server disconnected without sending a response.\n","start":"2024-11-08 09:32:48.916888","end":"2024-11-08 09:32:51.695125","duration":2}},{"council_id":"WLN","missing":false,"latest_run":{"status_code":1,"log_text":"[09:43:58] Fetching Scraper for: WLN handlers.py:23\n Begin attempting to scrape: WLN handlers.py:27\n[09:43:59] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 25 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n[09:44:00] ...found 25 files in Councillors/raw base.py:225\n ...found 51 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 51 files base.py:236\n ...data deleted. base.py:264\n Scraping from https://westlothian.gov.uk/councillors base.py:49\n[09:44:01] Scraping from base.py:49\n https://westlothian.gov.uk/article/33888/Linlithgow \n[09:44:02] Scraping from base.py:49\n https://westlothian.gov.uk/article/33889/Broxburn-Uphall-a \n nd-Winchburgh \n[09:44:03] Scraping from base.py:49\n https://westlothian.gov.uk/article/33890/Livingston-North \n[09:44:04] Scraping from base.py:49\n https://westlothian.gov.uk/article/33891/Livingston-South \n[09:44:05] Scraping from base.py:49\n https://westlothian.gov.uk/article/33893/East-Livingston-a \n nd-East-Calder \n[09:44:06] Scraping from base.py:49\n https://westlothian.gov.uk/article/33892/Fauldhouse-and-th \n e-Breich-Valley \n[09:44:08] Scraping from base.py:49\n https://westlothian.gov.uk/article/33894/Whitburn-and-Blac \n kburn \n[09:44:09] Scraping from base.py:49\n https://westlothian.gov.uk/article/33895/Bathgate \n[09:44:10] Scraping from base.py:49\n https://westlothian.gov.uk/article/33896/Armadale-and-Blac \n kridge \n[09:44:11] Scraping from base.py:49\n https://westlothian.gov.uk/article/33897/Councillor-Tom-Co \n nn \n[09:44:12] Scraping from base.py:49\n https://westlothian.gov.uk/article/33898/Councillor-Paulin \n e-Orr \n Scraping from base.py:49\n https://westlothian.gov.uk/article/33899/Councillor-Sally- \n Pattle \n[09:44:14] Scraping from base.py:49\n https://westlothian.gov.uk/article/34305/Councillor-Tony-B \n oyle \n[09:44:15] Scraping from base.py:49\n https://westlothian.gov.uk/article/34302/Councillor-Diane- \n Calder \n[09:44:16] Scraping from base.py:49\n https://westlothian.gov.uk/article/34303/Councillor-Janet- \n Campbell \n[09:44:17] Scraping from base.py:49\n https://westlothian.gov.uk/article/34304/Councillor-Angela \n -Doran-Timson \n Scraping from base.py:49\n https://westlothian.gov.uk/article/34307/Councillor-Alison \n -Adamson \n[09:44:18] Scraping from base.py:49\n https://westlothian.gov.uk/article/34306/Councillor-Robert \n -De-Bold \n[09:44:19] Scraping from base.py:49\n https://westlothian.gov.uk/article/34308/Councillor-Anne-M \n cMillan \n[09:44:20] Scraping from base.py:49\n https://westlothian.gov.uk/article/34309/Councillor-Andrew \n -Miller \n[09:44:22] Scraping from base.py:49\n https://westlothian.gov.uk/article/34311/Councillor-Lawren \n ce-Fitzpatrick \n[09:44:23] Scraping from base.py:49\n https://westlothian.gov.uk/article/34310/Councillor-Peter- \n Heggie \n[09:44:25] Scraping from base.py:49\n https://westlothian.gov.uk/article/34312/Councillor-Maria- \n MacAulay \n[09:44:26] Scraping from base.py:49\n https://westlothian.gov.uk/article/34313/Councillor-Moira- \n McKee-Shemilt \n[09:44:27] Scraping from base.py:49\n https://westlothian.gov.uk/article/34317/Councillor-Damian \n -Doran-Timson \n[09:44:28] Scraping from base.py:49\n https://westlothian.gov.uk/article/34315/Councillor-Carl-J \n ohn \n[09:44:29] Scraping from base.py:49\n https://westlothian.gov.uk/article/34316/Councillor-Danny- \n Logue \n[09:44:30] Scraping from base.py:49\n https://westlothian.gov.uk/article/34314/Councillor-Veroni \n ca-Smith \n[09:44:31] Scraping from base.py:49\n https://westlothian.gov.uk/article/34318/Councillor-Paulin \n e-Clark \n[09:44:32] Scraping from base.py:49\n https://westlothian.gov.uk/article/34319/Councillor-Craig- \n Meek \n[09:44:33] Scraping from base.py:49\n https://westlothian.gov.uk/article/34320/Councillor-Cathy- \n Muldoon \n[09:44:34] Scraping from base.py:49\n https://westlothian.gov.uk/article/34321/Councillor-Jim-Di \n ckson \n[09:44:35] Scraping from base.py:49\n https://westlothian.gov.uk/article/34322/Councillor-Mary-D \n ickson \n[09:44:36] Scraping from base.py:49\n https://westlothian.gov.uk/article/34323/Councillor-George \n -Paul \n[09:44:37] Scraping from base.py:49\n https://westlothian.gov.uk/article/34324/Councillor-Vacant \n[09:44:38] list index out of range handlers.py:36\n Committing batch 1 consisting of 50 files base.py:297\n[09:44:39] Finished attempting to scrape: WLN base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 57, in run\n councillor = self.get_single_councillor(councillor_html)\n File \"scrapers/WLN-west-lothian/councillors.py\", line 37, in get_single_councillor\n councillor.email = soup.select(\"a[href^=mailto]\")[0].get_text(\nIndexError: list index out of range\n","start":"2024-11-08 09:43:58.708361","end":"2024-11-08 09:44:39.594112","duration":40}},{"council_id":"WND","missing":false,"latest_run":{"status_code":1,"log_text":"[08:33:37] Fetching Scraper for: WND handlers.py:23\n Begin attempting to scrape: WND handlers.py:27\n Deleting existing data... base.py:257\n[08:33:38] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 58 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 58 files in Councillors/raw base.py:225\n ...found 117 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 100 files base.py:236\n[08:33:39] Deleting batch no. 2 consisting of 17 files base.py:236\n[08:33:40] ...data deleted. base.py:264\n Scraping from base.py:49\n https://democracy.wandsworth.gov.uk/mgWebService.asmx/GetC \n ouncillorsByWard \n Server error '502 Bad Gateway' for url handlers.py:36\n 'https://democracy.wandsworth.gov.uk/mgWebService.asmx \n /GetCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/502 \n Finished attempting to scrape: WND base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '502 Bad Gateway' for url 'https://democracy.wandsworth.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502\n","start":"2024-11-08 08:33:37.366808","end":"2024-11-08 08:33:40.701337","duration":3}},{"council_id":"WOC","missing":false,"latest_run":{"status_code":1,"log_text":"[08:20:26] Fetching Scraper for: WOC handlers.py:23\n Begin attempting to scrape: WOC handlers.py:27\n[08:20:27] Deleting existing data... base.py:257\n Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n[08:20:28] ...found 35 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 35 files in Councillors/raw base.py:225\n ...found 71 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 71 files base.py:236\n ...data deleted. base.py:264\n Scraping from base.py:49\n http://committee.worcester.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'http://committee.worcester.gov.uk/mgWebService.asmx/G \n etCouncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n[08:20:29] Finished attempting to scrape: WOC base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'http://committee.worcester.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 08:20:26.756220","end":"2024-11-08 08:20:29.166484","duration":2}},{"council_id":"WYC","missing":false,"latest_run":{"status_code":1,"log_text":"[08:58:31] Fetching Scraper for: WYC handlers.py:23\n Begin attempting to scrape: WYC handlers.py:27\n Deleting existing data... base.py:257\n[08:58:32] Getting all files in Councillors... base.py:209\n Getting all files in Councillors/json... base.py:209\n ...found 43 files in Councillors/json base.py:225\n Getting all files in Councillors/raw... base.py:209\n ...found 43 files in Councillors/raw base.py:225\n ...found 87 files in Councillors base.py:225\n Deleting batch no. 1 consisting of 87 files base.py:236\n[08:58:33] ...data deleted. base.py:264\n Scraping from base.py:49\n https://mgov.wychavon.gov.uk/mgWebService.asmx/GetCouncill \n orsByWard \n Server error '503 Service Unavailable' for url handlers.py:36\n 'https://mgov.wychavon.gov.uk/mgWebService.asmx/GetCou \n ncillorsByWard' \n For more information check: \n https://developer.mozilla.org/en-US/docs/Web/HTTP/Stat \n us/503 \n Finished attempting to scrape: WYC base.py:345\n","errors":"Traceback (most recent call last):\n File \"/var/task/lgsf/aws_lambda/handlers.py\", line 32, in scraper_worker_handler\n scraper.run(run_log)\n File \"/var/task/lgsf/councillors/scrapers.py\", line 197, in run\n wards = self.get_councillors()\n File \"/var/task/lgsf/councillors/scrapers.py\", line 216, in get_councillors\n req = self.get(self.format_councillor_api_url())\n File \"/var/task/lgsf/scrapers/base.py\", line 56, in get\n response.raise_for_status()\n File \"/opt/python/httpx/_models.py\", line 761, in raise_for_status\n raise HTTPStatusError(message, request=request, response=self)\nhttpx.HTTPStatusError: Server error '503 Service Unavailable' for url 'https://mgov.wychavon.gov.uk/mgWebService.asmx/GetCouncillorsByWard'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503\n","start":"2024-11-08 08:58:31.232361","end":"2024-11-08 08:58:33.732287","duration":2}}] diff --git a/index.html b/index.html index dd6ffadce2..24f9702191 100644 --- a/index.html +++ b/index.html @@ -49,10 +49,6 @@
[10:09:04] Fetching Scraper for: ABC handlers.py:23 + Begin attempting to scrape: ABC handlers.py:27 + Deleting existing data... base.py:257 +[10:09:05] Getting all files in Councillors... base.py:209 + Getting all files in Councillors/json... base.py:209 + ...found 16 files in Councillors/json base.py:225 + Getting all files in Councillors/raw... base.py:209 + ...found 16 files in Councillors/raw base.py:225 + ...found 33 files in Councillors base.py:225 + Deleting batch no. 1 consisting of 33 files base.py:236 +[10:09:06] ...data deleted. base.py:264 + Scraping from base.py:49 + https://www.armaghbanbridgecraigavon.gov.uk/councillors/ +[10:09:11] Committing batch 1 consisting of 32 files base.py:297 +[10:09:12] Finished attempting to scrape: ABC base.py:345 ++ + + + +
[10:42:42] Fetching Scraper for: ABC handlers.py:23 - Begin attempting to scrape: ABC handlers.py:27 - Deleting existing data... base.py:257 - Getting all files in Councillors... base.py:209 -[10:42:43] Getting all files in Councillors/json... base.py:209 - ...found 16 files in Councillors/json base.py:225 - Getting all files in Councillors/raw... base.py:209 - ...found 16 files in Councillors/raw base.py:225 - ...found 33 files in Councillors base.py:225 - Deleting batch no. 1 consisting of 33 files base.py:236 -[10:42:44] ...data deleted. base.py:264 - Scraping from base.py:49 - https://www.armaghbanbridgecraigavon.gov.uk/councillors/ -[10:42:48] Committing batch 1 consisting of 32 files base.py:297 -[10:42:49] Finished attempting to scrape: ABC base.py:345 -- -