From efe95730efb90ad2379c702322636b4f44e682fc Mon Sep 17 00:00:00 2001 From: Luiz Eduardo Date: Wed, 27 Nov 2024 20:24:23 -0300 Subject: [PATCH] =?UTF-8?q?Adicionar=20verifica=C3=A7=C3=A3o=20de=20ip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipelines/utils/crawler_tse_eleicoes/utils.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/pipelines/utils/crawler_tse_eleicoes/utils.py b/pipelines/utils/crawler_tse_eleicoes/utils.py index 85762bec3..ccb1d1e07 100644 --- a/pipelines/utils/crawler_tse_eleicoes/utils.py +++ b/pipelines/utils/crawler_tse_eleicoes/utils.py @@ -132,19 +132,16 @@ def download_extract_zip(self, url: str, chunk_size=128) -> None: "https": self.proxy } - proxy_test = requests.get('https://api.myip.com/', proxies=proxies, verify=False) - log(proxy_test) - - r = requests.get(url, headers=request_headers, proxies=proxies, verify=False, timeout=300) + r = requests.get(url, headers=request_headers, proxies=proxies, stream=True, verify=False, timeout=300) save_path = self.path_input / url.split("/")[-1] - with open(save_path, "wb") as fd: - fd.write(r.content) - # with open(save_path, "wb") as fd: - # for chunk in r.iter_content(chunk_size=chunk_size): - # fd.write(chunk) + # fd.write(r.content) + + with open(save_path, "wb") as fd: + for chunk in r.iter_content(chunk_size=chunk_size): + fd.write(chunk) with zipfile.ZipFile(save_path) as z: z.extractall(self.path_input)