diff --git a/pipelines/utils/crawler_tse_eleicoes/utils.py b/pipelines/utils/crawler_tse_eleicoes/utils.py index 85762bec3..ccb1d1e07 100644 --- a/pipelines/utils/crawler_tse_eleicoes/utils.py +++ b/pipelines/utils/crawler_tse_eleicoes/utils.py @@ -132,19 +132,16 @@ def download_extract_zip(self, url: str, chunk_size=128) -> None: "https": self.proxy } - proxy_test = requests.get('https://api.myip.com/', proxies=proxies, verify=False) - log(proxy_test) - - r = requests.get(url, headers=request_headers, proxies=proxies, verify=False, timeout=300) + r = requests.get(url, headers=request_headers, proxies=proxies, stream=True, verify=False, timeout=300) save_path = self.path_input / url.split("/")[-1] - with open(save_path, "wb") as fd: - fd.write(r.content) - # with open(save_path, "wb") as fd: - # for chunk in r.iter_content(chunk_size=chunk_size): - # fd.write(chunk) + # fd.write(r.content) + + with open(save_path, "wb") as fd: + for chunk in r.iter_content(chunk_size=chunk_size): + fd.write(chunk) with zipfile.ZipFile(save_path) as z: z.extractall(self.path_input)