diff --git a/scraper/src/custom_downloader_middleware.py b/scraper/src/custom_downloader_middleware.py index c3bea000..199d1213 100644 --- a/scraper/src/custom_downloader_middleware.py +++ b/scraper/src/custom_downloader_middleware.py @@ -28,7 +28,7 @@ def process_request(self, request, spider): self.driver.get(unquote_plus( request.url)) # Decode url otherwise firefox is not happy. Ex /#%21/ => /#!/%21 time.sleep(spider.js_wait) - body = self.driver.page_source.encode('utf-8') + body = self.driver.execute_script("return document.documentElement.getInnerHTML ? document.documentElement.getInnerHTML() : document.documentElement.outerHTML;") url = self.driver.current_url return HtmlResponse(