Custom downloader support ShadowDOM

algolia · Sep 3, 2021 · 50ab814 · 50ab814
1 parent 4267a74
commit 50ab814
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/scraper/src/custom_downloader_middleware.py b/scraper/src/custom_downloader_middleware.py
@@ -28,7 +28,7 @@ def process_request(self, request, spider):
         self.driver.get(unquote_plus(
             request.url))  # Decode url otherwise firefox is not happy. Ex /#%21/ => /#!/%21
         time.sleep(spider.js_wait)
-        body = self.driver.page_source.encode('utf-8')
+        body = self.driver.execute_script("return document.documentElement.getInnerHTML ? document.documentElement.getInnerHTML() : document.documentElement.outerHTML;")
         url = self.driver.current_url
 
         return HtmlResponse(