Merge pull request #93 from DirectRoot/timing-fixes

Timing fixes
joeyism · Apr 13, 2021 · f499086 · f499086
2 parents fed3399 + 9e7c5b9
commit f499086
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 11 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,5 @@ __pycache__
 *.pyc
 chromedriver
 .DS_Store
+.vscode/
+.env/
diff --git a/linkedin_scraper/actions.py b/linkedin_scraper/actions.py
@@ -27,4 +27,4 @@ def login(driver, email=None, password=None):
   password_elem.send_keys(password)
   password_elem.submit()
 
-  element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "profile-nav-item")))
+  element = WebDriverWait(driver, 300).until(EC.presence_of_element_located((By.ID, "profile-nav-item")))
diff --git a/linkedin_scraper/person.py b/linkedin_scraper/person.py
@@ -10,6 +10,7 @@
 class Person(Scraper):
 
     __TOP_CARD = "pv-top-card"
+    __WAIT_FOR_ELEMENT_TIMEOUT = 5
 
     def __init__(
         self,
@@ -90,7 +91,7 @@ def scrape(self, close_on_complete=True):
 
     def _click_see_more_by_class_name(self, class_name):
         try:
-            _ = WebDriverWait(self.driver, 3).until(
+            _ = WebDriverWait(self.driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
                 EC.presence_of_element_located((By.CLASS_NAME, class_name))
             )
             div = self.driver.find_element_by_class_name(class_name)
@@ -102,7 +103,7 @@ def scrape_logged_in(self, close_on_complete=True):
         driver = self.driver
         duration = None
 
-        root = WebDriverWait(driver, 3).until(
+        root = WebDriverWait(driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
             EC.presence_of_element_located(
                 (
                     By.CLASS_NAME,
@@ -114,7 +115,7 @@ def scrape_logged_in(self, close_on_complete=True):
 
         # get about
         try:
-            see_more = WebDriverWait(driver, 3).until(
+            see_more = WebDriverWait(driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
                 EC.presence_of_element_located(
                     (
                         By.XPATH,
@@ -124,7 +125,7 @@ def scrape_logged_in(self, close_on_complete=True):
             )
             driver.execute_script("arguments[0].click();", see_more)
 
-            about = WebDriverWait(driver, 3).until(
+            about = WebDriverWait(driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
                 EC.presence_of_element_located(
                     (
                         By.XPATH,
@@ -150,7 +151,7 @@ def scrape_logged_in(self, close_on_complete=True):
         self._click_see_more_by_class_name("pv-experience-section__see-more")
 
         try:
-            _ = WebDriverWait(driver, 3).until(
+            _ = WebDriverWait(driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
                 EC.presence_of_element_located((By.ID, "experience-section"))
             )
             exp = driver.find_element_by_id("experience-section")
@@ -207,7 +208,7 @@ def scrape_logged_in(self, close_on_complete=True):
         ## Click SEE MORE
         self._click_see_more_by_class_name("pv-education-section__see-more")
         try:
-            _ = WebDriverWait(driver, 3).until(
+            _ = WebDriverWait(driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
                 EC.presence_of_element_located((By.ID, "education-section"))
             )
             edu = driver.find_element_by_id("education-section")
@@ -245,7 +246,7 @@ def scrape_logged_in(self, close_on_complete=True):
         # get interest
         try:
 
-            _ = WebDriverWait(driver, 3).until(
+            _ = WebDriverWait(driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
                 EC.presence_of_element_located(
                     (
                         By.XPATH,
@@ -268,7 +269,7 @@ def scrape_logged_in(self, close_on_complete=True):
 
         # get accomplishment
         try:
-            _ = WebDriverWait(driver, 3).until(
+            _ = WebDriverWait(driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
                 EC.presence_of_element_located(
                     (
                         By.XPATH,
@@ -294,7 +295,7 @@ def scrape_logged_in(self, close_on_complete=True):
         # get connections
         try:
             driver.get("https://www.linkedin.com/mynetwork/invite-connect/connections/")
-            _ = WebDriverWait(driver, 5).until(
+            _ = WebDriverWait(driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
                 EC.presence_of_element_located((By.CLASS_NAME, "mn-connections"))
             )
             connections = driver.find_element_by_class_name("mn-connections")
@@ -327,7 +328,7 @@ def scrape_not_logged_in(self, close_on_complete=True, retry_limit=10):
 
         # get experience
         try:
-            _ = WebDriverWait(driver, 3).until(
+            _ = WebDriverWait(driver, self.__WAIT_FOR_ELEMENT_TIMEOUT).until(
                 EC.presence_of_element_located((By.CLASS_NAME, "experience"))
             )
             exp = driver.find_element_by_class_name("experience")