Merge pull request #3675 from MPMG-DCC-UFMG/issue-2559

Tornando a porta configurável na finalização dos crawlers

crawlers/base_spider.py

@@ -29,6 +29,8 @@
 from main.models import CrawlRequest
 from crawlers.constants import AUTO_ENCODE_DETECTION_CONFIDENCE_THRESHOLD
 
+from crawlers.port import *
+
 PUNCTUATIONS = "[{}]".format(string.punctuation)
 
 
@@ -465,7 +467,6 @@ def preprocess_link_configs(self):
     def spider_closed(self, spider):
         crawler_id = spider.crawler.settings.get('CRAWLER_ID')
 
-        # TODO: get port as variable
-        port = 8000
+        # PORT comes from the port module and is set on run.py directly from CLI input
         requests.get(
-            f'http://localhost:{port}/detail/stop_crawl/{crawler_id}')
+            f'http://localhost:{PORT}/detail/stop_crawl/{crawler_id}')

run.py

@@ -41,6 +41,16 @@ def run_django():
     signal.signal(signal.SIGCHLD, lambda _, __: os.wait())
     wait_for_port(9092)
 
+    # gets port from CLI parameter and sets module to be used on spider_closed()
+    if (len(sys.argv[1:]) == 0):
+        cli_port = 8000
+    else:
+        cli_port = str(sys.argv[1]).split(":",1)[1]
+
+    f = open("./crawlers/port.py", "w") 
+    f.write("PORT="+str(cli_port)) 
+    f.close()
+
     # Runs django repassing cli parameters
     subprocess.run(["python", "manage.py", "runserver"] + sys.argv[1:])