From 52d6911abb119e49f7555797921d1c1cffa1a1e2 Mon Sep 17 00:00:00 2001 From: Jennifer Chen Date: Mon, 15 Jul 2024 16:09:45 -0700 Subject: [PATCH] rebase master --- server/webdriver/base_utils.py | 10 +++- shared/lib/test_server.py | 103 +++++++++++++++++++++------------ shared/lib/test_setup.py | 5 +- 3 files changed, 78 insertions(+), 40 deletions(-) diff --git a/server/webdriver/base_utils.py b/server/webdriver/base_utils.py index df8ed96137..0049a5d277 100644 --- a/server/webdriver/base_utils.py +++ b/server/webdriver/base_utils.py @@ -31,7 +31,15 @@ def create_driver(preferences=None): chrome_options.add_argument('--hide-scrollbars') if preferences: chrome_options.add_experimental_option("prefs", preferences) - driver = webdriver.Chrome(options=chrome_options) + try: + print("before creating driver") + driver = webdriver.Chrome(options=chrome_options) + print("after creating driver") + except Exception as e: + print("caught exception") + print(str(e)) + + # Set a reliable window size for all tests (can be overwritten though) driver.set_window_size(DEFAULT_WIDTH, DEFAULT_HEIGHT) return driver diff --git a/shared/lib/test_server.py b/shared/lib/test_server.py index 2da4d74124..1a4b9ecca3 100644 --- a/shared/lib/test_server.py +++ b/shared/lib/test_server.py @@ -16,70 +16,101 @@ import os import platform import socket +import unittest import warnings -from flask_testing import LiveServerTestCase - from nl_server.flask import create_app as create_nl_app from server.__init__ import create_app as create_web_app import server.lib.util as libutil -def find_open_port(): +def find_open_port(skip_ports: set[int] | None = None): for port in range(12000, 13000): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + if skip_ports and port in skip_ports: + continue res = sock.connect_ex(('localhost', port)) if res != 0: return port - -is_nl_mode = os.environ.get('ENABLE_MODEL') == 'true' -if is_nl_mode: +def get_nl_port(web_port: int): # Start NL server on an unused port, so multiple integration tests can # run at the same time. if platform.system() == 'Darwin' and platform.processor() == 'arm': - msg = '\n\n!!!!! IMPORTANT NOTE !!!!!!\n' \ - 'Detected MacOS ARM processor! You need to have a local ' \ - 'NL server running (using run_nl_server.sh).\n' - warnings.warn(msg) - nl_port = 6060 - should_start_nl_server = False + msg = '\n\n!!!!! IMPORTANT NOTE !!!!!!\n' \ + 'Detected MacOS ARM processor! You need to have a local ' \ + 'NL server running (using run_nl_server.sh).\n' + warnings.warn(msg) + nl_port = 6060 + should_start_nl_server = False + else: + nl_port = find_open_port(set[web_port]) + should_start_nl_server = True + return nl_port, should_start_nl_server + + +def start_nl_server(port): + nl_app = create_nl_app() + nl_app.run(port=port, debug=False, use_reloader=False, threaded=True) + + +def start_web_server(web_port, nl_port=None): + if nl_port: + web_app = create_web_app(f'http://127.0.0.1:{nl_port}') else: - nl_port = find_open_port() - should_start_nl_server = True + web_app = create_web_app() + web_app.run(port=web_port, use_reloader=False) -class NLWebServerTestCase(LiveServerTestCase): +class NLWebServerTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - if is_nl_mode: + cls.web_port = find_open_port() + cls.is_nl_mode = os.environ.get('ENABLE_MODEL') == 'true' + nl_port = None + if cls.is_nl_mode: + nl_port, should_start_nl_server = get_nl_port(cls.web_port) if should_start_nl_server: - def start_nl_server(app): - app.run(port=nl_port, debug=False, use_reloader=False, threaded=True) - - nl_app = create_nl_app() # Create a thread that will contain our running server - cls.proc = multiprocessing.Process(target=start_nl_server, - args=(nl_app,), - daemon=True) - cls.proc.start() + cls.nl_proc = multiprocessing.Process(target=start_nl_server, + args=(nl_port,), + daemon=True) + cls.nl_proc.start() else: - cls.proc = None + cls.nl_proc = None libutil.check_backend_ready( ['http://127.0.0.1:{}/healthz'.format(nl_port)]) + skip_ports = set([cls.web_port, nl_port]) + # Start web app. + for _ in range(5): + try: + print("trying to start server") + cls.web_proc = multiprocessing.Process(target=start_web_server, + args=(cls.web_port, nl_port)) + cls.web_proc.start() + print("server started") + break + except Exception as e: + print("exception caught in test server") + print(str(e)) + if "already in use" in str(e): + cls.web_port = find_open_port(skip_ports) + skip_ports.add(cls.web_port) + print(skip_ports) + continue + else: + print("raising exception") + raise e + libutil.check_backend_ready([f'http://127.0.0.1:{cls.web_port}/healthz']) + + def get_server_url(cls): + return f'http://localhost:{cls.web_port}' + @classmethod def tearDownClass(cls): - if is_nl_mode and cls.proc: - cls.proc.terminate() - - def create_app(self): - """Returns the Flask Server running Data Commons.""" - if is_nl_mode: - app = create_web_app('http://127.0.0.1:{}'.format(nl_port)) - else: - app = create_web_app() - app.config['LIVESERVER_PORT'] = 0 - return app + if cls.is_nl_mode and cls.nl_proc: + cls.nl_proc.terminate() + cls.web_proc.terminate() \ No newline at end of file diff --git a/shared/lib/test_setup.py b/shared/lib/test_setup.py index b3b97f2e0b..c8ef3e9f68 100644 --- a/shared/lib/test_setup.py +++ b/shared/lib/test_setup.py @@ -34,6 +34,5 @@ def set_up_macos_for_tests(): if _set_up_done: return _set_up_done = True - if sys.version_info >= (3, 8) and sys.platform == "darwin": - multiprocessing.set_start_method("fork") - os.environ['no_proxy'] = '*' + multiprocessing.set_start_method("spawn") + os.environ['no_proxy'] = '*'