From bed6b42110ea65c9d88a1958ed6208540adb769b Mon Sep 17 00:00:00 2001 From: BPplays Date: Fri, 6 Dec 2024 04:24:04 -0800 Subject: [PATCH 1/9] started adding ipv6 support to openedai api --- extensions/openai/script.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 03d99e8ded..e794ed0319 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -352,23 +352,45 @@ async def handle_unload_loras(): return JSONResponse(content="OK") + + def run_server(): - server_addr = '0.0.0.0' if shared.args.listen else '127.0.0.1' + server_addrV6 = '[::]' if shared.args.listen else '[::1]' + server_addrV4 = '0.0.0.0' if shared.args.listen else '127.0.0.1' port = int(os.environ.get('OPENEDAI_PORT', shared.args.api_port)) + server_addr = [] + + disable_ipv6 = os.environ.get('OPENEDAI_DISABLE_IPV6', shared.args.api_disable_ipv6) + if not disable_ipv6: + server_addr.append(server_addrV6) + + disable_ipv4 = os.environ.get('OPENEDAI_DISABLE_IPV4', shared.args.api_disable_ipv4) + if not disable_ipv4: + server_addr.append(server_addrV4) ssl_certfile = os.environ.get('OPENEDAI_CERT_PATH', shared.args.ssl_certfile) ssl_keyfile = os.environ.get('OPENEDAI_KEY_PATH', shared.args.ssl_keyfile) + + url_proto = 'http://' + if ssl_keyfile and ssl_certfile: + url_proto = 'https://' + + url_s = "" + if len(server_addr) > 1: + url_s = "s" + + if shared.args.public_api: def on_start(public_url: str): logger.info(f'OpenAI-compatible API URL:\n\n{public_url}\n') _start_cloudflared(port, shared.args.public_api_id, max_attempts=3, on_start=on_start) else: - if ssl_keyfile and ssl_certfile: - logger.info(f'OpenAI-compatible API URL:\n\nhttps://{server_addr}:{port}\n') - else: - logger.info(f'OpenAI-compatible API URL:\n\nhttp://{server_addr}:{port}\n') + logger.info(f'OpenAI-compatible API URL{url_s}:\n\n') + for addr in server_addr: + logger.info(f'{url_proto}{addr}:{port}') + logger.info('\n') if shared.args.api_key: if not shared.args.admin_key: From 62d8ab761b0d5536de4fe1fdd79bef69ae963b16 Mon Sep 17 00:00:00 2001 From: BPplays Date: Fri, 6 Dec 2024 04:24:31 -0800 Subject: [PATCH 2/9] started adding ipv6 support to openedai api --- modules/shared.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/shared.py b/modules/shared.py index 894ed6fe56..9a7aad6eb1 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -195,6 +195,8 @@ # API group = parser.add_argument_group('API') group.add_argument('--api', action='store_true', help='Enable the API extension.') +group.add_argument('--api-disable-ipv6', type=bool, default=False, help='Disables IPv6 for the api') +group.add_argument('--api-disable-ipv4', type=bool, default=False, help='Disables IPv4 for the api') group.add_argument('--public-api', action='store_true', help='Create a public URL for the API using Cloudfare.') group.add_argument('--public-api-id', type=str, help='Tunnel ID for named Cloudflare Tunnel. Use together with public-api option.', default=None) group.add_argument('--api-port', type=int, default=5000, help='The listening port for the API.') From 30d097d3075e48ea5fbe5a42a6d7fbcafd66c87b Mon Sep 17 00:00:00 2001 From: BPplays Date: Fri, 6 Dec 2024 04:32:36 -0800 Subject: [PATCH 3/9] changed formatting --- extensions/openai/script.py | 11 +++++++---- modules/shared.py | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index e794ed0319..608e0a497f 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -387,10 +387,13 @@ def on_start(public_url: str): _start_cloudflared(port, shared.args.public_api_id, max_attempts=3, on_start=on_start) else: - logger.info(f'OpenAI-compatible API URL{url_s}:\n\n') - for addr in server_addr: - logger.info(f'{url_proto}{addr}:{port}') - logger.info('\n') + logger.info(f'\n\nOpenAI-compatible API URL{url_s}:') + for i, addr in enumerate(server_addr): + nl = '' + if i == len(server_addr) - 1: + nl = '\n\n' + + logger.info(f'{url_proto}{addr}:{port}{nl}') if shared.args.api_key: if not shared.args.admin_key: diff --git a/modules/shared.py b/modules/shared.py index 9a7aad6eb1..a79a6bd6bf 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -195,8 +195,8 @@ # API group = parser.add_argument_group('API') group.add_argument('--api', action='store_true', help='Enable the API extension.') -group.add_argument('--api-disable-ipv6', type=bool, default=False, help='Disables IPv6 for the api') -group.add_argument('--api-disable-ipv4', type=bool, default=False, help='Disables IPv4 for the api') +group.add_argument('--api-disable-ipv6', action='store_true', help='Disables IPv6 for the api') +group.add_argument('--api-disable-ipv4', action='store_true', help='Disables IPv4 for the api') group.add_argument('--public-api', action='store_true', help='Create a public URL for the API using Cloudfare.') group.add_argument('--public-api-id', type=str, help='Tunnel ID for named Cloudflare Tunnel. Use together with public-api option.', default=None) group.add_argument('--api-port', type=int, default=5000, help='The listening port for the API.') From 741736abbebf378e96ad622667762b08e64199d5 Mon Sep 17 00:00:00 2001 From: BPplays Date: Fri, 6 Dec 2024 04:36:19 -0800 Subject: [PATCH 4/9] changed wording and some code formatting --- extensions/openai/script.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 608e0a497f..420bce2eba 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -357,16 +357,19 @@ async def handle_unload_loras(): def run_server(): server_addrV6 = '[::]' if shared.args.listen else '[::1]' server_addrV4 = '0.0.0.0' if shared.args.listen else '127.0.0.1' + port = int(os.environ.get('OPENEDAI_PORT', shared.args.api_port)) - server_addr = [] + server_addrs = [] disable_ipv6 = os.environ.get('OPENEDAI_DISABLE_IPV6', shared.args.api_disable_ipv6) if not disable_ipv6: - server_addr.append(server_addrV6) + server_addrs.append(server_addrV6) + disable_ipv4 = os.environ.get('OPENEDAI_DISABLE_IPV4', shared.args.api_disable_ipv4) if not disable_ipv4: - server_addr.append(server_addrV4) + server_addrs.append(server_addrV4) + ssl_certfile = os.environ.get('OPENEDAI_CERT_PATH', shared.args.ssl_certfile) ssl_keyfile = os.environ.get('OPENEDAI_KEY_PATH', shared.args.ssl_keyfile) @@ -376,9 +379,6 @@ def run_server(): if ssl_keyfile and ssl_certfile: url_proto = 'https://' - url_s = "" - if len(server_addr) > 1: - url_s = "s" if shared.args.public_api: @@ -387,10 +387,10 @@ def on_start(public_url: str): _start_cloudflared(port, shared.args.public_api_id, max_attempts=3, on_start=on_start) else: - logger.info(f'\n\nOpenAI-compatible API URL{url_s}:') - for i, addr in enumerate(server_addr): + logger.info(f'\n\nRunning OpenAI-compatible API on:') + for i, addr in enumerate(server_addrs): nl = '' - if i == len(server_addr) - 1: + if i == len(server_addrs) - 1: nl = '\n\n' logger.info(f'{url_proto}{addr}:{port}{nl}') @@ -405,7 +405,7 @@ def on_start(public_url: str): logger.info(f'OpenAI API admin key (for loading/unloading models):\n\n{shared.args.admin_key}\n') logging.getLogger("uvicorn.error").propagate = False - uvicorn.run(app, host=server_addr, port=port, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile) + uvicorn.run(app, host=server_addrs, port=port, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile) def setup(): From acd697ea39c9686a5e7e4c299e41cd630ff84356 Mon Sep 17 00:00:00 2001 From: BPplays Date: Fri, 6 Dec 2024 04:40:52 -0800 Subject: [PATCH 5/9] linting --- extensions/openai/script.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 420bce2eba..504622bab6 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -352,8 +352,6 @@ async def handle_unload_loras(): return JSONResponse(content="OK") - - def run_server(): server_addrV6 = '[::]' if shared.args.listen else '[::1]' server_addrV4 = '0.0.0.0' if shared.args.listen else '127.0.0.1' @@ -365,29 +363,24 @@ def run_server(): if not disable_ipv6: server_addrs.append(server_addrV6) - disable_ipv4 = os.environ.get('OPENEDAI_DISABLE_IPV4', shared.args.api_disable_ipv4) if not disable_ipv4: server_addrs.append(server_addrV4) - ssl_certfile = os.environ.get('OPENEDAI_CERT_PATH', shared.args.ssl_certfile) ssl_keyfile = os.environ.get('OPENEDAI_KEY_PATH', shared.args.ssl_keyfile) - url_proto = 'http://' if ssl_keyfile and ssl_certfile: url_proto = 'https://' - - if shared.args.public_api: def on_start(public_url: str): logger.info(f'OpenAI-compatible API URL:\n\n{public_url}\n') _start_cloudflared(port, shared.args.public_api_id, max_attempts=3, on_start=on_start) else: - logger.info(f'\n\nRunning OpenAI-compatible API on:') + logger.info('\n\nRunning OpenAI-compatible API on:') for i, addr in enumerate(server_addrs): nl = '' if i == len(server_addrs) - 1: From aac37320a46740ba5820cf011ccfb528493f9d5a Mon Sep 17 00:00:00 2001 From: BPplays Date: Fri, 6 Dec 2024 04:46:16 -0800 Subject: [PATCH 6/9] fixed help wording --- modules/shared.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index a79a6bd6bf..341d9dc497 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -195,8 +195,8 @@ # API group = parser.add_argument_group('API') group.add_argument('--api', action='store_true', help='Enable the API extension.') -group.add_argument('--api-disable-ipv6', action='store_true', help='Disables IPv6 for the api') -group.add_argument('--api-disable-ipv4', action='store_true', help='Disables IPv4 for the api') +group.add_argument('--api-disable-ipv6', action='store_true', help='Disables IPv6 for the API') +group.add_argument('--api-disable-ipv4', action='store_true', help='Disables IPv4 for the API') group.add_argument('--public-api', action='store_true', help='Create a public URL for the API using Cloudfare.') group.add_argument('--public-api-id', type=str, help='Tunnel ID for named Cloudflare Tunnel. Use together with public-api option.', default=None) group.add_argument('--api-port', type=int, default=5000, help='The listening port for the API.') From c64eb0a58a52b1752db9cf912ec251f0609ac8d4 Mon Sep 17 00:00:00 2001 From: BPplays Date: Fri, 6 Dec 2024 04:46:47 -0800 Subject: [PATCH 7/9] fixed help wording --- modules/shared.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index 341d9dc497..39a290a192 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -195,8 +195,8 @@ # API group = parser.add_argument_group('API') group.add_argument('--api', action='store_true', help='Enable the API extension.') -group.add_argument('--api-disable-ipv6', action='store_true', help='Disables IPv6 for the API') -group.add_argument('--api-disable-ipv4', action='store_true', help='Disables IPv4 for the API') +group.add_argument('--api-disable-ipv6', action='store_true', help='Disable IPv6 for the API') +group.add_argument('--api-disable-ipv4', action='store_true', help='Disable IPv4 for the API') group.add_argument('--public-api', action='store_true', help='Create a public URL for the API using Cloudfare.') group.add_argument('--public-api-id', type=str, help='Tunnel ID for named Cloudflare Tunnel. Use together with public-api option.', default=None) group.add_argument('--api-port', type=int, default=5000, help='The listening port for the API.') From bdbd656d4a1c06b6f0df604e8fcef525abbdbcd1 Mon Sep 17 00:00:00 2001 From: BPplays Date: Fri, 6 Dec 2024 04:55:03 -0800 Subject: [PATCH 8/9] fixed broken arg condition --- extensions/openai/script.py | 4 ++++ modules/shared.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 504622bab6..6938eb26a8 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -367,6 +367,10 @@ def run_server(): if not disable_ipv4: server_addrs.append(server_addrV4) + if len(server_addrs) < 1: + logger.error('you MUST enable IPv6 or IPv4 for the API to work') + raise Exception('you MUST enable IPv6 or IPv4 for the API to work') + ssl_certfile = os.environ.get('OPENEDAI_CERT_PATH', shared.args.ssl_certfile) ssl_keyfile = os.environ.get('OPENEDAI_KEY_PATH', shared.args.ssl_keyfile) diff --git a/modules/shared.py b/modules/shared.py index 39a290a192..6e26b4976c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -9,6 +9,9 @@ from modules.logging_colors import logger +class BrokenArgs(Exception): + pass + # Model variables model = None tokenizer = None From 0a8170ed1953b2a31340d656735c3e0516ee20ef Mon Sep 17 00:00:00 2001 From: BPplays Date: Fri, 6 Dec 2024 04:55:49 -0800 Subject: [PATCH 9/9] removed trash --- modules/shared.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index 6e26b4976c..39a290a192 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -9,9 +9,6 @@ from modules.logging_colors import logger -class BrokenArgs(Exception): - pass - # Model variables model = None tokenizer = None