From 849451fa13ee99331fea5f262af8267f01b22742 Mon Sep 17 00:00:00 2001 From: Kyle Goyette Date: Wed, 30 Oct 2024 14:01:03 -0700 Subject: [PATCH] add model providers file, modify makefile, update server code --- weave/Makefile | 3 ++ .../clickhouse_trace_server_batched.py | 7 +--- .../model_providers/model_providers.json | 1 + .../model_providers/model_providers.py | 37 +++++++++++++++---- 4 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 weave/trace_server/model_providers/model_providers.json diff --git a/weave/Makefile b/weave/Makefile index 77943c2d9cb..2a9ae350dd6 100644 --- a/weave/Makefile +++ b/weave/Makefile @@ -1,2 +1,5 @@ update_costs: python trace_server/costs/update_costs.py + +update_model_providers: + python trace_server/model_providers/model_providers.py diff --git a/weave/trace_server/clickhouse_trace_server_batched.py b/weave/trace_server/clickhouse_trace_server_batched.py index 9a1f1a1303f..128c51369ac 100644 --- a/weave/trace_server/clickhouse_trace_server_batched.py +++ b/weave/trace_server/clickhouse_trace_server_batched.py @@ -95,8 +95,7 @@ ) from weave.trace_server.llm_completion import lite_llm_completion from weave.trace_server.model_providers.model_providers import ( - MODEL_PROVIDERS_FILE, - fetch_model_to_provider_info_map, + read_model_to_provider_info_map, ) from weave.trace_server.orm import ParamBuilder, Row from weave.trace_server.secret_fetcher_context import _secret_fetcher_context @@ -207,9 +206,7 @@ def __init__( self._flush_immediately = True self._call_batch: list[list[Any]] = [] self._use_async_insert = use_async_insert - self._model_to_provider_info_map = fetch_model_to_provider_info_map( - MODEL_PROVIDERS_FILE - ) + self._model_to_provider_info_map = read_model_to_provider_info_map() @classmethod def from_env(cls, use_async_insert: bool = False) -> "ClickHouseTraceServer": diff --git a/weave/trace_server/model_providers/model_providers.json b/weave/trace_server/model_providers/model_providers.json new file mode 100644 index 00000000000..5349ff3625a --- /dev/null +++ b/weave/trace_server/model_providers/model_providers.json @@ -0,0 +1 @@ +{"gpt-4": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4o": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4o-audio-preview": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4o-audio-preview-2024-10-01": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4o-mini": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4o-mini-2024-07-18": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "o1-mini": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "o1-mini-2024-09-12": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "o1-preview": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "o1-preview-2024-09-12": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "chatgpt-4o-latest": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4o-2024-05-13": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4o-2024-08-06": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-turbo-preview": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-0314": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-0613": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-32k": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-32k-0314": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-32k-0613": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-turbo": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-turbo-2024-04-09": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-1106-preview": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-0125-preview": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-vision-preview": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-4-1106-vision-preview": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-3.5-turbo": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-3.5-turbo-0301": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-3.5-turbo-0613": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-3.5-turbo-1106": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-3.5-turbo-0125": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-3.5-turbo-16k": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "gpt-3.5-turbo-16k-0613": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "ft:gpt-3.5-turbo": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "ft:gpt-3.5-turbo-0125": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "ft:gpt-3.5-turbo-1106": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "ft:gpt-3.5-turbo-0613": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "ft:gpt-4-0613": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "ft:gpt-4o-2024-08-06": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "ft:gpt-4o-mini-2024-07-18": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "text-embedding-3-large": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "text-embedding-3-small": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "text-embedding-ada-002": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "text-embedding-ada-002-v2": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "text-moderation-stable": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "text-moderation-007": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "text-moderation-latest": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "256-x-256/dall-e-2": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "512-x-512/dall-e-2": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "1024-x-1024/dall-e-2": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "hd/1024-x-1792/dall-e-3": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "hd/1792-x-1024/dall-e-3": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "hd/1024-x-1024/dall-e-3": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "standard/1024-x-1792/dall-e-3": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "standard/1792-x-1024/dall-e-3": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "standard/1024-x-1024/dall-e-3": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "whisper-1": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "tts-1": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "tts-1-hd": {"litellm_provider": "openai", "api_key_name": "OPENAI_API_KEY"}, "claude-instant-1": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}, "groq/llama2-70b-4096": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/llama3-8b-8192": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/llama3-70b-8192": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/llama-3.1-8b-instant": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/llama-3.1-70b-versatile": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/llama-3.1-405b-reasoning": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/mixtral-8x7b-32768": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/gemma-7b-it": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/gemma2-9b-it": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/llama3-groq-70b-8192-tool-use-preview": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "groq/llama3-groq-8b-8192-tool-use-preview": {"litellm_provider": "groq", "api_key_name": "GEMMA_API_KEY"}, "claude-instant-1.2": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}, "claude-2": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}, "claude-2.1": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}, "claude-3-haiku-20240307": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}, "claude-3-opus-20240229": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}, "claude-3-sonnet-20240229": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}, "claude-3-5-sonnet-20240620": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}, "claude-3-5-sonnet-20241022": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}, "gemini/gemini-1.5-flash-002": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-flash-001": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-flash": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-flash-latest": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-flash-8b-exp-0924": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-flash-exp-0827": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-flash-8b-exp-0827": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-pro": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-pro": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-pro-002": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-pro-001": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-pro-exp-0801": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-pro-exp-0827": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-1.5-pro-latest": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-pro-vision": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-gemma-2-27b-it": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "gemini/gemini-gemma-2-9b-it": {"litellm_provider": "gemini", "api_key_name": "GOOGLE_API_KEY"}, "anthropic/claude-3-5-sonnet-20241022": {"litellm_provider": "anthropic", "api_key_name": "ANTHROPIC_API_KEY"}} \ No newline at end of file diff --git a/weave/trace_server/model_providers/model_providers.py b/weave/trace_server/model_providers/model_providers.py index 3e7e8d01d76..c690bd37069 100644 --- a/weave/trace_server/model_providers/model_providers.py +++ b/weave/trace_server/model_providers/model_providers.py @@ -1,11 +1,12 @@ import json +import logging import os from typing import Dict, TypedDict import requests model_providers_url = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" -MODEL_PROVIDERS_FILE = "/tmp/model_providers/model_providers.json" +MODEL_PROVIDER_INFO_FILE = "model_providers.json" PROVIDER_TO_API_KEY_NAME_MAP = { @@ -22,12 +23,27 @@ class LLMModelProviderInfo(TypedDict): api_key_name: str -def fetch_model_to_provider_info_map( - cached_file_path: str = MODEL_PROVIDERS_FILE, +logger = logging.getLogger(__name__) + + +def read_model_to_provider_info_map( + file_name: str = MODEL_PROVIDER_INFO_FILE, ) -> Dict[str, LLMModelProviderInfo]: - if os.path.exists(cached_file_path): - with open(cached_file_path, "r") as f: + full_path = os.path.join(os.path.dirname(__file__), file_name) + try: + with open(full_path, "r") as f: return json.load(f) + except Exception as e: + logger.error( + f"Failed to read model to provider info file at: {full_path}", exc_info=e + ) + return {} + + +def main( + file_name: str = MODEL_PROVIDER_INFO_FILE, +) -> Dict[str, LLMModelProviderInfo]: + full_path = os.path.join(os.path.dirname(__file__), file_name) try: req = requests.get(model_providers_url) req.raise_for_status() @@ -43,7 +59,14 @@ def fetch_model_to_provider_info_map( providers[k] = LLMModelProviderInfo( litellm_provider=provider, api_key_name=api_key_name ) - os.makedirs(os.path.dirname(cached_file_path), exist_ok=True) - with open(cached_file_path, "w") as f: + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, "w") as f: json.dump(providers, f) + print( + f"Updated model to model provider info file at: {full_path}. {len(providers)} models updated." + ) return providers + + +if __name__ == "__main__": + main()