From 588012f25efa1759a83dff42f815cce87f4c43d5 Mon Sep 17 00:00:00 2001 From: Tim Sweeney Date: Tue, 17 Dec 2024 11:03:27 -0800 Subject: [PATCH] chore(weave): Fix notebook docs: OpenAI and model_output #3274 --- docs/intro_notebook.ipynb | 19 ++++++++++++++++--- docs/notebooks/audio_with_weave.ipynb | 13 +++++++++++++ docs/notebooks/chain_of_density.ipynb | 5 ++--- docs/notebooks/codegen.ipynb | 17 +++++++++++++++-- docs/notebooks/dspy_prompt_optimization.ipynb | 5 ++--- docs/notebooks/feedback_prod.ipynb | 14 +++++++++++++- .../multi-agent-structured-output.ipynb | 13 +++++++++++++ .../notebooks/notdiamond_custom_routing.ipynb | 2 +- examples/tutorial_scripts/05_eval_pipeline.py | 4 ++-- .../tutorial_scripts/06_eval_pipeline_all.py | 4 ++-- ...nai_proxy_quickstart_enterprise_mode.ipynb | 13 +++++++++++++ 11 files changed, 92 insertions(+), 17 deletions(-) diff --git a/docs/intro_notebook.ipynb b/docs/intro_notebook.ipynb index d21cc81336f..ecf9e612ec4 100644 --- a/docs/intro_notebook.ipynb +++ b/docs/intro_notebook.ipynb @@ -11,7 +11,6 @@ "---\n", "docusaurus_head_meta::end -->\n", "\n", - "" ] }, @@ -54,6 +53,20 @@ "!pip install weave openai set-env-colab-kaggle-dotenv" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e188ab8", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "# Temporary workaround to fix bug in openai:\n", + "# TypeError: Client.__init__() got an unexpected keyword argument 'proxies'\n", + "# See https://community.openai.com/t/error-with-openai-1-56-0-client-init-got-an-unexpected-keyword-argument-proxies/1040332/15\n", + "!pip install \"httpx<0.28\"" + ] + }, { "cell_type": "code", "execution_count": null, @@ -586,9 +599,9 @@ "\n", "# Define any custom scoring function\n", "@weave.op()\n", - "def exact_match(expected: str, model_output: dict) -> dict:\n", + "def exact_match(expected: str, output: dict) -> dict:\n", " # Here is where you'd define the logic to score the model output\n", - " return {\"match\": expected == model_output}\n", + " return {\"match\": expected == output}\n", "\n", "\n", "# Score your examples using scoring functions\n", diff --git a/docs/notebooks/audio_with_weave.ipynb b/docs/notebooks/audio_with_weave.ipynb index 9cf2526144a..881b39a5d8f 100644 --- a/docs/notebooks/audio_with_weave.ipynb +++ b/docs/notebooks/audio_with_weave.ipynb @@ -55,6 +55,19 @@ "!pip install set-env-colab-kaggle-dotenv -q # for env var" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "# Temporary workaround to fix bug in openai:\n", + "# TypeError: Client.__init__() got an unexpected keyword argument 'proxies'\n", + "# See https://community.openai.com/t/error-with-openai-1-56-0-client-init-got-an-unexpected-keyword-argument-proxies/1040332/15\n", + "!pip install \"httpx<0.28\"" + ] + }, { "cell_type": "markdown", "metadata": { diff --git a/docs/notebooks/chain_of_density.ipynb b/docs/notebooks/chain_of_density.ipynb index 9b6869d8c3a..acf969d531b 100644 --- a/docs/notebooks/chain_of_density.ipynb +++ b/docs/notebooks/chain_of_density.ipynb @@ -10,7 +10,6 @@ "---\n", "docusaurus_head_meta::end -->\n", "\n", - "" ] }, @@ -502,8 +501,8 @@ "source": [ "# Define the scorer function\n", "@weave.op()\n", - "def quality_scorer(instruction: str, model_output: dict) -> dict:\n", - " result = evaluate_summary(model_output[\"final_summary\"], instruction)\n", + "def quality_scorer(instruction: str, output: dict) -> dict:\n", + " result = evaluate_summary(output[\"final_summary\"], instruction)\n", " return result" ] }, diff --git a/docs/notebooks/codegen.ipynb b/docs/notebooks/codegen.ipynb index e354686d556..90c7239d930 100644 --- a/docs/notebooks/codegen.ipynb +++ b/docs/notebooks/codegen.ipynb @@ -67,6 +67,19 @@ "!pip install -qU autopep8 autoflake weave isort openai set-env-colab-kaggle-dotenv datasets" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "# Temporary workaround to fix bug in openai:\n", + "# TypeError: Client.__init__() got an unexpected keyword argument 'proxies'\n", + "# See https://community.openai.com/t/error-with-openai-1-56-0-client-init-got-an-unexpected-keyword-argument-proxies/1040332/15\n", + "!pip install \"httpx<0.28\"" + ] + }, { "cell_type": "code", "execution_count": null, @@ -383,8 +396,8 @@ "outputs": [], "source": [ "@weave.op()\n", - "async def score_humaneval_test(test: str, entry_point: str, model_output: str):\n", - " generated_code = model_output\n", + "async def score_humaneval_test(test: str, entry_point: str, output: str):\n", + " generated_code = output\n", "\n", " # Extract test cases from the test string\n", " test_cases = re.findall(r\"assert.*\", test)\n", diff --git a/docs/notebooks/dspy_prompt_optimization.ipynb b/docs/notebooks/dspy_prompt_optimization.ipynb index 74eba9f390f..14afde3f634 100644 --- a/docs/notebooks/dspy_prompt_optimization.ipynb +++ b/docs/notebooks/dspy_prompt_optimization.ipynb @@ -10,7 +10,6 @@ "---\n", "docusaurus_head_meta::end -->\n", "\n", - "" ] }, @@ -300,8 +299,8 @@ "outputs": [], "source": [ "@weave.op()\n", - "def weave_evaluation_scorer(answer: str, model_output: Output) -> dict:\n", - " return {\"match\": int(answer.lower() == model_output[\"answer\"].lower())}" + "def weave_evaluation_scorer(answer: str, output: Output) -> dict:\n", + " return {\"match\": int(answer.lower() == output[\"answer\"].lower())}" ] }, { diff --git a/docs/notebooks/feedback_prod.ipynb b/docs/notebooks/feedback_prod.ipynb index fad44ddd310..7cdcaf71634 100644 --- a/docs/notebooks/feedback_prod.ipynb +++ b/docs/notebooks/feedback_prod.ipynb @@ -10,7 +10,6 @@ "---\n", "docusaurus_head_meta::end -->\n", "\n", - "" ] }, @@ -49,6 +48,19 @@ "!pip install weave openai streamlit" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "# Temporary workaround to fix bug in openai:\n", + "# TypeError: Client.__init__() got an unexpected keyword argument 'proxies'\n", + "# See https://community.openai.com/t/error-with-openai-1-56-0-client-init-got-an-unexpected-keyword-argument-proxies/1040332/15\n", + "!pip install \"httpx<0.28\"" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/notebooks/multi-agent-structured-output.ipynb b/docs/notebooks/multi-agent-structured-output.ipynb index 0db085fead4..41ea08dbd51 100644 --- a/docs/notebooks/multi-agent-structured-output.ipynb +++ b/docs/notebooks/multi-agent-structured-output.ipynb @@ -63,6 +63,19 @@ "!pip install -qU openai weave wandb" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "# Temporary workaround to fix bug in openai:\n", + "# TypeError: Client.__init__() got an unexpected keyword argument 'proxies'\n", + "# See https://community.openai.com/t/error-with-openai-1-56-0-client-init-got-an-unexpected-keyword-argument-proxies/1040332/15\n", + "!pip install \"httpx<0.28\"" + ] + }, { "cell_type": "markdown", "metadata": { diff --git a/docs/notebooks/notdiamond_custom_routing.ipynb b/docs/notebooks/notdiamond_custom_routing.ipynb index 4a6f52a2d89..8ae3041b69a 100644 --- a/docs/notebooks/notdiamond_custom_routing.ipynb +++ b/docs/notebooks/notdiamond_custom_routing.ipynb @@ -251,7 +251,7 @@ "outputs": [], "source": [ "@weave.op()\n", - "def is_correct(score: int, model_output: dict) -> dict:\n", + "def is_correct(score: int, output: dict) -> dict:\n", " # We hack score, since we already have model responses\n", " return {\"correct\": score}\n", "\n", diff --git a/examples/tutorial_scripts/05_eval_pipeline.py b/examples/tutorial_scripts/05_eval_pipeline.py index 0a6a5baf9ab..a4f2296b44a 100644 --- a/examples/tutorial_scripts/05_eval_pipeline.py +++ b/examples/tutorial_scripts/05_eval_pipeline.py @@ -64,8 +64,8 @@ async def predict(self, sentence: str) -> dict: @weave.op() -def fruit_name_score(target: dict, model_output: dict) -> dict: - return {"correct": target["fruit"] == model_output["fruit"]} +def fruit_name_score(target: dict, output: dict) -> dict: + return {"correct": target["fruit"] == output["fruit"]} evaluation = weave.Evaluation( diff --git a/examples/tutorial_scripts/06_eval_pipeline_all.py b/examples/tutorial_scripts/06_eval_pipeline_all.py index 0d5fe8fd3b2..8d3ccce430f 100644 --- a/examples/tutorial_scripts/06_eval_pipeline_all.py +++ b/examples/tutorial_scripts/06_eval_pipeline_all.py @@ -65,8 +65,8 @@ async def predict(self, sentence: str) -> dict: # We define a scoring functions to compare our model predictions with a ground truth label. @weave.op() -def fruit_name_score(target: dict, model_output: dict) -> dict: - return {"correct": target["fruit"] == model_output["fruit"]} +def fruit_name_score(target: dict, output: dict) -> dict: + return {"correct": target["fruit"] == output["fruit"]} # Finally, we run an evaluation of this model. diff --git a/weave_query/examples/prompts/llm_monitoring/openai_proxy_quickstart_enterprise_mode.ipynb b/weave_query/examples/prompts/llm_monitoring/openai_proxy_quickstart_enterprise_mode.ipynb index b2c1cda36cb..e44c24274d7 100644 --- a/weave_query/examples/prompts/llm_monitoring/openai_proxy_quickstart_enterprise_mode.ipynb +++ b/weave_query/examples/prompts/llm_monitoring/openai_proxy_quickstart_enterprise_mode.ipynb @@ -25,6 +25,19 @@ "!pip install -qqq openai" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "# Temporary workaround to fix bug in openai:\n", + "# TypeError: Client.__init__() got an unexpected keyword argument 'proxies'\n", + "# See https://community.openai.com/t/error-with-openai-1-56-0-client-init-got-an-unexpected-keyword-argument-proxies/1040332/15\n", + "!pip install \"httpx<0.28\"" + ] + }, { "cell_type": "markdown", "metadata": {},