From a00732de80e3b519a688a737ecb14850531bed58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20K=C3=B6hnecke?= Date: Mon, 16 Dec 2024 09:44:54 +0100 Subject: [PATCH] fix: clean up studio notebooks to reuse example and make them execution order independent --- src/documentation/how_tos/example_data.py | 2 ++ .../how_to_aggregate_evaluations.ipynb | 4 ++-- .../studio/how_to_execute_a_benchmark.ipynb | 20 +++++++++---------- ...o_upload_existing_datasets_to_studio.ipynb | 8 +++++--- .../how_to_use_studio_with_traces.ipynb | 16 +++++++-------- 5 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/documentation/how_tos/example_data.py b/src/documentation/how_tos/example_data.py index 281f3cd6f..9434641c3 100644 --- a/src/documentation/how_tos/example_data.py +++ b/src/documentation/how_tos/example_data.py @@ -112,6 +112,7 @@ class ExampleData: run_overview_2: RunOverview evaluation_overview_1: EvaluationOverview evaluation_overview_2: EvaluationOverview + studio_project_name: str def example_data() -> ExampleData: @@ -159,6 +160,7 @@ def example_data() -> ExampleData: example_data.run_overview_2 = run_overview_2 example_data.evaluation_overview_1 = evaluation_overview_1 example_data.evaluation_overview_2 = evaluation_overview_2 + example_data.studio_project_name = "My Example Project" return example_data diff --git a/src/documentation/how_tos/how_to_aggregate_evaluations.ipynb b/src/documentation/how_tos/how_to_aggregate_evaluations.ipynb index 873861633..b64462376 100644 --- a/src/documentation/how_tos/how_to_aggregate_evaluations.ipynb +++ b/src/documentation/how_tos/how_to_aggregate_evaluations.ipynb @@ -70,7 +70,7 @@ ], "metadata": { "kernelspec": { - "display_name": "intelligence-layer-aL2cXmJM-py3.11", + "display_name": "intelligence-layer-LP3DLT23-py3.12", "language": "python", "name": "python3" }, @@ -84,7 +84,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/src/documentation/how_tos/studio/how_to_execute_a_benchmark.ipynb b/src/documentation/how_tos/studio/how_to_execute_a_benchmark.ipynb index 5d96397f0..4b1b25e4a 100644 --- a/src/documentation/how_tos/studio/how_to_execute_a_benchmark.ipynb +++ b/src/documentation/how_tos/studio/how_to_execute_a_benchmark.ipynb @@ -9,11 +9,10 @@ "from dotenv import load_dotenv\n", "\n", "from documentation.how_tos.example_data import (\n", - " EXAMPLE_1_INPUT,\n", " DummyAggregationLogic,\n", " DummyEvaluationLogic,\n", - " DummyExample,\n", " DummyTask,\n", + " example_data,\n", ")\n", "from intelligence_layer.connectors.studio.studio import StudioClient\n", "from intelligence_layer.evaluation.benchmark.studio_benchmark import (\n", @@ -24,13 +23,8 @@ ")\n", "\n", "load_dotenv()\n", - "\n", - "examples = [\n", - " DummyExample(input=\"input0\", expected_output=\"expected_output0\", data=\"data0\"),\n", - " DummyExample(\n", - " input=EXAMPLE_1_INPUT, expected_output=\"expected_output1\", data=\"data1\"\n", - " ),\n", - "]" + "my_example_data = example_data()\n", + "examples = my_example_data.examples" ] }, { @@ -69,7 +63,11 @@ "outputs": [], "source": [ "# Step 0\n", - "studio_client = StudioClient(project=\"my project_name\", create_project=True)\n", + "from uuid import uuid4\n", + "\n", + "studio_client = StudioClient(\n", + " project=my_example_data.studio_project_name, create_project=True\n", + ")\n", "\n", "# Step 1\n", "studio_dataset_repository = StudioDatasetRepository(studio_client)\n", @@ -80,7 +78,7 @@ "evaluation_logic = DummyEvaluationLogic()\n", "aggregation_logic = DummyAggregationLogic()\n", "benchmark = studio_benchmark_repository.create_benchmark(\n", - " dataset.id, evaluation_logic, aggregation_logic, \"my_benchmark\"\n", + " dataset.id, evaluation_logic, aggregation_logic, f\"my_benchmark-{uuid4()}\"\n", ")\n", "\n", "# Step 3\n", diff --git a/src/documentation/how_tos/studio/how_to_upload_existing_datasets_to_studio.ipynb b/src/documentation/how_tos/studio/how_to_upload_existing_datasets_to_studio.ipynb index d3bebc674..f6971f785 100644 --- a/src/documentation/how_tos/studio/how_to_upload_existing_datasets_to_studio.ipynb +++ b/src/documentation/how_tos/studio/how_to_upload_existing_datasets_to_studio.ipynb @@ -60,7 +60,9 @@ ")\n", "\n", "# Step 1\n", - "studio_client = StudioClient(project=\"my project_name\")\n", + "studio_client = StudioClient(\n", + " project=my_example_data.studio_project_name, create_project=True\n", + ")\n", "\n", "# Step 2\n", "studio_dataset_repo = StudioDatasetRepository(studio_client=studio_client)\n", @@ -76,7 +78,7 @@ ], "metadata": { "kernelspec": { - "display_name": "intelligence-layer-aL2cXmJM-py3.11", + "display_name": "intelligence-layer-LP3DLT23-py3.12", "language": "python", "name": "python3" }, @@ -90,7 +92,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/src/documentation/how_tos/studio/how_to_use_studio_with_traces.ipynb b/src/documentation/how_tos/studio/how_to_use_studio_with_traces.ipynb index 5f7ed9450..405bbe14c 100644 --- a/src/documentation/how_tos/studio/how_to_use_studio_with_traces.ipynb +++ b/src/documentation/how_tos/studio/how_to_use_studio_with_traces.ipynb @@ -6,8 +6,11 @@ "metadata": {}, "outputs": [], "source": [ + "from documentation.how_tos.example_data import DummyTask, example_data\n", "from intelligence_layer.connectors import StudioClient\n", - "from intelligence_layer.core import InMemoryTracer, Task, TaskSpan" + "from intelligence_layer.core import InMemoryTracer\n", + "\n", + "my_example_data = example_data()" ] }, { @@ -45,16 +48,13 @@ "outputs": [], "source": [ "# Step 0\n", - "class DummyTask(Task[str, str]):\n", - " def do_run(self, input: str, task_span: TaskSpan) -> str:\n", - " return f\"{input} -> output\"\n", - "\n", - "\n", "tracer = InMemoryTracer()\n", "DummyTask().run(\"My Dummy Run\", tracer=tracer)\n", "\n", "# Step 1\n", - "studio_client = StudioClient(project=\"my project_name\", create_project=True)\n", + "studio_client = StudioClient(\n", + " project=my_example_data.studio_project_name, create_project=True\n", + ")\n", "\n", "# Step 2.1\n", "trace_to_submit = tracer.export_for_viewing()\n", @@ -70,7 +70,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "intelligence-layer-LP3DLT23-py3.12", "language": "python", "name": "python3" },