fix: clean up studio notebooks to reuse example and make them executi…

…on order independent
Aleph-Alpha · Dec 16, 2024 · a00732d · a00732d
1 parent 31e5afb
commit a00732d
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 24 deletions.
diff --git a/src/documentation/how_tos/example_data.py b/src/documentation/how_tos/example_data.py
@@ -112,6 +112,7 @@ class ExampleData:
     run_overview_2: RunOverview
     evaluation_overview_1: EvaluationOverview
     evaluation_overview_2: EvaluationOverview
+    studio_project_name: str
 
 
 def example_data() -> ExampleData:
@@ -159,6 +160,7 @@ def example_data() -> ExampleData:
     example_data.run_overview_2 = run_overview_2
     example_data.evaluation_overview_1 = evaluation_overview_1
     example_data.evaluation_overview_2 = evaluation_overview_2
+    example_data.studio_project_name = "My Example Project"
 
     return example_data
 

diff --git a/src/documentation/how_tos/how_to_aggregate_evaluations.ipynb b/src/documentation/how_tos/how_to_aggregate_evaluations.ipynb
@@ -70,7 +70,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "intelligence-layer-aL2cXmJM-py3.11",
+   "display_name": "intelligence-layer-LP3DLT23-py3.12",
    "language": "python",
    "name": "python3"
   },
@@ -84,7 +84,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.12.2"
   }
  },
  "nbformat": 4,

diff --git a/src/documentation/how_tos/studio/how_to_execute_a_benchmark.ipynb b/src/documentation/how_tos/studio/how_to_execute_a_benchmark.ipynb
@@ -9,11 +9,10 @@
     "from dotenv import load_dotenv\n",
     "\n",
     "from documentation.how_tos.example_data import (\n",
-    "    EXAMPLE_1_INPUT,\n",
     "    DummyAggregationLogic,\n",
     "    DummyEvaluationLogic,\n",
-    "    DummyExample,\n",
     "    DummyTask,\n",
+    "    example_data,\n",
     ")\n",
     "from intelligence_layer.connectors.studio.studio import StudioClient\n",
     "from intelligence_layer.evaluation.benchmark.studio_benchmark import (\n",
@@ -24,13 +23,8 @@
     ")\n",
     "\n",
     "load_dotenv()\n",
-    "\n",
-    "examples = [\n",
-    "    DummyExample(input=\"input0\", expected_output=\"expected_output0\", data=\"data0\"),\n",
-    "    DummyExample(\n",
-    "        input=EXAMPLE_1_INPUT, expected_output=\"expected_output1\", data=\"data1\"\n",
-    "    ),\n",
-    "]"
+    "my_example_data = example_data()\n",
+    "examples = my_example_data.examples"
    ]
   },
   {
@@ -69,7 +63,11 @@
    "outputs": [],
    "source": [
     "# Step 0\n",
-    "studio_client = StudioClient(project=\"my project_name\", create_project=True)\n",
+    "from uuid import uuid4\n",
+    "\n",
+    "studio_client = StudioClient(\n",
+    "    project=my_example_data.studio_project_name, create_project=True\n",
+    ")\n",
     "\n",
     "# Step 1\n",
     "studio_dataset_repository = StudioDatasetRepository(studio_client)\n",
@@ -80,7 +78,7 @@
     "evaluation_logic = DummyEvaluationLogic()\n",
     "aggregation_logic = DummyAggregationLogic()\n",
     "benchmark = studio_benchmark_repository.create_benchmark(\n",
-    "    dataset.id, evaluation_logic, aggregation_logic, \"my_benchmark\"\n",
+    "    dataset.id, evaluation_logic, aggregation_logic, f\"my_benchmark-{uuid4()}\"\n",
     ")\n",
     "\n",
     "# Step 3\n",

diff --git a/src/documentation/how_tos/studio/how_to_upload_existing_datasets_to_studio.ipynb b/src/documentation/how_tos/studio/how_to_upload_existing_datasets_to_studio.ipynb
@@ -60,7 +60,9 @@
     ")\n",
     "\n",
     "# Step 1\n",
-    "studio_client = StudioClient(project=\"my project_name\")\n",
+    "studio_client = StudioClient(\n",
+    "    project=my_example_data.studio_project_name, create_project=True\n",
+    ")\n",
     "\n",
     "# Step 2\n",
     "studio_dataset_repo = StudioDatasetRepository(studio_client=studio_client)\n",
@@ -76,7 +78,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "intelligence-layer-aL2cXmJM-py3.11",
+   "display_name": "intelligence-layer-LP3DLT23-py3.12",
    "language": "python",
    "name": "python3"
   },
@@ -90,7 +92,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.12.2"
   }
  },
  "nbformat": 4,

diff --git a/src/documentation/how_tos/studio/how_to_use_studio_with_traces.ipynb b/src/documentation/how_tos/studio/how_to_use_studio_with_traces.ipynb
@@ -6,8 +6,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from documentation.how_tos.example_data import DummyTask, example_data\n",
     "from intelligence_layer.connectors import StudioClient\n",
-    "from intelligence_layer.core import InMemoryTracer, Task, TaskSpan"
+    "from intelligence_layer.core import InMemoryTracer\n",
+    "\n",
+    "my_example_data = example_data()"
    ]
   },
   {
@@ -45,16 +48,13 @@
    "outputs": [],
    "source": [
     "# Step 0\n",
-    "class DummyTask(Task[str, str]):\n",
-    "    def do_run(self, input: str, task_span: TaskSpan) -> str:\n",
-    "        return f\"{input} -> output\"\n",
-    "\n",
-    "\n",
     "tracer = InMemoryTracer()\n",
     "DummyTask().run(\"My Dummy Run\", tracer=tracer)\n",
     "\n",
     "# Step 1\n",
-    "studio_client = StudioClient(project=\"my project_name\", create_project=True)\n",
+    "studio_client = StudioClient(\n",
+    "    project=my_example_data.studio_project_name, create_project=True\n",
+    ")\n",
     "\n",
     "# Step 2.1\n",
     "trace_to_submit = tracer.export_for_viewing()\n",
@@ -70,7 +70,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv",
+   "display_name": "intelligence-layer-LP3DLT23-py3.12",
    "language": "python",
    "name": "python3"
   },