feat: add ontology <-> json translation utililty and object cropping (#…

…20)

* refactor: printable exceptions and wrapper for formating them

* perf(tasks.runner): only initialise label rows when needed

* feat: storage item lookup for easy access to eg client metadata

* docs: make doc string more readable

* feat(dependency): storage item

Note could be done faster by bulk loading storage items directly via the runner rather than doing it once for each task.

* feat(dependency): storage item for gcp

* fix: only load gcp label row if needed

* feat(dependency): better doc strings for gcp and fast api

* feat: ontology to pydantic and back

* add tests for ontology and back

* fix: ruff

* dep: pytest

* test: test for single object nested classifications.

* feat: add object crop dependency to gcp agents

* fix: make test command correspond to new FE approach

* feat: allow changing client domain via env variables

You can set `ENCORD_DOMAIN='https://api.encord.com'`

* fix: use form data in fastapi examples and dependencies

* fix: use form data in gcp agents

* chore: get rid of old cli commands that are not currently maintained

* fix: remove todos and better prints

* fix: better settings parsing and let printable error just take care of formatting on its own

* refactor: type object crops

* fix: typo

* docs: object classification with claude

docs/code_examples/gcp/object_classification.py

@@ -0,0 +1,85 @@
+import os
+
+from anthropic import Anthropic
+from encord.objects.ontology_labels_impl import LabelRowV2
+from encord_agents.core.ontology import OntologyDataModel
+from encord_agents.core.utils import get_user_client
+from encord_agents.gcp import Depends, editor_agent
+from encord_agents.gcp.dependencies import FrameData, InstanceCrop, dep_object_crops
+from typing_extensions import Annotated
+
+# User client
+client = get_user_client()
+project = client.get_project("<project_hash>")
+generic_ont_obj, *other_objects = sorted(
+    project.ontology_structure.objects,
+    key=lambda o: o.title.lower() == "generic",
+    reverse=True,
+)
+
+# Data model
+data_model = OntologyDataModel(other_objects)
+system_prompt = f"""
+You're a helpful assistant that's supposed to help fill in 
+json objects according to this schema:
+
+`{data_model.model_json_schema_str}`
+
+Please only respond with valid json.
+"""
+
+# Prompts
+ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY")
+anthropic_client = Anthropic(api_key=ANTHROPIC_API_KEY)
+
+
+# Setup agent
+@editor_agent()
+def agent(
+    frame_data: FrameData,
+    lr: LabelRowV2,
+    crops: Annotated[
+        list[InstanceCrop],
+        Depends(dep_object_crops(filter_ontology_objects=[generic_ont_obj])),
+    ],
+):
+    # Query Claude
+    changes = False
+    for crop in crops:
+        message = anthropic_client.messages.create(
+            model="claude-3-5-sonnet-20241022",
+            max_tokens=1024,
+            system=system_prompt,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [crop.b64_encoding(output_format="anthropic")],
+                }
+            ],
+        )
+
+        # Parse result
+        try:
+            instance = data_model(message.content[0].text)
+
+            coordinates = crop.instance.get_annotation(
+                frame=frame_data.frame
+            ).coordinates
+            instance.set_for_frames(
+                coordinates=coordinates,
+                frames=frame_data.frame,
+                confidence=0.5,
+                manual_annotation=False,
+            )
+            lr.remove_object(crop.instance)
+            lr.add_object_instance(instance)
+            changes = True
+        except Exception:
+            import traceback
+
+            traceback.print_exc()
+            print(f"Response from model: {message.content[0].text}")
+
+    # Save changes
+    if changes:
+        lr.save()