diff --git a/dev_docs/BaseObjectClasses.md b/dev_docs/BaseObjectClasses.md index a571af49755..20a104e2e90 100644 --- a/dev_docs/BaseObjectClasses.md +++ b/dev_docs/BaseObjectClasses.md @@ -116,7 +116,7 @@ curl -X POST 'https://trace.wandb.ai/obj/create' \ "project_id": "user/project", "object_id": "my_config", "val": {...}, - "set_base_object_class": "MyConfig" + "set_leaf_object_class": "MyConfig" } }' @@ -162,7 +162,7 @@ Run `make synchronize-base-object-schemas` to ensure the frontend TypeScript typ 4. Now, each use case uses different parts: 1. `Python Writing`. Users can directly import these classes and use them as normal Pydantic models, which get published with `weave.publish`. The python client correct builds the requisite payload. 2. `Python Reading`. Users can `weave.ref().get()` and the weave python SDK will return the instance with the correct type. Note: we do some special handling such that the returned object is not a WeaveObject, but literally the exact pydantic class. - 3. `HTTP Writing`. In cases where the client/user does not want to add the special type information, users can publish base objects by setting the `set_base_object_class` setting on `POST obj/create` to the name of the class. The weave server will validate the object against the schema, update the metadata fields, and store the object. + 3. `HTTP Writing`. In cases where the client/user does not want to add the special type information, users can publish objects by setting the `set_leaf_object_class` setting on `POST obj/create` to the name of the class. The weave server will validate the object against the schema, update the metadata fields, and store the object. 4. `HTTP Reading`. When querying for objects, the server will return the object with the correct type if the `base_object_class` metadata field is set. 5. `Frontend`. The frontend will read the zod schema from `weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts` and use that to provide compile time type safety when using `useBaseObjectInstances` and runtime type safety when using `useCreateBaseObjectInstance`. * Note: it is critical that all techniques produce the same digest for the same data - which is tested in the tests. This way versions are not thrashed by different clients/users. @@ -185,7 +185,7 @@ graph TD subgraph "Trace Server" subgraph "HTTP API" - R --> |validates using| HW["POST obj/create
set_base_object_class"] + R --> |validates using| HW["POST obj/create
set_leaf_object_class"] HW --> DB[(Weave Object Store)] HR["POST objs/query
base_object_classes"] --> |Filters base_object_class| DB end @@ -203,7 +203,7 @@ graph TD Z --> |import| UBI["useBaseObjectInstances"] Z --> |import| UCI["useCreateBaseObjectInstance"] UBI --> |Filters base_object_class| HR - UCI --> |set_base_object_class| HW + UCI --> |set_leaf_object_class| HW UI[React UI] --> UBI UI --> UCI end diff --git a/tests/trace/test_base_object_classes.py b/tests/trace/test_base_object_classes.py index a264941f7b0..98b5dd8458e 100644 --- a/tests/trace/test_base_object_classes.py +++ b/tests/trace/test_base_object_classes.py @@ -139,7 +139,7 @@ def test_interface_creation(client): "project_id": client._project_id(), "object_id": nested_obj_id, "val": nested_obj.model_dump(), - "set_base_object_class": "TestOnlyNestedBaseObject", + "set_leaf_object_class": "TestOnlyNestedBaseObject", } } ) @@ -164,7 +164,7 @@ def test_interface_creation(client): "project_id": client._project_id(), "object_id": top_level_obj_id, "val": top_obj.model_dump(), - "set_base_object_class": "TestOnlyExample", + "set_leaf_object_class": "TestOnlyExample", } } ) @@ -271,7 +271,7 @@ def test_digest_equality(client): "project_id": client._project_id(), "object_id": nested_obj_id, "val": nested_obj.model_dump(), - "set_base_object_class": "TestOnlyNestedBaseObject", + "set_leaf_object_class": "TestOnlyNestedBaseObject", } } ) @@ -300,7 +300,7 @@ def test_digest_equality(client): "project_id": client._project_id(), "object_id": top_level_obj_id, "val": top_obj.model_dump(), - "set_base_object_class": "TestOnlyExample", + "set_leaf_object_class": "TestOnlyExample", } } ) @@ -322,7 +322,7 @@ def test_schema_validation(client): "object_id": "nested_obj", # Incorrect schema, should raise! "val": {"a": 2}, - "set_base_object_class": "TestOnlyNestedBaseObject", + "set_leaf_object_class": "TestOnlyNestedBaseObject", } } ) @@ -340,7 +340,7 @@ def test_schema_validation(client): "_class_name": "TestOnlyNestedBaseObject", "_bases": ["BaseObject", "BaseModel"], }, - "set_base_object_class": "TestOnlyNestedBaseObject", + "set_leaf_object_class": "TestOnlyNestedBaseObject", } } ) @@ -359,7 +359,7 @@ def test_schema_validation(client): "_class_name": "TestOnlyNestedBaseObject", "_bases": ["BaseObject", "BaseModel"], }, - "set_base_object_class": "TestOnlyExample", + "set_leaf_object_class": "TestOnlyExample", } } ) diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallScoresViewer.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallScoresViewer.tsx index 0b4e9374a26..e27f8d06117 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallScoresViewer.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallScoresViewer.tsx @@ -13,7 +13,7 @@ import {NotApplicable} from '../../../Browse2/NotApplicable'; import {SmallRef} from '../../../Browse2/SmallRef'; import {StyledDataGrid} from '../../StyledDataGrid'; // Import the StyledDataGrid component import { - TraceObjSchemaForBaseObjectClass, + TraceObjSchemaForObjectClass, useBaseObjectInstances, } from '../wfReactInterface/baseObjectClassQuery'; import {WEAVE_REF_SCHEME} from '../wfReactInterface/constants'; @@ -61,7 +61,7 @@ const useRunnableFeedbacksForCall = (call: CallSchema) => { const useRunnableFeedbackTypeToLatestActionRef = ( call: CallSchema, - actionSpecs: Array> + actionSpecs: Array> ): Record => { return useMemo(() => { return _.fromPairs( @@ -92,7 +92,7 @@ type GroupedRowType = { }; const useTableRowsForRunnableFeedbacks = ( - actionSpecs: Array>, + actionSpecs: Array>, runnableFeedbacks: Feedback[], runnableFeedbackTypeToLatestActionRef: Record ): GroupedRowType[] => { diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx index 52d6795806c..4d3d37caa57 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx @@ -12,7 +12,7 @@ import {SimplePageLayout} from '../common/SimplePageLayout'; import {ObjectVersionsTable} from '../ObjectVersionsPage'; import { useBaseObjectInstances, - useCreateBaseObjectInstance, + useCreateLeafObjectInstance, } from '../wfReactInterface/baseObjectClassQuery'; import {sanitizeObjectId} from '../wfReactInterface/traceServerDirectClient'; import { @@ -162,7 +162,7 @@ const generateLeaderboardId = () => { }; const useCreateLeaderboard = (entity: string, project: string) => { - const createLeaderboardInstance = useCreateBaseObjectInstance('Leaderboard'); + const createLeaderboardInstance = useCreateLeafObjectInstance('Leaderboard'); const createLeaderboard = async () => { const objectId = sanitizeObjectId(generateLeaderboardId()); diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx index 6fac8eaa599..fa1881462a0 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx @@ -26,7 +26,7 @@ import {LeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardCon import {SimplePageLayout} from '../common/SimplePageLayout'; import { useBaseObjectInstances, - useCreateBaseObjectInstance, + useCreateLeafObjectInstance, } from '../wfReactInterface/baseObjectClassQuery'; import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks'; import {LeaderboardConfigEditor} from './LeaderboardConfigEditor'; @@ -131,7 +131,7 @@ const useUpdateLeaderboard = ( project: string, objectId: string ) => { - const createLeaderboard = useCreateBaseObjectInstance('Leaderboard'); + const createLeaderboard = useCreateLeafObjectInstance('Leaderboard'); const updateLeaderboard = async (leaderboardVal: LeaderboardObjectVal) => { return await createLeaderboard({ diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/AnnotationScorerForm.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/AnnotationScorerForm.tsx index 9acbdfe6c2f..051797c79d0 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/AnnotationScorerForm.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/AnnotationScorerForm.tsx @@ -2,7 +2,7 @@ import {Box} from '@material-ui/core'; import React, {FC, useCallback, useState} from 'react'; import {z} from 'zod'; -import {createBaseObjectInstance} from '../wfReactInterface/baseObjectClassQuery'; +import {createLeafObjectInstance} from '../wfReactInterface/baseObjectClassQuery'; import {TraceServerClient} from '../wfReactInterface/traceServerClient'; import {sanitizeObjectId} from '../wfReactInterface/traceServerDirectClient'; import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks'; @@ -83,7 +83,7 @@ export const onAnnotationScorerSave = async ( ) => { const jsonSchemaType = convertTypeToJsonSchemaType(data.Type.type); const typeExtras = convertTypeExtrasToJsonSchema(data); - return createBaseObjectInstance(client, 'AnnotationSpec', { + return createLeafObjectInstance(client, 'AnnotationSpec', { obj: { project_id: projectIdFromParts({entity, project}), object_id: sanitizeObjectId(data.Name), diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/LLMJudgeScorerForm.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/LLMJudgeScorerForm.tsx index 64823e9d551..31195a5cb10 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/LLMJudgeScorerForm.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/LLMJudgeScorerForm.tsx @@ -11,7 +11,7 @@ import React, {FC, useCallback, useState} from 'react'; import {z} from 'zod'; import {LlmJudgeActionSpecSchema} from '../wfReactInterface/baseObjectClasses.zod'; -import {createBaseObjectInstance} from '../wfReactInterface/baseObjectClassQuery'; +import {createLeafObjectInstance} from '../wfReactInterface/baseObjectClassQuery'; import {ActionSpecSchema} from '../wfReactInterface/generatedBaseObjectClasses.zod'; import {TraceServerClient} from '../wfReactInterface/traceServerClient'; import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks'; @@ -185,7 +185,7 @@ export const onLLMJudgeScorerSave = async ( config: judgeAction, }); - return createBaseObjectInstance(client, 'ActionSpec', { + return createLeafObjectInstance(client, 'ActionSpec', { obj: { project_id: projectIdFromParts({entity, project}), object_id: objectId, diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts index 9918ae7f285..f61f56afa7d 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts @@ -2,7 +2,7 @@ import {expectType} from 'tsd'; import { useBaseObjectInstances, - useCreateBaseObjectInstance, + useCreateLeafObjectInstance, } from './baseObjectClassQuery'; import { TestOnlyExample, @@ -74,7 +74,7 @@ describe('Type Tests', () => { it('useCreateCollectionObject return type matches expected structure', () => { type CreateCollectionObjectReturn = ReturnType< - typeof useCreateBaseObjectInstance<'TestOnlyExample'> + typeof useCreateLeafObjectInstance<'TestOnlyExample'> >; // Define the expected type structure diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts index 6ceb39daa70..9125217f614 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts @@ -13,22 +13,22 @@ import { } from './traceServerClientTypes'; import {Loadable} from './wfDataModelHooksInterface'; -type BaseObjectClassRegistry = typeof baseObjectClassRegistry; -type BaseObjectClassRegistryKeys = keyof BaseObjectClassRegistry; -type BaseObjectClassType = z.infer< - BaseObjectClassRegistry[C] +type ObjectClassRegistry = typeof baseObjectClassRegistry; // TODO: Add more here - not just bases! +type ObjectClassRegistryKeys = keyof ObjectClassRegistry; +type ObjectClassType = z.infer< + ObjectClassRegistry[C] >; -export type TraceObjSchemaForBaseObjectClass< - C extends BaseObjectClassRegistryKeys -> = TraceObjSchema, C>; +export type TraceObjSchemaForObjectClass< + C extends ObjectClassRegistryKeys +> = TraceObjSchema, C>; -export const useBaseObjectInstances = ( +export const useBaseObjectInstances = ( baseObjectClassName: C, req: TraceObjQueryReq -): Loadable>> => { +): Loadable>> => { const [objects, setObjects] = useState< - Array> + Array> >([]); const getTsClient = useGetTraceServerClientContext(); const client = getTsClient(); @@ -56,11 +56,11 @@ export const useBaseObjectInstances = ( return {result: objects, loading}; }; -const getBaseObjectInstances = async ( +const getBaseObjectInstances = async ( client: TraceServerClient, baseObjectClassName: C, req: TraceObjQueryReq -): Promise, C>>> => { +): Promise, C>>> => { const knownObjectClass = baseObjectClassRegistry[baseObjectClassName]; if (!knownObjectClass) { console.warn(`Unknown object class: ${baseObjectClassName}`); @@ -86,47 +86,47 @@ const getBaseObjectInstances = async ( .map( ({obj, parsed}) => ({...obj, val: parsed.data} as TraceObjSchema< - BaseObjectClassType, + ObjectClassType, C >) ); }; -export const useCreateBaseObjectInstance = < - C extends BaseObjectClassRegistryKeys, - T = BaseObjectClassType +export const useCreateLeafObjectInstance = < + C extends ObjectClassRegistryKeys, + T = ObjectClassType >( - baseObjectClassName: C + leafObjectClassName: C ): ((req: TraceObjCreateReq) => Promise) => { const getTsClient = useGetTraceServerClientContext(); const client = getTsClient(); return (req: TraceObjCreateReq) => - createBaseObjectInstance(client, baseObjectClassName, req); + createLeafObjectInstance(client, leafObjectClassName, req); }; -export const createBaseObjectInstance = async < - C extends BaseObjectClassRegistryKeys, - T = BaseObjectClassType +export const createLeafObjectInstance = async < + C extends ObjectClassRegistryKeys, + T = ObjectClassType >( client: TraceServerClient, - baseObjectClassName: C, + leafObjectClassName: C, req: TraceObjCreateReq ): Promise => { if ( - req.obj.set_base_object_class != null && - req.obj.set_base_object_class !== baseObjectClassName + req.obj.set_leaf_object_class != null && + req.obj.set_leaf_object_class !== leafObjectClassName ) { throw new Error( - `set_base_object_class must match baseObjectClassName: ${baseObjectClassName}` + `set_leaf_object_class must match leafObjectClassName: ${leafObjectClassName}` ); } - const knownBaseObjectClass = baseObjectClassRegistry[baseObjectClassName]; - if (!knownBaseObjectClass) { - throw new Error(`Unknown object class: ${baseObjectClassName}`); + const knownObjectClass = baseObjectClassRegistry[leafObjectClassName]; + if (!knownObjectClass) { + throw new Error(`Unknown object class: ${leafObjectClassName}`); } - const verifiedObject = knownBaseObjectClass.safeParse(req.obj.val); + const verifiedObject = knownObjectClass.safeParse(req.obj.val); if (!verifiedObject.success) { throw new Error( @@ -134,13 +134,13 @@ export const createBaseObjectInstance = async < ); } - const reqWithBaseObjectClass: TraceObjCreateReq = { + const reqWithLeafObjectClass: TraceObjCreateReq = { ...req, obj: { ...req.obj, - set_base_object_class: baseObjectClassName, + set_leaf_object_class: leafObjectClassName, }, }; - return client.objCreate(reqWithBaseObjectClass); + return client.objCreate(reqWithLeafObjectClass); }; diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts index c396962f0fb..520926532c1 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts @@ -243,7 +243,7 @@ export type TraceObjCreateReq = { project_id: string; object_id: string; val: T; - set_base_object_class?: string; + set_leaf_object_class?: string; }; }; diff --git a/weave/trace_server/base_object_class_util.py b/weave/trace_server/base_object_class_util.py index 1c52f766c0c..81aa3b10d1b 100644 --- a/weave/trace_server/base_object_class_util.py +++ b/weave/trace_server/base_object_class_util.py @@ -29,65 +29,76 @@ def get_base_object_class(val: Any) -> Optional[str]: return None +def get_leaf_object_class(val: Any) -> Optional[str]: + if isinstance(val, dict): + if "_bases" in val: + if isinstance(val["_bases"], list): + if len(val["_bases"]) >= 2: + if val["_bases"][-1] == "BaseModel": + if val["_bases"][-2] in base_object_class_names: + if "_class_name" in val: + return val["_class_name"] + return None + + def process_incoming_object_val( - val: Any, req_base_object_class: Optional[str] = None + val: Any, req_leaf_object_class: Optional[str] = None ) -> tuple[dict, Optional[str]]: """ This method is responsible for accepting an incoming object from the user, validating it - against the base object class, and returning the object with the base object class + against the leaf object class, and returning the object with the base object class set. It does not mutate the original object, but returns a new object with values set if needed. Specifically,: 1. If the object is not a dict, it is returned as is, and the base object class is set to None. - 2. There are 2 ways to specify the base object class: - a. The `req_base_object_class` argument. + 2. There are 2 ways to specify the leaf object class: + a. The `req_leaf_object_class` argument. * used by non-pythonic writers of weave objects b. The `_bases` & `_class_name` attributes of the object, which is a list of base class names. * used by pythonic weave object writers (legacy) - 3. If the object has a base object class that does not match the requested base object class, + 3. If the object has a leaf object class that does not match the requested leaf object class, an error is thrown. - 4. if the object contains a base object class inside the payload, then we simply validate - the object against the base object class (if a match is found in BASE_OBJECT_REGISTRY) - 5. If the object does not have a base object class and a requested base object class is + 4. if the object contains a leaf object class inside the payload, then we simply validate + the object against the leaf object class (if a match is found in BASE_OBJECT_REGISTRY) + 5. If the object does not have a leaf object class and a requested leaf object class is provided, we require a match in BASE_OBJECT_REGISTRY and validate the object against - the requested base object class. Finally, we set the correct feilds. + the requested leaf object class. Finally, we set the correct fields. """ if not isinstance(val, dict): - if req_base_object_class is not None: + if req_leaf_object_class is not None: raise ValueError( - "set_base_object_class cannot be provided for non-dict objects" + "set_leaf_object_class cannot be provided for non-dict objects" ) return val, None dict_val = val.copy() - val_base_object_class = get_base_object_class(dict_val) + val_leaf_object_class = get_leaf_object_class(dict_val) if ( - val_base_object_class != None - and req_base_object_class != None - and val_base_object_class != req_base_object_class + val_leaf_object_class != None + and req_leaf_object_class != None + and val_leaf_object_class != req_leaf_object_class ): raise ValueError( - f"set_base_object_class must match base_object_class: {val_base_object_class} != {req_base_object_class}" + f"set_leaf_object_class must match found leaf class: {req_leaf_object_class} != {val_leaf_object_class}" ) - if val_base_object_class is not None: + if val_leaf_object_class is not None: # In this case, we simply validate if the match is found - if base_object_class_type := BASE_OBJECT_REGISTRY.get(val_base_object_class): - base_object_class_type.model_validate(dict_val) - elif req_base_object_class is not None: + if object_class_type := BASE_OBJECT_REGISTRY.get(val_leaf_object_class): + object_class_type.model_validate(dict_val) + elif req_leaf_object_class is not None: # In this case, we require that the base object class is registered - if base_object_class_type := BASE_OBJECT_REGISTRY.get(req_base_object_class): - dict_val = dump_base_object(base_object_class_type.model_validate(dict_val)) + if object_class_type := BASE_OBJECT_REGISTRY.get(req_leaf_object_class): + dict_val = dump_base_object(object_class_type.model_validate(dict_val)) else: - raise ValueError(f"Unknown base object class: {req_base_object_class}") - - base_object_class = val_base_object_class or req_base_object_class + raise ValueError(f"Unknown base object class: {req_leaf_object_class}") - return dict_val, base_object_class + return dict_val, get_base_object_class(dict_val) +# BIG TODO: Replace this with a true object serialization step using some synthetic weave client! # Server-side version of `pydantic_object_record` def dump_base_object(val: BaseModel) -> dict: cls = val.__class__ diff --git a/weave/trace_server/clickhouse_trace_server_batched.py b/weave/trace_server/clickhouse_trace_server_batched.py index d40d7bcc2a3..10512c0c827 100644 --- a/weave/trace_server/clickhouse_trace_server_batched.py +++ b/weave/trace_server/clickhouse_trace_server_batched.py @@ -564,7 +564,7 @@ def ops_query(self, req: tsi.OpQueryReq) -> tsi.OpQueryRes: def obj_create(self, req: tsi.ObjCreateReq) -> tsi.ObjCreateRes: val, base_object_class = process_incoming_object_val( - req.obj.val, req.obj.set_base_object_class + req.obj.val, req.obj.set_leaf_object_class ) json_val = json.dumps(val) diff --git a/weave/trace_server/interface/base_object_classes/base_object_registry.py b/weave/trace_server/interface/base_object_classes/base_object_registry.py index 19eb865daea..bc1bcc6d7a7 100644 --- a/weave/trace_server/interface/base_object_classes/base_object_registry.py +++ b/weave/trace_server/interface/base_object_classes/base_object_registry.py @@ -9,6 +9,7 @@ TestOnlyNestedBaseObject, ) +# TODO: Migrate everything to normal Objects! BASE_OBJECT_REGISTRY: dict[str, type[BaseObject]] = {} diff --git a/weave/trace_server/sqlite_trace_server.py b/weave/trace_server/sqlite_trace_server.py index e14cef8041e..b68af1ac238 100644 --- a/weave/trace_server/sqlite_trace_server.py +++ b/weave/trace_server/sqlite_trace_server.py @@ -612,7 +612,7 @@ def obj_create(self, req: tsi.ObjCreateReq) -> tsi.ObjCreateRes: conn, cursor = get_conn_cursor(self.db_path) val, base_object_class = process_incoming_object_val( - req.obj.val, req.obj.set_base_object_class + req.obj.val, req.obj.set_leaf_object_class ) json_val = json.dumps(val) digest = str_digest(json_val) diff --git a/weave/trace_server/todo.md b/weave/trace_server/todo.md new file mode 100644 index 00000000000..728e73d2fac --- /dev/null +++ b/weave/trace_server/todo.md @@ -0,0 +1,18 @@ +* Finish Porting over weave client +* [ ] Add end-to-end test with new lifecycle: + * Create Dummy Model via API -> See in object explorer + * Requires finishing the leaf object registry + * Requires sub process construction & saving? + * Invoke Dummy Model via API -> See in trace explorer + * Requires execution in sub process + * Create a Dummy Scorer via API -> See in object explorer + * Should be pretty straight forward at this point + * Invoke the Scorer against the previous call via API -> see in traces AND in feedback + * Should be mostly straight forward (the Scorer API itself is a bit wonky) + * Important Proof of system: + * create the same dummy model locally & invoke -> notice no version change + * Run locally against the call -> notice that there are no extra objects +* [ ] Refactor the entire "base model" system to conform to this new way of doing things (leaf models) +* [ ] Figure out how to refactor scorers that use LLMs + * [ ] a new process with correct env setup (from secret fetcher?) + * [ ] scorers should have a client-spec, not a specific client \ No newline at end of file diff --git a/weave/trace_server/trace_server_interface.py b/weave/trace_server/trace_server_interface.py index 77ef6198ecf..42141128427 100644 --- a/weave/trace_server/trace_server_interface.py +++ b/weave/trace_server/trace_server_interface.py @@ -191,7 +191,7 @@ class ObjSchemaForInsert(BaseModel): project_id: str object_id: str val: Any - set_base_object_class: Optional[str] = None + set_leaf_object_class: Optional[str] = None class TableSchemaForInsert(BaseModel):