Merge branch 'master' into xtra-scorers

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 requires-python = ">=3.9"
 dynamic = ["version"]
 dependencies = [
-  "pydantic>=2.0.0",
+  "pydantic>=2.0.0,<2.10.0", # Pinning to resolve issues caused in 2.10.0 release
   "wandb>=0.17.1",
   "packaging>=21.0",         # For version parsing in integrations
   "tenacity>=8.3.0,!=8.4.0", # Excluding 8.4.0 because it had a bug on import of AsyncRetrying
@@ -43,6 +43,7 @@ dependencies = [
   "numpy>1.21.0",            # Used in box.py and scorer.py (should be made optional)
   "rich",                    # Used for special formatting of tables (should be made optional)
   "gql[aiohttp,requests]",   # Used exclusively in wandb_api.py
+  "jsonschema>=4.23.0",      # Used by scorers for field validation
 ]
 
 [project.optional-dependencies]
@@ -93,7 +94,6 @@ test = [
   "clickhouse_connect==0.7.0",
   "fastapi>=0.110.0",
   "sqlparse==0.5.0",
-  "jsonschema>=4.23.0",
 
   # Integration Tests
   "pytest-recording>=0.13.2",
@@ -225,7 +225,7 @@ module = "weave_query.*"
 ignore_errors = true
 
 [tool.bumpversion]
-current_version = "0.51.20-dev0"
+current_version = "0.51.22-dev0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

tests/trace/test_annotation_feedback.py

@@ -10,8 +10,8 @@ def test_human_feedback_basic(client):
         description="A numerical field with a range of -1 to 1",
         json_schema={
             "type": "number",
-            "min": -1,
-            "max": 1,
+            "minimum": -1,
+            "maximum": 1,
         },
         unique_among_creators=True,
         op_scope=None,
@@ -49,8 +49,8 @@ def test_human_feedback_basic(client):
     assert objects.objs[1].val["op_scope"] == ["weave:///entity/project/op/name:digest"]
     assert objects.objs[0].val["json_schema"] == {
         "type": "number",
-        "min": -1,
-        "max": 1,
+        "minimum": -1,
+        "maximum": 1,
     }
     assert objects.objs[1].val["json_schema"] == {
         "type": "string",

wb_schema.gql

@@ -169,6 +169,7 @@ type User implements Node {
   photoUrl: String
   deletedAt: DateTime
   teams(before: String, after: String, first: Int, last: Int): EntityConnection
+  admin: Boolean
 }
 
 type UserConnection {

weave-js/src/common/hooks/useViewerInfo.ts

@@ -11,6 +11,7 @@ const VIEWER_QUERY = gql`
     viewer {
       id
       username
+      admin
       teams {
         edges {
           node {
@@ -28,6 +29,7 @@ type UserInfo = {
   id: string;
   username: string;
   teams: string[];
+  admin: boolean;
 };
 type UserInfoResponseLoading = {
   loading: true;
@@ -71,6 +73,7 @@ export const useViewerInfo = (): UserInfoResponse => {
           id,
           username,
           teams,
+          admin: userInfo.admin,
         },
       });
     });

weave-js/src/components/PagePanelComponents/Home/Browse2/Browse2RootObjectVersionOutputOf.tsx

@@ -1,11 +1,11 @@
 import Typography from '@mui/material/Typography';
-import {useNodeValue} from '@wandb/weave/react';
+import {parseRefMaybe, useNodeValue} from '@wandb/weave/react';
 import React, {FC, useMemo} from 'react';
 import {useParams} from 'react-router-dom';
 
 import {callsTableFilter, callsTableNode, callsTableOpCounts} from './callTree';
 import {Browse2RootObjectVersionItemParams} from './CommonLib';
-import {parseRefMaybe, SmallRef} from './SmallRef';
+import {SmallRef} from './SmallRef';
 
 export const Browse2RootObjectVersionOutputOf: FC<{uri: string}> = ({uri}) => {
   const params = useParams<Browse2RootObjectVersionItemParams>();

weave-js/src/components/PagePanelComponents/Home/Browse2/CellValue.tsx

@@ -19,7 +19,6 @@ import {SmallRef} from './SmallRef';
 
 type CellValueProps = {
   value: any;
-  isExpanded?: boolean;
 };
 
 const Collapsed = styled.div<{hasScrolling: boolean}>`
@@ -32,15 +31,15 @@ const Collapsed = styled.div<{hasScrolling: boolean}>`
 `;
 Collapsed.displayName = 'S.Collapsed';
 
-export const CellValue = ({value, isExpanded = false}: CellValueProps) => {
+export const CellValue = ({value}: CellValueProps) => {
   if (value === undefined) {
     return null;
   }
   if (value === null) {
     return <ValueViewPrimitive>null</ValueViewPrimitive>;
   }
   if (isWeaveRef(value)) {
-    return <SmallRef objRef={parseRef(value)} iconOnly={isExpanded} />;
+    return <SmallRef objRef={parseRef(value)} />;
   }
   if (typeof value === 'boolean') {
     return (

weave-js/src/components/PagePanelComponents/Home/Browse2/SmallRef.tsx

@@ -4,7 +4,6 @@ import {
   isWandbArtifactRef,
   isWeaveObjectRef,
   ObjectRef,
-  parseRef,
   refUri,
 } from '@wandb/weave/react';
 import React, {FC} from 'react';
@@ -201,11 +200,3 @@ export const SmallRef: FC<{
     </Link>
   );
 };
-
-export const parseRefMaybe = (s: string): ObjectRef | null => {
-  try {
-    return parseRef(s);
-  } catch (e) {
-    return null;
-  }
-};

weave-js/src/components/PagePanelComponents/Home/Browse2/SpanDetails.tsx

@@ -4,6 +4,7 @@ import * as globals from '@wandb/weave/common/css/globals.styles';
 import * as _ from 'lodash';
 import React, {FC} from 'react';
 
+import {parseRefMaybe} from '../../../../react';
 import {StatusChip} from '../Browse3/pages/common/StatusChip';
 import {Call} from './callTree';
 import {DisplayControlChars} from './CommonLib';
@@ -13,7 +14,7 @@ import {
   OpenAIChatInputView,
   OpenAIChatOutputView,
 } from './openai';
-import {parseRefMaybe, SmallRef} from './SmallRef';
+import {SmallRef} from './SmallRef';
 
 const ObjectView: FC<{obj: any}> = ({obj}) => {
   if (_.isPlainObject(obj)) {

weave-js/src/components/PagePanelComponents/Home/Browse3.tsx

@@ -73,7 +73,6 @@ import {CallPage} from './Browse3/pages/CallPage/CallPage';
 import {CallsPage} from './Browse3/pages/CallsPage/CallsPage';
 import {
   ALWAYS_PIN_LEFT_CALLS,
-  DEFAULT_COLUMN_VISIBILITY_CALLS,
   DEFAULT_FILTER_CALLS,
   DEFAULT_PIN_CALLS,
   DEFAULT_SORT_CALLS,
@@ -736,7 +735,7 @@ const CallsPageBinding = () => {
     try {
       return JSON.parse(query.cols);
     } catch (e) {
-      return DEFAULT_COLUMN_VISIBILITY_CALLS;
+      return {};
     }
   }, [query.cols]);
   const setColumnVisibilityModel = (newModel: GridColumnVisibilityModel) => {

weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx

@@ -336,7 +336,8 @@ export const browse3ContextGen = (
       traceId: string,
       callId: string,
       path?: string | null,
-      tracetree?: boolean
+      tracetree?: boolean,
+      feedbackExpand?: boolean
     ) => {
       let url = `${projectRoot(entityName, projectName)}/calls/${callId}`;
       const params = new URLSearchParams();
@@ -346,6 +347,9 @@ export const browse3ContextGen = (
       if (tracetree !== undefined) {
         params.set(TRACETREE_PARAM, tracetree ? '1' : '0');
       }
+      if (feedbackExpand !== undefined) {
+        params.set(FEEDBACK_EXPAND_PARAM, feedbackExpand ? '1' : '0');
+      }
       if (params.toString()) {
         url += '?' + params.toString();
       }
@@ -497,7 +501,8 @@ type RouteType = {
     traceId: string,
     callId: string,
     path?: string | null,
-    tracetree?: boolean
+    tracetree?: boolean,
+    feedbackExpand?: boolean
   ) => string;
   tracesUIUrl: (entityName: string, projectName: string) => string;
   callsUIUrl: (
@@ -564,6 +569,7 @@ const useSetSearchParam = () => {
 
 export const PEEK_PARAM = 'peekPath';
 export const TRACETREE_PARAM = 'tracetree';
+export const FEEDBACK_EXPAND_PARAM = 'feedbackExpand';
 export const PATH_PARAM = 'path';
 
 export const baseContext = browse3ContextGen(

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/FeedbackGrid.tsx

@@ -1,16 +1,20 @@
 import {Box} from '@mui/material';
 import _ from 'lodash';
-import React, {useEffect} from 'react';
+import React, {useEffect, useMemo} from 'react';
 
 import {useViewerInfo} from '../../../../../common/hooks/useViewerInfo';
 import {TargetBlank} from '../../../../../common/util/links';
 import {Alert} from '../../../../Alert';
 import {Loading} from '../../../../Loading';
 import {Tailwind} from '../../../../Tailwind';
+import {RUNNABLE_FEEDBACK_TYPE_PREFIX} from '../pages/CallPage/CallScoresViewer';
 import {Empty} from '../pages/common/Empty';
 import {useWFHooks} from '../pages/wfReactInterface/context';
 import {useGetTraceServerClientContext} from '../pages/wfReactInterface/traceServerClientContext';
 import {FeedbackGridInner} from './FeedbackGridInner';
+import {HUMAN_ANNOTATION_BASE_TYPE} from './StructuredFeedback/humanAnnotationTypes';
+
+const ANNOTATION_PREFIX = `${HUMAN_ANNOTATION_BASE_TYPE}.`;
 
 type FeedbackGridProps = {
   entity: string;
@@ -40,6 +44,38 @@ export const FeedbackGrid = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
+  // Group by feedback on this object vs. descendent objects
+  const grouped = useMemo(() => {
+    // Exclude runnables as they are presented in a different tab
+    const withoutRunnables = (query.result ?? []).filter(
+      f => !f.feedback_type.startsWith(RUNNABLE_FEEDBACK_TYPE_PREFIX)
+    );
+    // Combine annotation feedback on (feedback_type, creator)
+    const combined = _.groupBy(
+      withoutRunnables.filter(f =>
+        f.feedback_type.startsWith(ANNOTATION_PREFIX)
+      ),
+      f => `${f.feedback_type}-${f.creator}`
+    );
+    // only keep the most recent feedback for each (feedback_type, creator)
+    const combinedFiltered = Object.values(combined).map(
+      fs => fs.sort((a, b) => b.created_at - a.created_at)[0]
+    );
+    // add the non-annotation feedback to the combined object
+    combinedFiltered.push(
+      ...withoutRunnables.filter(
+        f => !f.feedback_type.startsWith(ANNOTATION_PREFIX)
+      )
+    );
+
+    // Group by feedback on this object vs. descendent objects
+    return _.groupBy(combinedFiltered, f =>
+      f.weave_ref.substring(weaveRef.length)
+    );
+  }, [query.result, weaveRef]);
+
+  const paths = useMemo(() => Object.keys(grouped).sort(), [grouped]);
+
   if (query.loading || loadingUserInfo) {
     return (
       <Box
@@ -61,7 +97,7 @@ export const FeedbackGrid = ({
     );
   }
 
-  if (!query.result || !query.result.length) {
+  if (!paths.length) {
     return (
       <Empty
         size="small"
@@ -81,12 +117,6 @@ export const FeedbackGrid = ({
     );
   }
 
-  // Group by feedback on this object vs. descendent objects
-  const grouped = _.groupBy(query.result, f =>
-    f.weave_ref.substring(weaveRef.length)
-  );
-  const paths = Object.keys(grouped).sort();
-
   const currentViewerId = userInfo ? userInfo.id : null;
   return (
     <Tailwind>

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/FeedbackGridInner.tsx

@@ -42,6 +42,13 @@ export const FeedbackGridInner = ({
         if (params.row.feedback_type === 'wandb.reaction.1') {
           return params.row.payload.emoji;
         }
+        if (params.row.feedback_type.startsWith('wandb.annotation.')) {
+          return (
+            <CellValueString
+              value={JSON.stringify(params.row.payload.value ?? null)}
+            />
+          );
+        }
         return <CellValueString value={JSON.stringify(params.row.payload)} />;
       },
     },

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/FeedbackTypeChip.tsx

@@ -15,6 +15,9 @@ export const FeedbackTypeChip = ({feedbackType}: FeedbackTypeChipProps) => {
   } else if (feedbackType === 'wandb.note.1') {
     color = 'gold';
     label = 'Note';
+  } else if (feedbackType.includes('wandb.annotation.')) {
+    color = 'magenta';
+    label = 'Annotation';
   }
   return <Pill color={color} label={label} />;
 };

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/HumanFeedback/tsHumanFeedback.ts

@@ -0,0 +1,59 @@
+export type FeedbackTypeParts = {
+  field: string;
+  userDefinedType: string;
+  feedbackType: string;
+  displayName: string;
+};
+
+export const parseFeedbackType = (
+  inputField: string
+): FeedbackTypeParts | null => {
+  // input: summary.weave.feedback.wandb.annotation.Numerical-field-2.payload.value
+  // or input: wandb.annotation.Numerical-field-2.payload.value
+  // or input: feedback.[wandb.annotation.Numerical-field-2].payload.value
+  //
+  // output:
+  // field: wandb.annotation.Numerical-field-2
+  // userDefinedType: Numerical-field-2
+  // type: annotation
+  // displayName: Annotation.Numerical-field-2
+
+  // If the field is coming from the flattened table, remove the
+  // summary portion
+  const field = inputField.startsWith('summary.weave.feedback')
+    ? inputField.replace('summary.weave.feedback.', '')
+    : inputField;
+  const deBracketed = field.replace(/\[.*\]/g, '');
+  const split = deBracketed.split('.');
+  if (split.length !== 5) {
+    return null;
+  }
+  const [w, type, userDefinedType, p, v] = split;
+
+  if (v !== 'value') {
+    throw new Error(`Expected 'value' prefix, got '${v}'`);
+  }
+  if (p !== 'payload') {
+    throw new Error(`Expected 'payload' prefix, got '${p}'`);
+  }
+  if (w !== 'wandb') {
+    return null;
+  }
+  return {
+    field,
+    feedbackType: type,
+    userDefinedType,
+    displayName: `${
+      type.charAt(0).toUpperCase() + type.slice(1)
+    }.${userDefinedType}`,
+  };
+};
+
+export const convertFeedbackFieldToBackendFilter = (field: string): string => {
+  const parsed = parseFeedbackType(field);
+  if (parsed === null) {
+    return field;
+  }
+  const {feedbackType, userDefinedType} = parsed;
+  return `feedback.[wandb.${feedbackType}.${userDefinedType}].payload.value`;
+};