diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx index 4e080624ae2e..405e26ccd99d 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx @@ -73,7 +73,6 @@ import {CallPage} from './Browse3/pages/CallPage/CallPage'; import {CallsPage} from './Browse3/pages/CallsPage/CallsPage'; import { ALWAYS_PIN_LEFT_CALLS, - DEFAULT_COLUMN_VISIBILITY_CALLS, DEFAULT_FILTER_CALLS, DEFAULT_PIN_CALLS, DEFAULT_SORT_CALLS, @@ -736,7 +735,7 @@ const CallsPageBinding = () => { try { return JSON.parse(query.cols); } catch (e) { - return DEFAULT_COLUMN_VISIBILITY_CALLS; + return {}; } }, [query.cols]); const setColumnVisibilityModel = (newModel: GridColumnVisibilityModel) => { diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/HumanFeedback/tsHumanFeedback.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/HumanFeedback/tsHumanFeedback.ts new file mode 100644 index 000000000000..47f4ebe3f9fd --- /dev/null +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/HumanFeedback/tsHumanFeedback.ts @@ -0,0 +1,59 @@ +export type FeedbackTypeParts = { + field: string; + userDefinedType: string; + feedbackType: string; + displayName: string; +}; + +export const parseFeedbackType = ( + inputField: string +): FeedbackTypeParts | null => { + // input: summary.weave.feedback.wandb.annotation.Numerical-field-2.payload.value + // or input: wandb.annotation.Numerical-field-2.payload.value + // or input: feedback.[wandb.annotation.Numerical-field-2].payload.value + // + // output: + // field: wandb.annotation.Numerical-field-2 + // userDefinedType: Numerical-field-2 + // type: annotation + // displayName: Annotation.Numerical-field-2 + + // If the field is coming from the flattened table, remove the + // summary portion + const field = inputField.startsWith('summary.weave.feedback') + ? inputField.replace('summary.weave.feedback.', '') + : inputField; + const deBracketed = field.replace(/\[.*\]/g, ''); + const split = deBracketed.split('.'); + if (split.length !== 5) { + return null; + } + const [w, type, userDefinedType, p, v] = split; + + if (v !== 'value') { + throw new Error(`Expected 'value' prefix, got '${v}'`); + } + if (p !== 'payload') { + throw new Error(`Expected 'payload' prefix, got '${p}'`); + } + if (w !== 'wandb') { + return null; + } + return { + field, + feedbackType: type, + userDefinedType, + displayName: `${ + type.charAt(0).toUpperCase() + type.slice(1) + }.${userDefinedType}`, + }; +}; + +export const convertFeedbackFieldToBackendFilter = (field: string): string => { + const parsed = parseFeedbackType(field); + if (parsed === null) { + return field; + } + const {feedbackType, userDefinedType} = parsed; + return `feedback.[wandb.${feedbackType}.${userDefinedType}].payload.value`; +}; diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/filters/FilterBar.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/filters/FilterBar.tsx index d81f4694ecc5..8247d5e6ffaf 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/filters/FilterBar.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/filters/FilterBar.tsx @@ -10,6 +10,10 @@ import {Button} from '../../../../Button'; import {DraggableGrow, DraggableHandle} from '../../../../DraggablePopups'; import {IconFilterAlt} from '../../../../Icon'; import {Tailwind} from '../../../../Tailwind'; +import { + convertFeedbackFieldToBackendFilter, + parseFeedbackType, +} from '../feedback/HumanFeedback/tsHumanFeedback'; import {ColumnInfo} from '../types'; import { FIELD_DESCRIPTIONS, @@ -105,6 +109,18 @@ export const FilterBar = ({ label: (col.headerName ?? col.field).substring('attributes.'.length), description: FIELD_DESCRIPTIONS[col.field], }); + } else if ( + col.field.startsWith('summary.weave.feedback.wandb.annotation') + ) { + const parsed = parseFeedbackType(col.field); + if (!parsed) { + continue; + } + const backendFilter = convertFeedbackFieldToBackendFilter(parsed.field); + (options[0] as GroupedOption).options.push({ + value: backendFilter, + label: parsed ? parsed.displayName : col.field, + }); } else { (options[0] as GroupedOption).options.push({ value: col.field, diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/filters/common.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/filters/common.ts index a6d92bf3041f..17cfbf45ac49 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/filters/common.ts +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/filters/common.ts @@ -11,6 +11,7 @@ import { import {isWeaveObjectRef, parseRefMaybe} from '@wandb/weave/react'; import _ from 'lodash'; +import {parseFeedbackType} from '../feedback/HumanFeedback/tsHumanFeedback'; import {WEAVE_REF_PREFIX} from '../pages/wfReactInterface/constants'; import {TraceCallSchema} from '../pages/wfReactInterface/traceServerClientTypes'; @@ -40,6 +41,15 @@ export const FIELD_LABELS: Record = { }; export const getFieldLabel = (field: string): string => { + if (field.startsWith('feedback.')) { + // Here the field is coming from convertFeedbackFieldToBackendFilter + // so the field should start with 'feedback.' if feedback + const parsed = parseFeedbackType(field); + if (parsed === null) { + return field; + } + return parsed.displayName; + } return FIELD_LABELS[field] ?? field; }; diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx index b05fe855f65b..1b6a0ccbaa37 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx @@ -50,6 +50,10 @@ import { useWeaveflowCurrentRouteContext, WeaveflowPeekContext, } from '../../context'; +import { + convertFeedbackFieldToBackendFilter, + parseFeedbackType, +} from '../../feedback/HumanFeedback/tsHumanFeedback'; import {OnAddFilter} from '../../filters/CellFilterWrapper'; import {getDefaultOperatorForValue} from '../../filters/common'; import {FilterPanel} from '../../filters/FilterPanel'; @@ -100,14 +104,10 @@ import {ManageColumnsButton} from './ManageColumnsButton'; const MAX_EVAL_COMPARISONS = 5; const MAX_SELECT = 100; -export const DEFAULT_COLUMN_VISIBILITY_CALLS = { - 'attributes.weave.client_version': false, - 'attributes.weave.source': false, - 'attributes.weave.os_name': false, - 'attributes.weave.os_version': false, - 'attributes.weave.os_release': false, - 'attributes.weave.sys_version': false, -}; +export const DEFAULT_HIDDEN_COLUMN_PREFIXES = [ + 'attributes.weave', + 'summary.weave.feedback', +]; export const ALWAYS_PIN_LEFT_CALLS = ['CustomCheckbox']; @@ -519,6 +519,39 @@ export const CallsTable: FC<{ project ); + // Set default hidden columns to be hidden + useEffect(() => { + if (!setColumnVisibilityModel || !columnVisibilityModel) { + return; + } + const hiddenColumns: string[] = []; + for (const hiddenColPrefix of DEFAULT_HIDDEN_COLUMN_PREFIXES) { + const cols = columns.cols.filter(col => + col.field.startsWith(hiddenColPrefix) + ); + hiddenColumns.push(...cols.map(col => col.field)); + } + // Check if we need to update - only update if any annotation columns are missing from the model + const needsUpdate = hiddenColumns.some( + col => columnVisibilityModel[col] === undefined + ); + if (!needsUpdate) { + return; + } + const hiddenColumnVisiblityFalse = hiddenColumns.reduce((acc, col) => { + // Only add columns=false when not already in the model + if (columnVisibilityModel[col] === undefined) { + acc[col] = false; + } + return acc; + }, {} as Record); + + setColumnVisibilityModel({ + ...columnVisibilityModel, + ...hiddenColumnVisiblityFalse, + }); + }, [columns.cols, columnVisibilityModel, setColumnVisibilityModel]); + // Selection Management const [selectedCalls, setSelectedCalls] = useState([]); const clearSelectedCalls = useCallback(() => { @@ -663,6 +696,19 @@ export const CallsTable: FC<{ if (!muiColumns.some(col => col.field.startsWith('output'))) { return; } + + // handle feedback conversion from weave summary to backend filter + for (const sort of newModel) { + if (sort.field.startsWith('summary.weave.feedback')) { + const parsed = parseFeedbackType(sort.field); + if (parsed) { + const backendFilter = convertFeedbackFieldToBackendFilter( + parsed.field + ); + sort.field = backendFilter; + } + } + } setSortModel(newModel); }, [callsLoading, setSortModel, muiColumns] diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableColumns.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableColumns.tsx index 2ef528c2cf0a..29f72a4eef9c 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableColumns.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableColumns.tsx @@ -18,6 +18,10 @@ import {monthRoundedTime} from '../../../../../../common/util/time'; import {isWeaveObjectRef, parseRef} from '../../../../../../react'; import {makeRefCall} from '../../../../../../util/refs'; import {Timestamp} from '../../../../../Timestamp'; +import { + convertFeedbackFieldToBackendFilter, + parseFeedbackType, +} from '../../feedback/HumanFeedback/tsHumanFeedback'; import {Reactions} from '../../feedback/Reactions'; import {CellFilterWrapper, OnAddFilter} from '../../filters/CellFilterWrapper'; import {isWeaveRef} from '../../filters/common'; @@ -44,7 +48,11 @@ import { import {WFHighLevelCallFilter} from './callsTableFilter'; import {OpVersionIndexText} from './OpVersionIndexText'; -const HIDDEN_DYNAMIC_COLUMN_PREFIXES = ['summary.usage', 'summary.weave']; +const HIDDEN_DYNAMIC_COLUMN_PREFIXES = [ + 'summary.usage', + 'summary.weave', + 'feedback', +]; export const useCallsTableColumns = ( entity: string, @@ -327,6 +335,44 @@ function buildCallsTableColumns( ); cols.push(...newCols); + // Create special feedback columns with grouping model + const annotationColNames = allDynamicColumnNames.filter( + c => + c.startsWith('summary.weave.feedback.wandb.annotation') && + c.endsWith('payload.value') + ); + if (annotationColNames.length > 0) { + // Add feedback group to grouping model + groupingModel.push({ + groupId: 'feedback', + headerName: 'Annotations', + children: annotationColNames.map(col => ({ + field: convertFeedbackFieldToBackendFilter(col), + })), + }); + + // Add feedback columns + const annotationColumns: Array> = + annotationColNames.map(c => { + const parsed = parseFeedbackType(c); + return { + field: convertFeedbackFieldToBackendFilter(c), + headerName: parsed ? parsed.displayName : `${c}`, + width: 150, + renderHeader: () => { + return
{parsed ? parsed.userDefinedType : c}
; + }, + valueGetter: (unused: any, row: any) => { + return row[c]; + }, + renderCell: (params: GridRenderCellParams) => { + return
{params.value}
; + }, + }; + }); + cols.push(...annotationColumns); + } + cols.push({ field: 'wb_user_id', headerName: 'User', diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableColumnsUtil.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableColumnsUtil.ts index 60717246c014..b02dce4b7738 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableColumnsUtil.ts +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableColumnsUtil.ts @@ -20,7 +20,7 @@ export function isDynamicCallColumn(path: Path): boolean { } return ( path.length > 1 && - ['attributes', 'inputs', 'output', 'summary'].includes(path[0]) + ['attributes', 'inputs', 'output', 'summary', 'feedback'].includes(path[0]) ); } diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableQuery.ts index de221b652dcb..9d52bdf8b4fa 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableQuery.ts +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableQuery.ts @@ -64,6 +64,7 @@ export const useCallsForQuery = ( expandedColumns, { refetchOnDelete: true, + includeFeedback: true, } ); @@ -72,7 +73,7 @@ export const useCallsForQuery = ( }); const callResults = useMemo(() => { - return calls.result ?? []; + return getFeedbackMerged(calls.result ?? []); }, [calls]); const total = useMemo(() => { @@ -108,11 +109,12 @@ export const useCallsForQuery = ( { skip: calls.loading, includeCosts: true, + includeFeedback: true, } ); const costResults = useMemo(() => { - return costs.result ?? []; + return getFeedbackMerged(costs.result ?? []); }, [costs]); const refetch = useCallback(() => { calls.refetch(); @@ -121,6 +123,16 @@ export const useCallsForQuery = ( }, [calls, callsStats, costs]); return useMemo(() => { + if (calls.loading) { + return { + costsLoading: costs.loading, + loading: calls.loading, + result: [], + total: 0, + refetch, + }; + } + return { costsLoading: costs.loading, loading: calls.loading, @@ -217,3 +229,34 @@ const convertHighLevelFilterToLowLevelFilter = ( : undefined, }; }; + +const getFeedbackMerged = (calls: CallSchema[]) => { + // for each call, reduce all feedback to the latest feedback of each type + return calls.map(c => { + if (!c.traceCall?.summary?.weave?.feedback) { + return c; + } + const feedback = c.traceCall?.summary?.weave?.feedback?.reduce( + (acc: Record, curr: Record) => { + // keep most recent feedback of each type + if (acc[curr.feedback_type]?.created_at > curr.created_at) { + return acc; + } + acc[curr.feedback_type] = curr; + return acc; + }, + {} + ); + c.traceCall = { + ...c.traceCall, + summary: { + ...c.traceCall.summary, + weave: { + ...c.traceCall.summary.weave, + feedback, + }, + }, + }; + return c; + }); +}; diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooks.ts index d1cd773f0713..8e2f42035a1f 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooks.ts +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooks.ts @@ -243,7 +243,12 @@ const useCallsNoExpansion = ( sortBy?: traceServerTypes.SortBy[], query?: Query, columns?: string[], - opts?: {skip?: boolean; refetchOnDelete?: boolean; includeCosts?: boolean} + opts?: { + skip?: boolean; + refetchOnDelete?: boolean; + includeCosts?: boolean; + includeFeedback?: boolean; + } ): Loadable & Refetchable => { const getTsClient = useGetTraceServerClientContext(); const loadingRef = useRef(false); @@ -276,6 +281,7 @@ const useCallsNoExpansion = ( query, columns, include_costs: opts?.includeCosts, + include_feedback: opts?.includeFeedback, }; const onSuccess = (res: traceServerTypes.TraceCallsQueryRes) => { loadingRef.current = false; @@ -294,6 +300,7 @@ const useCallsNoExpansion = ( limit, opts?.skip, opts?.includeCosts, + opts?.includeFeedback, getTsClient, offset, sortBy, @@ -380,7 +387,12 @@ const useCalls = ( query?: Query, columns?: string[], expandedRefColumns?: Set, - opts?: {skip?: boolean; refetchOnDelete?: boolean; includeCosts?: boolean} + opts?: { + skip?: boolean; + refetchOnDelete?: boolean; + includeCosts?: boolean; + includeFeedback?: boolean; + } ): Loadable & Refetchable => { const calls = useCallsNoExpansion( entity, diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts index 2102c016c750..f28c7be67144 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts @@ -176,7 +176,12 @@ export type WFDataModelHooksInterface = { query?: Query, columns?: string[], expandedRefColumns?: Set, - opts?: {skip?: boolean; refetchOnDelete?: boolean; includeCosts?: boolean} + opts?: { + skip?: boolean; + refetchOnDelete?: boolean; + includeCosts?: boolean; + includeFeedback?: boolean; + } ) => Loadable & Refetchable; useCallsStats: ( entity: string,