From 15e4a0a75ebd2c047dbc546233d24dd74a142005 Mon Sep 17 00:00:00 2001 From: gtarpenning Date: Thu, 10 Oct 2024 09:44:26 -0700 Subject: [PATCH 1/4] merge griffin/eval-no-model-name --- .../pages/CallsPage/callsTableQuery.ts | 5 +- .../CompareEvaluationsPage.tsx | 6 +- .../compareEvaluationsContext.tsx | 18 +- .../ComparisonDefinitionSection.tsx | 217 ++++++++++++++++-- .../EvaluationDefinition.tsx | 57 ++++- 5 files changed, 279 insertions(+), 24 deletions(-) diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableQuery.ts index 87a5ec9423a..b80a9905f9d 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableQuery.ts +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/callsTableQuery.ts @@ -33,7 +33,8 @@ export const useCallsForQuery = ( gridFilter: GridFilterModel, gridSort: GridSortModel, gridPage: GridPaginationModel, - expandedColumns: Set + expandedColumns: Set, + columns?: string[] ): { result: CallSchema[]; loading: boolean; @@ -57,7 +58,7 @@ export const useCallsForQuery = ( offset, sortBy, filterBy, - undefined, + columns, expandedColumns, { refetchOnDelete: true, diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx index a269463ff4e..05da3ba49ed 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx @@ -95,7 +95,9 @@ export const CompareEvaluationsPageContent: React.FC< ); React.useEffect(() => { - if (props.evaluationCallIds.length > 0) { + // Only update the baseline if we are switching evaluations, if there + // is more than 1, we are in the compare view and baseline is auto set + if (props.evaluationCallIds.length === 1) { setBaselineEvaluationCallId(props.evaluationCallIds[0]); } }, [props.evaluationCallIds]); @@ -108,7 +110,7 @@ export const CompareEvaluationsPageContent: React.FC< >; setSelectedInputDigest: React.Dispatch>; + addEvaluationCall: (newCallId: string) => void; + removeEvaluationCall: (callId: string) => void; } | null>(null); export const useCompareEvaluationsState = () => { @@ -29,7 +31,7 @@ export const useCompareEvaluationsState = () => { export const CompareEvaluationsProvider: React.FC<{ entity: string; project: string; - evaluationCallIds: string[]; + initialEvaluationCallIds: string[]; setBaselineEvaluationCallId: React.Dispatch< React.SetStateAction >; @@ -43,7 +45,7 @@ export const CompareEvaluationsProvider: React.FC<{ }> = ({ entity, project, - evaluationCallIds, + initialEvaluationCallIds, setBaselineEvaluationCallId, setComparisonDimensions, @@ -54,6 +56,9 @@ export const CompareEvaluationsProvider: React.FC<{ selectedInputDigest, children, }) => { + const [evaluationCallIds, setEvaluationCallIds] = useState( + initialEvaluationCallIds + ); const initialState = useEvaluationComparisonState( entity, project, @@ -72,10 +77,17 @@ export const CompareEvaluationsProvider: React.FC<{ setBaselineEvaluationCallId, setComparisonDimensions, setSelectedInputDigest, + addEvaluationCall: (newCallId: string) => { + setEvaluationCallIds(prev => [...prev, newCallId]); + }, + removeEvaluationCall: (callId: string) => { + setEvaluationCallIds(prev => prev.filter(id => id !== callId)); + }, }; }, [ initialState.loading, initialState.result, + setEvaluationCallIds, setBaselineEvaluationCallId, setComparisonDimensions, setSelectedInputDigest, diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx index a7fb35f1746..3d461681a3c 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx @@ -1,12 +1,28 @@ -import React, {useMemo} from 'react'; +import {Popover} from '@mui/material'; +import Input from '@wandb/weave/common/components/Input'; +import {Tailwind} from '@wandb/weave/components/Tailwind'; +import {parseRef, WeaveObjectRef} from '@wandb/weave/react'; +import React, {useEffect, useMemo, useRef, useState} from 'react'; import {Button} from '../../../../../../../Button'; +import { + DEFAULT_FILTER_CALLS, + DEFAULT_SORT_CALLS, +} from '../../../CallsPage/CallsTable'; +import {useCallsForQuery} from '../../../CallsPage/callsTableQuery'; +import {useEvaluationsFilter} from '../../../CallsPage/evaluationsFilter'; +import {Id} from '../../../common/Id'; +import {useWFHooks} from '../../../wfReactInterface/context'; +import { + CallSchema, + ObjectVersionKey, +} from '../../../wfReactInterface/wfDataModelHooksInterface'; import {useCompareEvaluationsState} from '../../compareEvaluationsContext'; import {STANDARD_PADDING} from '../../ecpConstants'; import {getOrderedCallIds} from '../../ecpState'; import {EvaluationComparisonState} from '../../ecpState'; import {HorizontalBox} from '../../Layout'; -import {EvaluationDefinition} from './EvaluationDefinition'; +import {EvaluationDefinition, VerticalBar} from './EvaluationDefinition'; export const ComparisonDefinitionSection: React.FC<{ state: EvaluationComparisonState; @@ -28,25 +44,200 @@ export const ComparisonDefinitionSection: React.FC<{ {evalCallIds.map((key, ndx) => { return ( - {ndx !== 0 && } ); })} + ); }; -const SwapPositionsButton: React.FC<{callId: string}> = props => { - const {setBaselineEvaluationCallId} = useCompareEvaluationsState(); +const ModelRefLabel: React.FC<{modelRef: string}> = props => { + const {useObjectVersion} = useWFHooks(); + const objRef = useMemo( + () => parseRef(props.modelRef) as WeaveObjectRef, + [props.modelRef] + ); + const objVersionKey = useMemo(() => { + return { + scheme: 'weave', + entity: objRef.entityName, + project: objRef.projectName, + weaveKind: objRef.weaveKind, + objectId: objRef.artifactName, + versionHash: objRef.artifactVersion, + path: '', + refExtra: objRef.artifactRefExtra, + } as ObjectVersionKey; + }, [ + objRef.artifactName, + objRef.artifactRefExtra, + objRef.artifactVersion, + objRef.entityName, + objRef.projectName, + objRef.weaveKind, + ]); + const objectVersion = useObjectVersion(objVersionKey); + return ( + + {objectVersion.result?.objectId}:{objectVersion.result?.versionIndex} + + ); +}; + +const AddEvaluationButton: React.FC<{ + state: EvaluationComparisonState; +}> = props => { + const {addEvaluationCall} = useCompareEvaluationsState(); + + // Calls query for just evaluations + const evaluationsFilter = useEvaluationsFilter( + props.state.data.entity, + props.state.data.project + ); + const page = useMemo( + () => ({ + pageSize: 100, + page: 0, + }), + [] + ); + const expandedRefCols = useMemo(() => new Set(), []); + // Don't query for output here, re-queried in tsDataModelHooksEvaluationComparison.ts + const columns = useMemo(() => ['inputs'], []); + const calls = useCallsForQuery( + props.state.data.entity, + props.state.data.project, + evaluationsFilter, + DEFAULT_FILTER_CALLS, + DEFAULT_SORT_CALLS, + page, + expandedRefCols, + columns + ); + + const evalsNotComparing = useMemo(() => { + return calls.result.filter( + call => + !Object.keys(props.state.data.evaluationCalls).includes(call.callId) + ); + }, [calls.result, props.state.data.evaluationCalls]); + + const [menuOptions, setMenuOptions] = + useState(evalsNotComparing); + useEffect(() => { + setMenuOptions(evalsNotComparing); + }, [evalsNotComparing]); + + const onSearchChange = (e: React.ChangeEvent) => { + const search = e.target.value; + if (search === '') { + setMenuOptions(evalsNotComparing); + return; + } + + const filteredOptions = calls.result.filter(call => { + if ( + (call.displayName ?? call.spanName) + .toLowerCase() + .includes(search.toLowerCase()) + ) { + return true; + } + if (call.callId.slice(-4).includes(search)) { + return true; + } + const modelRef = parseRef(call.traceCall?.inputs.model) as WeaveObjectRef; + if (modelRef.artifactName.toLowerCase().includes(search.toLowerCase())) { + return true; + } + return false; + }); + + setMenuOptions(filteredOptions); + }; + + // Popover management + const refBar = useRef(null); + const refLabel = useRef(null); + const [anchorEl, setAnchorEl] = React.useState(null); + const onClick = (event: React.MouseEvent) => { + setAnchorEl(anchorEl ? null : refBar.current); + }; + const open = Boolean(anchorEl); + const id = open ? 'simple-popper' : undefined; + return ( - + + + setAnchorEl(null)}> + +
+ +
+ {menuOptions.length === 0 && ( +
No evaluations
+ )} + {menuOptions.map(call => ( +
+ +
+ ))} +
+
+ + + ); }; diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx index 576b17f5822..7fdb0c6c948 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx @@ -1,5 +1,8 @@ import {Box} from '@material-ui/core'; import {Circle} from '@mui/icons-material'; +import {PopupDropdown} from '@wandb/weave/common/components/PopupDropdown'; +import {Button} from '@wandb/weave/components/Button'; +import {Pill} from '@wandb/weave/components/Tag'; import React, {useMemo} from 'react'; import { @@ -14,6 +17,7 @@ import {SmallRef} from '../../../../../Browse2/SmallRef'; import {CallLink, ObjectVersionLink} from '../../../common/Links'; import {useWFHooks} from '../../../wfReactInterface/context'; import {ObjectVersionKey} from '../../../wfReactInterface/wfDataModelHooksInterface'; +import {useCompareEvaluationsState} from '../../compareEvaluationsContext'; import { BOX_RADIUS, CIRCLE_SIZE, @@ -27,6 +31,36 @@ export const EvaluationDefinition: React.FC<{ state: EvaluationComparisonState; callId: string; }> = props => { + const {removeEvaluationCall, setBaselineEvaluationCallId} = + useCompareEvaluationsState(); + + const menuOptions = useMemo(() => { + return [ + { + key: 'add-to-baseline', + content: 'Set as baseline', + onClick: () => { + setBaselineEvaluationCallId(props.callId); + }, + disabled: props.callId === props.state.baselineEvaluationCallId, + }, + { + key: 'remove', + content: 'Remove', + onClick: () => { + removeEvaluationCall(props.callId); + }, + disabled: Object.keys(props.state.data.evaluationCalls).length === 1, + }, + ]; + }, [ + props.callId, + props.state.baselineEvaluationCallId, + props.state.data.evaluationCalls, + removeEvaluationCall, + setBaselineEvaluationCallId, + ]); + return ( - - + {props.callId === props.state.baselineEvaluationCallId && ( + + )} + + } + /> ); }; + export const EvaluationCallLink: React.FC<{ callId: string; state: EvaluationComparisonState; @@ -147,11 +195,12 @@ const ModelIcon: React.FC = () => { ); }; -const VerticalBar: React.FC = () => { + +export const VerticalBar: React.FC = () => { return (
Date: Thu, 10 Oct 2024 09:56:30 -0700 Subject: [PATCH 2/4] chore: added/removed evals reflected in URL state --- .../PagePanelComponents/Home/Browse3.tsx | 11 +++++++++++ .../Home/Browse3/pages/CallPage/CallPage.tsx | 2 ++ .../CompareEvaluationsPage.tsx | 3 +++ .../compareEvaluationsContext.tsx | 14 ++++++++++++-- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx index fc004b04717..0635fbfbb50 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx @@ -921,16 +921,27 @@ const OpPageBinding = () => { }; const CompareEvaluationsBinding = () => { + const history = useHistory(); + const location = useLocation(); const {entity, project} = useParamsDecoded(); const query = useURLSearchParamsDict(); const evaluationCallIds = useMemo(() => { return JSON.parse(query.evaluationCallIds); }, [query.evaluationCallIds]); + const onEvaluationCallIdsUpdate = useCallback( + (newEvaluationCallIds: string[]) => { + const newQuery = new URLSearchParams(location.search); + newQuery.set('evaluationCallIds', JSON.stringify(newEvaluationCallIds)); + history.push({search: newQuery.toString()}); + }, + [history, location.search] + ); return ( ); }; diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx index 9f885268642..8e179da6742 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx @@ -64,6 +64,8 @@ const useCallTabs = (call: CallSchema) => { entity={call.entity} project={call.project} evaluationCallIds={[call.callId]} + // Dont persist changes to evaluationCallIds in the URL + onEvaluationCallIdsUpdate={() => {}} /> ), }, diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx index 05da3ba49ed..7cd3385f8ae 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx @@ -38,6 +38,7 @@ type CompareEvaluationsPageProps = { entity: string; project: string; evaluationCallIds: string[]; + onEvaluationCallIdsUpdate: (newEvaluationCallIds: string[]) => void; }; export const CompareEvaluationsPage: React.FC< @@ -55,6 +56,7 @@ export const CompareEvaluationsPage: React.FC< entity={props.entity} project={props.project} evaluationCallIds={props.evaluationCallIds} + onEvaluationCallIdsUpdate={props.onEvaluationCallIdsUpdate} /> ), }, @@ -113,6 +115,7 @@ export const CompareEvaluationsPageContent: React.FC< initialEvaluationCallIds={props.evaluationCallIds} baselineEvaluationCallId={baselineEvaluationCallId ?? undefined} comparisonDimensions={comparisonDimensions ?? undefined} + onEvaluationCallIdsUpdate={props.onEvaluationCallIdsUpdate} setBaselineEvaluationCallId={setBaselineEvaluationCallId} setComparisonDimensions={setComparisonDimensionsAndClearInputDigest} selectedInputDigest={selectedInputDigest ?? undefined} diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/compareEvaluationsContext.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/compareEvaluationsContext.tsx index aa26f7ca516..768ab2555a2 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/compareEvaluationsContext.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/compareEvaluationsContext.tsx @@ -32,6 +32,7 @@ export const CompareEvaluationsProvider: React.FC<{ entity: string; project: string; initialEvaluationCallIds: string[]; + onEvaluationCallIdsUpdate: (newEvaluationCallIds: string[]) => void; setBaselineEvaluationCallId: React.Dispatch< React.SetStateAction >; @@ -46,6 +47,7 @@ export const CompareEvaluationsProvider: React.FC<{ entity, project, initialEvaluationCallIds, + onEvaluationCallIdsUpdate, setBaselineEvaluationCallId, setComparisonDimensions, @@ -78,15 +80,23 @@ export const CompareEvaluationsProvider: React.FC<{ setComparisonDimensions, setSelectedInputDigest, addEvaluationCall: (newCallId: string) => { - setEvaluationCallIds(prev => [...prev, newCallId]); + const newEvaluationCallIds = [...evaluationCallIds, newCallId]; + setEvaluationCallIds(newEvaluationCallIds); + onEvaluationCallIdsUpdate(newEvaluationCallIds); }, removeEvaluationCall: (callId: string) => { - setEvaluationCallIds(prev => prev.filter(id => id !== callId)); + const newEvaluationCallIds = evaluationCallIds.filter( + id => id !== callId + ); + setEvaluationCallIds(newEvaluationCallIds); + onEvaluationCallIdsUpdate(newEvaluationCallIds); }, }; }, [ initialState.loading, initialState.result, + evaluationCallIds, + onEvaluationCallIdsUpdate, setEvaluationCallIds, setBaselineEvaluationCallId, setComparisonDimensions, From 1f51fc5c83bf00790493da4ea00ec4de9e5090ae Mon Sep 17 00:00:00 2001 From: gtarpenning Date: Thu, 10 Oct 2024 10:07:06 -0700 Subject: [PATCH 3/4] add small margin adjustment --- .../EvaluationDefinition.tsx | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx index 7fdb0c6c948..ce1b22e77f8 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx @@ -75,18 +75,20 @@ export const EvaluationDefinition: React.FC<{ {props.callId === props.state.baselineEvaluationCallId && ( )} - - } - /> +
+ + } + /> +
); }; From c226a9b370de23a956664f385127309e256c516e Mon Sep 17 00:00:00 2001 From: gtarpenning Date: Thu, 10 Oct 2024 10:07:40 -0700 Subject: [PATCH 4/4] small --- .../ComparisonDefinitionSection/EvaluationDefinition.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx index ce1b22e77f8..adc80d044f6 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx @@ -82,7 +82,7 @@ export const EvaluationDefinition: React.FC<{