Merge branch 'master' into tim/scorer_mega_pr

weave-js/src/common/util/SdkPointCloudToBabylon.test.ts

@@ -4,6 +4,7 @@ import {
   DEFAULT_POINT_COLOR,
   getFilteringOptionsForPointCloud,
   getVertexCompatiblePositionsAndColors,
+  loadPointCloud,
   MAX_BOUNDING_BOX_LABELS_FOR_DISPLAY,
   MaxAlphaValue,
 } from './SdkPointCloudToBabylon';
@@ -174,3 +175,16 @@ describe('getFilteringOptionsForPointCloud', () => {
     expect(newClassIdToLabel.get(49)).toEqual('label49');
   });
 });
+describe('loadPointCloud', () => {
+  it('appropriate defaults set when loading point cloud from file', () => {
+    const fileContents = JSON.stringify({
+      boxes: [],
+      points: [[]],
+      type: 'lidar/beta',
+      vectors: [],
+    });
+    const babylonPointCloud = loadPointCloud(fileContents);
+    expect(babylonPointCloud.points).toHaveLength(1);
+    expect(babylonPointCloud.points[0].position).toEqual([0, 0, 0]);
+  });
+});

weave-js/src/common/util/SdkPointCloudToBabylon.ts

@@ -160,7 +160,7 @@ export const handlePoints = (object3D: Object3DScene): ScenePoint[] => {
   // Draw Points
   return truncatedPoints.map(point => {
     const [x, y, z, r, g, b] = point;
-    const position: Position = [x, y, z];
+    const position: Position = [x ?? 0, y ?? 0, z ?? 0];
     const category = r;
 
     if (r !== undefined && g !== undefined && b !== undefined) {

weave-js/src/common/util/render_babylon.ts

@@ -394,6 +394,15 @@ const pointCloudScene = (
   // Apply vertexData to custom mesh
   vertexData.applyToMesh(pcMesh);
 
+  // A file without any points defined still includes a single, empty "point".
+  // In order to play nice with Babylon, we position this empty point at 0,0,0.
+  // Hence, a pointCloud with a single point at 0,0,0 is likely empty.
+  const isEmpty =
+    pointCloud.points.length === 1 &&
+    pointCloud.points[0].position[0] === 0 &&
+    pointCloud.points[0].position[1] === 0 &&
+    pointCloud.points[0].position[2] === 0;
+
   camera.parent = pcMesh;
 
   const pcMaterial = new Babylon.StandardMaterial('mat', scene);
@@ -472,8 +481,8 @@ const pointCloudScene = (
         new Vector3(edges.length * 2, edges.length * 2, edges.length * 2)
       );
 
-    // If we are iterating over camera, target a box
-    if (index === meta?.cameraIndex) {
+    // If we are iterating over camera or the cloud is empty, target a box
+    if (index === meta?.cameraIndex || (index === 0 && isEmpty)) {
       camera.position = center.add(new Vector3(0, 0, 1000));
       camera.target = center;
       camera.zoomOn([lines]);

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/FeedbackSidebar.tsx

@@ -96,7 +96,7 @@ export const FeedbackSidebar = ({
         <div className="text-lg font-semibold">Feedback</div>
         <div className="flex-grow" />
       </div>
-      <div className="min-h-1 mb-8 h-1 flex-grow overflow-auto bg-moon-300" />
+      <div className="min-h-1 mb-8 h-1 overflow-auto bg-moon-300" />
       {humanAnnotationSpecs.length > 0 ? (
         <>
           <div className="ml-6 h-full flex-grow overflow-auto">

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/hooks.ts

@@ -246,6 +246,63 @@ export const isTraceCallChatFormatGemini = (call: TraceCallSchema): boolean => {
   );
 };
 
+export const isAnthropicContentBlock = (block: any): boolean => {
+  if (!_.isPlainObject(block)) {
+    return false;
+  }
+  // TODO: Are there other types?
+  if (block.type !== 'text') {
+    return false;
+  }
+  if (!hasStringProp(block, 'text')) {
+    return false;
+  }
+  return true;
+};
+
+export const isAnthropicCompletionFormat = (output: any): boolean => {
+  if (output !== null) {
+    // TODO: Could have additional checks here on things like usage
+    if (
+      _.isPlainObject(output) &&
+      output.type === 'message' &&
+      output.role === 'assistant' &&
+      hasStringProp(output, 'model') &&
+      _.isArray(output.content) &&
+      output.content.every((c: any) => isAnthropicContentBlock(c))
+    ) {
+      return true;
+    }
+    return false;
+  }
+  return true;
+};
+
+type AnthropicContentBlock = {
+  type: 'text';
+  text: string;
+};
+
+export const anthropicContentBlocksToChoices = (
+  blocks: AnthropicContentBlock[],
+  stopReason: string
+): Choice[] => {
+  const choices: Choice[] = [];
+  for (let i = 0; i < blocks.length; i++) {
+    const block = blocks[i];
+    choices.push({
+      index: i,
+      message: {
+        role: 'assistant',
+        content: block.text,
+      },
+      // TODO: What is correct way to map this?
+      finish_reason: stopReason,
+    });
+  }
+  return choices;
+};
+
 export const isTraceCallChatFormatOpenAI = (call: TraceCallSchema): boolean => {
   if (!('messages' in call.inputs)) {
     return false;
@@ -336,6 +393,19 @@ export const normalizeChatRequest = (request: any): ChatRequest => {
       ],
     };
   }
+  // Anthropic has system message as a top-level request field
+  if (hasStringProp(request, 'system')) {
+    return {
+      ...request,
+      messages: [
+        {
+          role: 'system',
+          content: request.system,
+        },
+        ...request.messages,
+      ],
+    };
+  }
   return request as ChatRequest;
 };
 
@@ -360,6 +430,24 @@ export const normalizeChatCompletion = (
       },
     };
   }
+  if (isAnthropicCompletionFormat(completion)) {
+    return {
+      id: completion.id,
+      choices: anthropicContentBlocksToChoices(
+        completion.content,
+        completion.stop_reason
+      ),
+      created: 0,
+      model: completion.model,
+      system_fingerprint: '',
+      usage: {
+        prompt_tokens: completion.usage.input_tokens,
+        completion_tokens: completion.usage.output_tokens,
+        total_tokens:
+          completion.usage.input_tokens + completion.usage.output_tokens,
+      },
+    };
+  }
   return completion as ChatCompletion;
 };
 

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx

@@ -77,8 +77,6 @@ export const CompareEvaluationsPage: React.FC<
 export const CompareEvaluationsPageContent: React.FC<
   CompareEvaluationsPageProps
 > = props => {
-  const [baselineEvaluationCallId, setBaselineEvaluationCallId] =
-    React.useState<string | null>(null);
   const [comparisonDimensions, setComparisonDimensions] =
     React.useState<ComparisonDimensionsType | null>(null);
 
@@ -104,14 +102,6 @@ export const CompareEvaluationsPageContent: React.FC<
     [comparisonDimensions]
   );
 
-  React.useEffect(() => {
-    // Only update the baseline if we are switching evaluations, if there
-    // is more than 1, we are in the compare view and baseline is auto set
-    if (props.evaluationCallIds.length === 1) {
-      setBaselineEvaluationCallId(props.evaluationCallIds[0]);
-    }
-  }, [props.evaluationCallIds]);
-
   if (props.evaluationCallIds.length === 0) {
     return <div>No evaluations to compare</div>;
   }
@@ -120,13 +110,11 @@ export const CompareEvaluationsPageContent: React.FC<
     <CompareEvaluationsProvider
       entity={props.entity}
       project={props.project}
+      initialEvaluationCallIds={props.evaluationCallIds}
       selectedMetrics={props.selectedMetrics}
       setSelectedMetrics={props.setSelectedMetrics}
-      initialEvaluationCallIds={props.evaluationCallIds}
-      baselineEvaluationCallId={baselineEvaluationCallId ?? undefined}
       comparisonDimensions={comparisonDimensions ?? undefined}
       onEvaluationCallIdsUpdate={props.onEvaluationCallIdsUpdate}
-      setBaselineEvaluationCallId={setBaselineEvaluationCallId}
       setComparisonDimensions={setComparisonDimensionsAndClearInputDigest}
       selectedInputDigest={selectedInputDigest ?? undefined}
       setSelectedInputDigest={setSelectedInputDigest}>

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/compareEvaluationsContext.tsx

@@ -10,16 +10,14 @@ import {ComparisonDimensionsType} from './ecpState';
 
 const CompareEvaluationsContext = React.createContext<{
   state: EvaluationComparisonState;
-  setBaselineEvaluationCallId: React.Dispatch<
-    React.SetStateAction<string | null>
-  >;
   setComparisonDimensions: React.Dispatch<
     React.SetStateAction<ComparisonDimensionsType | null>
   >;
   setSelectedInputDigest: React.Dispatch<React.SetStateAction<string | null>>;
   setSelectedMetrics: (newModel: Record<string, boolean>) => void;
   addEvaluationCall: (newCallId: string) => void;
   removeEvaluationCall: (callId: string) => void;
+  setEvaluationCallOrder: (newCallIdOrder: string[]) => void;
 } | null>(null);
 
 export const useCompareEvaluationsState = () => {
@@ -33,34 +31,26 @@ export const useCompareEvaluationsState = () => {
 export const CompareEvaluationsProvider: React.FC<{
   entity: string;
   project: string;
+  initialEvaluationCallIds: string[];
   selectedMetrics: Record<string, boolean> | null;
   setSelectedMetrics: (newModel: Record<string, boolean>) => void;
 
-  initialEvaluationCallIds: string[];
   onEvaluationCallIdsUpdate: (newEvaluationCallIds: string[]) => void;
-  setBaselineEvaluationCallId: React.Dispatch<
-    React.SetStateAction<string | null>
-  >;
   setComparisonDimensions: React.Dispatch<
     React.SetStateAction<ComparisonDimensionsType | null>
   >;
   setSelectedInputDigest: React.Dispatch<React.SetStateAction<string | null>>;
-  baselineEvaluationCallId?: string;
   comparisonDimensions?: ComparisonDimensionsType;
   selectedInputDigest?: string;
 }> = ({
   entity,
   project,
+  initialEvaluationCallIds,
   selectedMetrics,
   setSelectedMetrics,
-
-  initialEvaluationCallIds,
   onEvaluationCallIdsUpdate,
-  setBaselineEvaluationCallId,
   setComparisonDimensions,
-
   setSelectedInputDigest,
-  baselineEvaluationCallId,
   comparisonDimensions,
   selectedInputDigest,
   children,
@@ -77,7 +67,6 @@ export const CompareEvaluationsProvider: React.FC<{
     entity,
     project,
     evaluationCallIds,
-    baselineEvaluationCallId,
     comparisonDimensions,
     selectedInputDigest,
     selectedMetrics ?? undefined
@@ -89,7 +78,6 @@ export const CompareEvaluationsProvider: React.FC<{
     }
     return {
       state: initialState.result,
-      setBaselineEvaluationCallId,
       setComparisonDimensions,
       setSelectedInputDigest,
       setSelectedMetrics,
@@ -105,14 +93,17 @@ export const CompareEvaluationsProvider: React.FC<{
         setEvaluationCallIds(newEvaluationCallIds);
         onEvaluationCallIdsUpdate(newEvaluationCallIds);
       },
+      setEvaluationCallOrder: (newCallIdOrder: string[]) => {
+        setEvaluationCallIds(newCallIdOrder);
+        onEvaluationCallIdsUpdate(newCallIdOrder);
+      },
     };
   }, [
     initialState.loading,
     initialState.result,
+    setEvaluationCallIds,
     evaluationCallIds,
     onEvaluationCallIdsUpdate,
-    setEvaluationCallIds,
-    setBaselineEvaluationCallId,
     setComparisonDimensions,
     setSelectedInputDigest,
     setSelectedMetrics,