From a06b6ace52988bff45012d78a29c9adbbeb32249 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Fri, 6 Dec 2024 08:57:21 -0800
Subject: [PATCH 01/52] chore(weave): Improve large object viewer renders
 (#3102)

* init

* init

* Larger refactor

* Larger refactor

* Larger refactor

* Larger refactor

* Larger refactor
---
 .../Browse3/pages/CallPage/CallDetails.tsx    |  41 ++-----
 .../Browse3/pages/CallPage/ObjectViewer.tsx   |  81 ++++++++-----
 .../pages/CallPage/ObjectViewerSection.tsx    |   4 +-
 .../traceServerCachingClient.ts               | 106 ++++++++++++++++++
 .../wfReactInterface/traceServerClient.ts     |   4 +-
 .../typeViews/CustomWeaveTypeDispatcher.tsx   |   8 ++
 .../PIL.Image.Image/PILImageImage.tsx         |   5 +-
 7 files changed, 183 insertions(+), 66 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerCachingClient.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallDetails.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallDetails.tsx
index 8a9c011b5de..26be9f0c5d0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallDetails.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallDetails.tsx
@@ -1,4 +1,3 @@
-import {Typography} from '@mui/material';
 import Box from '@mui/material/Box';
 import _ from 'lodash';
 import React, {FC, useContext, useMemo} from 'react';
@@ -10,9 +9,7 @@ import {Button} from '../../../../../Button';
 import {useWeaveflowRouteContext, WeaveflowPeekContext} from '../../context';
 import {CustomWeaveTypeProjectContext} from '../../typeViews/CustomWeaveTypeDispatcher';
 import {CallsTable} from '../CallsPage/CallsTable';
-import {KeyValueTable} from '../common/KeyValueTable';
-import {CallLink, opNiceName} from '../common/Links';
-import {CenteredAnimatedLoader} from '../common/Loader';
+import {CallLink} from '../common/Links';
 import {useWFHooks} from '../wfReactInterface/context';
 import {CallSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 import {ButtonOverlay} from './ButtonOverlay';
@@ -20,6 +17,8 @@ import {ExceptionDetails, getExceptionInfo} from './Exceptions';
 import {ObjectViewerSection} from './ObjectViewerSection';
 import {OpVersionText} from './OpVersionText';
 
+const HEADER_HEIGHT_BUFFER = 60;
+
 const Heading = styled.div`
   color: ${MOON_800};
   font-weight: 600;
@@ -104,7 +103,7 @@ export const CallDetails: FC<{
     columns
   );
 
-  const {singularChildCalls, multipleChildCallOpRefs} = useMemo(
+  const {multipleChildCallOpRefs} = useMemo(
     () => callGrouping(!childCalls.loading ? childCalls.result ?? [] : []),
     [childCalls.loading, childCalls.result]
   );
@@ -133,6 +132,7 @@ export const CallDetails: FC<{
         <Box
           sx={{
             flex: '0 0 auto',
+            maxHeight: `calc(100% - ${HEADER_HEIGHT_BUFFER}px)`,
             p: 2,
           }}>
           <CustomWeaveTypeProjectContext.Provider
@@ -143,6 +143,9 @@ export const CallDetails: FC<{
         <Box
           sx={{
             flex: '0 0 auto',
+            maxHeight: `calc(100% - ${
+              multipleChildCallOpRefs.length > 0 ? HEADER_HEIGHT_BUFFER : 0
+            }px)`,
             p: 2,
           }}>
           {'traceback' in excInfo ? (
@@ -201,7 +204,6 @@ export const CallDetails: FC<{
               sx={{
                 flex: '0 0 auto',
                 height: '500px',
-                maxHeight: '95%',
                 p: 2,
                 display: 'flex',
                 flexDirection: 'column',
@@ -240,33 +242,6 @@ export const CallDetails: FC<{
             </Box>
           );
         })}
-        {childCalls.loading && <CenteredAnimatedLoader />}
-        {/* Disabling display of singular children while we decide if we want them here. */}
-        {false && singularChildCalls.length > 0 && (
-          <Box
-            sx={{
-              flex: '0 0 auto',
-            }}>
-            {multipleChildCallOpRefs.length === 0 ? (
-              <Typography pl={1}>Child calls</Typography>
-            ) : (
-              <Typography pl={1}>Singular child calls</Typography>
-            )}
-            {singularChildCalls.map(c => (
-              <Box
-                key={c.callId}
-                sx={{
-                  flex: '0 0 auto',
-                  p: 2,
-                }}>
-                <KeyValueTable
-                  headerTitle={opNiceName(c.spanName)}
-                  data={getDisplayInputsAndOutput(c)}
-                />
-              </Box>
-            ))}
-          </Box>
-        )}
       </Box>
     </Box>
   );
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/ObjectViewer.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/ObjectViewer.tsx
index d741db64771..e8f39ad2880 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/ObjectViewer.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/ObjectViewer.tsx
@@ -23,7 +23,11 @@ import {LoadingDots} from '../../../../../LoadingDots';
 import {Browse2OpDefCode} from '../../../Browse2/Browse2OpDefCode';
 import {isWeaveRef} from '../../filters/common';
 import {StyledDataGrid} from '../../StyledDataGrid';
-import {isCustomWeaveTypePayload} from '../../typeViews/customWeaveType.types';
+import {
+  CustomWeaveTypePayload,
+  isCustomWeaveTypePayload,
+} from '../../typeViews/customWeaveType.types';
+import {getCustomWeaveTypePreferredRowHeight} from '../../typeViews/CustomWeaveTypeDispatcher';
 import {
   LIST_INDEX_EDGE_NAME,
   OBJECT_ATTR_EDGE_NAME,
@@ -48,6 +52,10 @@ import {
 } from './traverse';
 import {ValueView} from './ValueView';
 
+const DEFAULT_ROW_HEIGHT = 38;
+const CODE_ROW_HEIGHT = 350;
+const TABLE_ROW_HEIGHT = 450;
+
 type Data = Record<string, any> | any[];
 
 type ObjectViewerProps = {
@@ -440,6 +448,47 @@ export const ObjectViewer = ({
     });
   }, [apiRef, expandedIds, updateRowExpand]);
 
+  // Per https://mui.com/x/react-data-grid/row-height/#dynamic-row-height, always
+  // memoize the getRowHeight function.
+  const getRowHeight = useCallback((params: GridRowHeightParams) => {
+    const isNonRefString =
+      params.model.valueType === 'string' && !isWeaveRef(params.model.value);
+    const isArray = params.model.valueType === 'array';
+    const isTableRef =
+      isWeaveRef(params.model.value) &&
+      (parseRefMaybe(params.model.value) as any).weaveKind === 'table';
+    const {isCode} = params.model;
+    const isCustomWeaveType = isCustomWeaveTypePayload(params.model.value);
+    if (!params.model.isLeaf) {
+      // This is a group header, so we want to use the default height
+      return DEFAULT_ROW_HEIGHT;
+    } else if (isNonRefString) {
+      // This is the only special case where we will allow for dynamic height.
+      // Since strings have special renders that take up different amounts of
+      // space, we will allow for dynamic height.
+      return 'auto';
+    } else if (isCustomWeaveType) {
+      const type = (params.model.value as CustomWeaveTypePayload).weave_type
+        .type;
+      const preferredRowHeight = getCustomWeaveTypePreferredRowHeight(type);
+      if (preferredRowHeight) {
+        return preferredRowHeight;
+      }
+      return DEFAULT_ROW_HEIGHT;
+    } else if ((isArray && USE_TABLE_FOR_ARRAYS) || isTableRef) {
+      // Perfectly enough space for 1 page of data rows
+      return TABLE_ROW_HEIGHT;
+    } else if (isCode) {
+      // Probably will get negative feedback here since code that is < 20 lines
+      // will have some whitespace below the code. However, we absolutely need
+      // to have static height for all cells else the MUI data grid will jump around
+      // when cleaning up virtual rows.
+      return CODE_ROW_HEIGHT;
+    } else {
+      return DEFAULT_ROW_HEIGHT;
+    }
+  }, []);
+
   // Finally, we memoize the inner data grid component. This is important to
   // reduce the number of re-renders when the data changes.
   const inner = useMemo(() => {
@@ -473,31 +522,9 @@ export const ObjectViewer = ({
         isGroupExpandedByDefault={node => {
           return expandedIds.includes(node.id);
         }}
-        autoHeight
         columnHeaderHeight={38}
-        getRowHeight={(params: GridRowHeightParams) => {
-          const isNonRefString =
-            params.model.valueType === 'string' &&
-            !isWeaveRef(params.model.value);
-          const isArray = params.model.valueType === 'array';
-          const isTableRef =
-            isWeaveRef(params.model.value) &&
-            (parseRefMaybe(params.model.value) as any).weaveKind === 'table';
-          const {isCode} = params.model;
-          const isCustomWeaveType = isCustomWeaveTypePayload(
-            params.model.value
-          );
-          if (
-            isNonRefString ||
-            (isArray && USE_TABLE_FOR_ARRAYS) ||
-            isTableRef ||
-            isCode ||
-            isCustomWeaveType
-          ) {
-            return 'auto';
-          }
-          return 38;
-        }}
+        rowHeight={DEFAULT_ROW_HEIGHT}
+        getRowHeight={getRowHeight}
         hideFooter
         rowSelection={false}
         groupingColDef={groupingColDef}
@@ -517,10 +544,10 @@ export const ObjectViewer = ({
         }}
       />
     );
-  }, [apiRef, rows, columns, groupingColDef, expandedIds]);
+  }, [apiRef, rows, columns, getRowHeight, groupingColDef, expandedIds]);
 
   // Return the inner data grid wrapped in a div with overflow hidden.
-  return <div style={{overflow: 'hidden'}}>{inner}</div>;
+  return <div style={{overflow: 'hidden', height: '100%'}}>{inner}</div>;
 };
 
 // Helper function to build the base ref for a given path. This function is used
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/ObjectViewerSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/ObjectViewerSection.tsx
index d95d9a841de..d02a2e881ca 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/ObjectViewerSection.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/ObjectViewerSection.tsx
@@ -185,7 +185,7 @@ const ObjectViewerSectionNonEmpty = ({
   }, [data, isExpanded]);
 
   return (
-    <>
+    <Box sx={{height: '100%', display: 'flex', flexDirection: 'column'}}>
       <TitleRow>
         <Title>{title}</Title>
         <Button
@@ -224,7 +224,7 @@ const ObjectViewerSectionNonEmpty = ({
         )}
       </TitleRow>
       {body}
-    </>
+    </Box>
   );
 };
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerCachingClient.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerCachingClient.ts
new file mode 100644
index 00000000000..d1ca67c7c4c
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerCachingClient.ts
@@ -0,0 +1,106 @@
+import LRUCache from 'lru-cache';
+
+import {
+  TraceFileContentReadReq,
+  TraceFileContentReadRes,
+  TraceObjReadReq,
+  TraceObjReadRes,
+  TraceTableQueryReq,
+  TraceTableQueryRes,
+  TraceTableQueryStatsReq,
+  TraceTableQueryStatsRes,
+} from './traceServerClientTypes';
+import {DirectTraceServerClient} from './traceServerDirectClient';
+
+interface CacheConfig {
+  max: number;
+  getCacheKey: (req: any) => string;
+}
+
+export class CachingTraceServerClient extends DirectTraceServerClient {
+  private caches: Map<
+    string,
+    {cache: LRUCache<string, any>; config: CacheConfig}
+  >;
+
+  constructor(baseUrl: string) {
+    super(baseUrl);
+    this.caches = new Map();
+
+    // Configure caches for specific methods
+    this.addCache('fileContent', {
+      max: 1000,
+      getCacheKey: (req: TraceFileContentReadReq) => JSON.stringify(req),
+    });
+
+    this.addCache('objRead', {
+      max: 1000,
+      getCacheKey: (req: TraceObjReadReq) => JSON.stringify(req),
+    });
+
+    this.addCache('tableQuery', {
+      max: 1000,
+      getCacheKey: (req: TraceTableQueryReq) => JSON.stringify(req),
+    });
+
+    this.addCache('tableQueryStats', {
+      max: 1000,
+      getCacheKey: (req: TraceTableQueryStatsReq) => JSON.stringify(req),
+    });
+  }
+
+  protected addCache(methodName: string, config: CacheConfig) {
+    this.caches.set(methodName, {
+      cache: new LRUCache({max: config.max}),
+      config,
+    });
+  }
+
+  protected withCache<Req, Res>(
+    methodName: string,
+    req: Req,
+    getFresh: () => Promise<Res>
+  ): Promise<Res> {
+    const cacheInfo = this.caches.get(methodName);
+    if (!cacheInfo) {
+      return getFresh();
+    }
+
+    const {cache, config} = cacheInfo;
+    const key = config.getCacheKey(req);
+    const cached = cache.get(key) as Res | undefined;
+
+    if (cached) {
+      return Promise.resolve(cached);
+    }
+
+    return getFresh().then(result => {
+      cache.set(key, result);
+      return result;
+    });
+  }
+
+  public override fileContent(
+    req: TraceFileContentReadReq
+  ): Promise<TraceFileContentReadRes> {
+    return this.withCache('fileContent', req, () => super.fileContent(req));
+  }
+
+  public override objRead(req: TraceObjReadReq): Promise<TraceObjReadRes> {
+    return this.withCache('objRead', req, () => super.objRead(req));
+  }
+
+  public override tableQuery(
+    req: TraceTableQueryReq
+  ): Promise<TraceTableQueryRes> {
+    return this.withCache('tableQuery', req, () => super.tableQuery(req));
+  }
+
+  public override tableQueryStats(
+    req: TraceTableQueryStatsReq
+  ): Promise<TraceTableQueryStatsRes> {
+    return this.withCache('tableQueryStats', req, () =>
+      super.tableQueryStats(req)
+    );
+  }
+}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClient.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClient.ts
index 2c38bcef551..b3a15a8a595 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClient.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClient.ts
@@ -1,5 +1,6 @@
 import _ from 'lodash';
 
+import {CachingTraceServerClient} from './traceServerCachingClient';
 import {
   CompletionsCreateReq,
   CompletionsCreateRes,
@@ -14,12 +15,11 @@ import {
   TraceRefsReadBatchReq,
   TraceRefsReadBatchRes,
 } from './traceServerClientTypes';
-import {DirectTraceServerClient} from './traceServerDirectClient';
 
 const DEFAULT_BATCH_INTERVAL = 150;
 const MAX_REFS_PER_BATCH = 1000;
 
-export class TraceServerClient extends DirectTraceServerClient {
+export class TraceServerClient extends CachingTraceServerClient {
   private readBatchCollectors: Array<{
     req: TraceRefsReadBatchReq;
     resolvePromise: (res: TraceRefsReadBatchRes) => void;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/typeViews/CustomWeaveTypeDispatcher.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/typeViews/CustomWeaveTypeDispatcher.tsx
index ec246ceb8cc..7edce4908da 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/typeViews/CustomWeaveTypeDispatcher.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/typeViews/CustomWeaveTypeDispatcher.tsx
@@ -24,16 +24,24 @@ const customWeaveTypeRegistry: {
       project: string;
       data: any; // I wish this could be typed more specifically
     }>;
+    preferredRowHeight?: number;
   };
 } = {
   'PIL.Image.Image': {
     component: PILImageImage,
+    preferredRowHeight: 350,
   },
   'wave.Wave_read': {
     component: AudioPlayer,
   },
 };
 
+export const getCustomWeaveTypePreferredRowHeight = (
+  typeId: string
+): number | undefined => {
+  return customWeaveTypeRegistry[typeId]?.preferredRowHeight;
+};
+
 /**
  * This context is used to provide the entity and project to the
  * CustomWeaveTypeDispatcher. Importantly, what this does is allows the
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/typeViews/PIL.Image.Image/PILImageImage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/typeViews/PIL.Image.Image/PILImageImage.tsx
index fb07477497f..03944dca959 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/typeViews/PIL.Image.Image/PILImageImage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/typeViews/PIL.Image.Image/PILImageImage.tsx
@@ -45,8 +45,9 @@ export const PILImageImage: React.FC<{
       src={url}
       alt="Custom"
       style={{
-        maxWidth: '100%',
-        maxHeight: '100%',
+        width: '100%',
+        height: '100%',
+        objectFit: 'contain',
       }}
     />
   );

From 8d010651bb2a06d8c69cc3fa316870441e7e760d Mon Sep 17 00:00:00 2001
From: Ben Sherman <bcanfieldsherman@gmail.com>
Date: Fri, 6 Dec 2024 09:00:43 -0800
Subject: [PATCH 02/52] chore(weave): capture request id for openai completions
 (#3153)

---
 .../instructor/instructor_test.py             |  6 ++--
 tests/integrations/openai/openai_test.py      | 36 +++++++++----------
 weave/integrations/openai/openai_sdk.py       | 13 ++++---
 3 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/tests/integrations/instructor/instructor_test.py b/tests/integrations/instructor/instructor_test.py
index 761d961b9af..b9667ebb086 100644
--- a/tests/integrations/instructor/instructor_test.py
+++ b/tests/integrations/instructor/instructor_test.py
@@ -64,7 +64,7 @@ def test_instructor_openai(
     assert op_name_from_ref(call.op_name) == "openai.chat.completions.create"
     output = call.output
     output_arguments = json.loads(
-        output.choices[0].message.tool_calls[0].function.arguments
+        output["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"]
     )
     assert "person_name" in output_arguments
     assert "age" in output_arguments
@@ -112,7 +112,7 @@ async def extract_person(text: str) -> Person:
     assert op_name_from_ref(call.op_name) == "openai.chat.completions.create"
     output = call.output
     output_arguments = json.loads(
-        output.choices[0].message.tool_calls[0].function.arguments
+        output["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"]
     )
     assert "person_name" in output_arguments
     assert "age" in output_arguments
@@ -166,7 +166,7 @@ def test_instructor_iterable(
     assert call.started_at < call.ended_at
     assert op_name_from_ref(call.op_name) == "openai.chat.completions.create"
     output = call.output
-    output_arguments = json.loads(output.choices[0].message.content)
+    output_arguments = json.loads(output["choices"][0]["message"]["content"])
     assert "tasks" in output_arguments
     assert "person_name" in output_arguments["tasks"][0]
     assert "age" in output_arguments["tasks"][0]
diff --git a/tests/integrations/openai/openai_test.py b/tests/integrations/openai/openai_test.py
index 331a2580030..3022defc5a1 100644
--- a/tests/integrations/openai/openai_test.py
+++ b/tests/integrations/openai/openai_test.py
@@ -38,10 +38,10 @@ def test_openai_quickstart(client: weave.trace.weave_client.WeaveClient) -> None
     assert call.started_at < call.ended_at  # type: ignore
 
     output = call.output
-    assert output.model == "gpt-4o-2024-05-13"
-    assert output.object == "chat.completion"
+    assert output["model"] == "gpt-4o-2024-05-13"
+    assert output["object"] == "chat.completion"
 
-    usage = call.summary["usage"][output.model]  # type: ignore
+    usage = call.summary["usage"][output["model"]]  # type: ignore
     assert usage["requests"] == 1
     assert usage["completion_tokens"] == 28
     assert usage["prompt_tokens"] == 11
@@ -86,10 +86,10 @@ async def test_openai_async_quickstart(
     assert call.started_at < call.ended_at  # type: ignore
 
     output = call.output
-    assert output.model == "gpt-4o-2024-05-13"
-    assert output.object == "chat.completion"
+    assert output["model"] == "gpt-4o-2024-05-13"
+    assert output["object"] == "chat.completion"
 
-    usage = call.summary["usage"][output.model]  # type: ignore
+    usage = call.summary["usage"][output["model"]]  # type: ignore
     assert usage["requests"] == 1
     assert usage["completion_tokens"] == 28
     assert usage["prompt_tokens"] == 11
@@ -315,10 +315,10 @@ def test_openai_function_call(client: weave.trace.weave_client.WeaveClient) -> N
     assert call.started_at < call.ended_at  # type: ignore
 
     output = call.output
-    assert output.model == "gpt-4o-2024-05-13"
-    assert output.object == "chat.completion"
+    assert output["model"] == "gpt-4o-2024-05-13"
+    assert output["object"] == "chat.completion"
 
-    usage = call.summary["usage"][output.model]  # type: ignore
+    usage = call.summary["usage"][output["model"]]  # type: ignore
     assert usage["total_tokens"] == 117
     assert usage["completion_tokens"] == 18
     assert usage["prompt_tokens"] == 99
@@ -401,10 +401,10 @@ async def test_openai_function_call_async(
     assert call.started_at < call.ended_at  # type: ignore
 
     output = call.output
-    assert output.model == "gpt-4o-2024-05-13"
-    assert output.object == "chat.completion"
+    assert output["model"] == "gpt-4o-2024-05-13"
+    assert output["object"] == "chat.completion"
 
-    usage = call.summary["usage"][output.model]  # type: ignore
+    usage = call.summary["usage"][output["model"]]  # type: ignore
     assert usage["total_tokens"] == 117
     assert usage["completion_tokens"] == 18
     assert usage["prompt_tokens"] == 99
@@ -577,10 +577,10 @@ def test_openai_tool_call(client: weave.trace.weave_client.WeaveClient) -> None:
     assert call.started_at < call.ended_at  # type: ignore
 
     output = call.output
-    assert output.model == "gpt-4o-2024-05-13"
-    assert output.object == "chat.completion"
+    assert output["model"] == "gpt-4o-2024-05-13"
+    assert output["object"] == "chat.completion"
 
-    usage = call.summary["usage"][output.model]  # type: ignore
+    usage = call.summary["usage"][output["model"]]  # type: ignore
     assert usage["total_tokens"] == 117
     assert usage["completion_tokens"] == 27
     assert usage["prompt_tokens"] == 90
@@ -664,10 +664,10 @@ async def test_openai_tool_call_async(
     assert call.started_at < call.ended_at  # type: ignore
 
     output = call.output
-    assert output.model == "gpt-4o-2024-05-13"
-    assert output.object == "chat.completion"
+    assert output["model"] == "gpt-4o-2024-05-13"
+    assert output["object"] == "chat.completion"
 
-    usage = call.summary["usage"][output.model]  # type: ignore
+    usage = call.summary["usage"][output["model"]]  # type: ignore
     assert usage["total_tokens"] == 117
     assert usage["completion_tokens"] == 27
     assert usage["prompt_tokens"] == 90
diff --git a/weave/integrations/openai/openai_sdk.py b/weave/integrations/openai/openai_sdk.py
index e68661875f3..7814700d4d3 100644
--- a/weave/integrations/openai/openai_sdk.py
+++ b/weave/integrations/openai/openai_sdk.py
@@ -94,8 +94,9 @@ def _get_tool_calls(
                 )
         return _tool_calls
 
+    dump = None
     if isinstance(value, ChatCompletionChunk):
-        final_value = ChatCompletion(
+        dump = ChatCompletion(
             id=value.id,
             choices=[
                 {
@@ -116,10 +117,14 @@ def _get_tool_calls(
             object="chat.completion",
             system_fingerprint=value.system_fingerprint,
             usage=value.usage if hasattr(value, "usage") else None,
-        )
-        return final_value.model_dump(exclude_unset=True, exclude_none=True)
-    else:
+        ).model_dump(exclude_unset=True, exclude_none=True)
+    elif not hasattr(value, "model_dump"):
         return value
+    else:
+        dump = value.model_dump(exclude_unset=True, exclude_none=True)
+    if hasattr(value, "_request_id"):
+        dump["request_id"] = value._request_id
+    return dump
 
 
 def openai_accumulator(

From dd43774237789b8ecda6fcb263853a23224e1c45 Mon Sep 17 00:00:00 2001
From: Ben Sherman <bcanfieldsherman@gmail.com>
Date: Fri, 6 Dec 2024 10:11:04 -0800
Subject: [PATCH 03/52] chore(ui): update calls table column menu icons (#3162)

---
 weave-js/src/assets/icons/icon-not-visible.svg    |  5 +++++
 weave-js/src/assets/icons/icon-pin-to-right.svg   |  5 +++++
 weave-js/src/components/Icon/Icon.tsx             | 10 ++++++++++
 weave-js/src/components/Icon/index.ts             |  2 ++
 weave-js/src/components/Icon/types.ts             |  2 ++
 .../Home/Browse3/pages/CallsPage/CallsTable.tsx   | 15 ++++++++++++++-
 6 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 weave-js/src/assets/icons/icon-not-visible.svg
 create mode 100644 weave-js/src/assets/icons/icon-pin-to-right.svg

diff --git a/weave-js/src/assets/icons/icon-not-visible.svg b/weave-js/src/assets/icons/icon-not-visible.svg
new file mode 100644
index 00000000000..766810c7811
--- /dev/null
+++ b/weave-js/src/assets/icons/icon-not-visible.svg
@@ -0,0 +1,5 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path
+        d="M21.3458 10.0012C21.7133 10.1923 21.8563 10.6452 21.6652 11.0127L20.9998 10.6667C21.6652 11.0127 21.6652 11.0127 21.6652 11.0127L21.6644 11.0142L21.6634 11.0162L21.6605 11.0216L21.6519 11.0377C21.6448 11.0508 21.6351 11.0685 21.6227 11.0906C21.5979 11.1347 21.5622 11.1963 21.5155 11.2727C21.422 11.4256 21.2838 11.6385 21.0984 11.8914C20.8469 12.2344 20.5068 12.653 20.0715 13.0959C19.9136 13.2565 19.9094 13.5154 20.0687 13.6747L21.2303 14.8363C21.5232 15.1292 21.5232 15.6041 21.2303 15.897C20.9374 16.1899 20.4625 16.1899 20.1696 15.897L18.8569 14.5843C18.7183 14.4457 18.5002 14.4286 18.3376 14.538C17.7094 14.961 16.9854 15.3543 16.1606 15.6706C15.9444 15.7535 15.8335 15.9974 15.9226 16.2112L16.4922 17.5782C16.6515 17.9605 16.4707 18.3997 16.0883 18.559C15.706 18.7183 15.2669 18.5375 15.1076 18.1551L14.4192 16.5031C14.3463 16.328 14.1606 16.229 13.9735 16.2596C13.3566 16.3605 12.6991 16.4167 11.9998 16.4167C11.3006 16.4167 10.6432 16.3605 10.0263 16.2596C9.8392 16.229 9.65352 16.3281 9.58058 16.5031L8.89224 18.1551C8.73293 18.5375 8.29382 18.7183 7.91147 18.559C7.52912 18.3997 7.34831 17.9605 7.50763 17.5782L8.07719 16.2112C8.16627 15.9975 8.0554 15.7536 7.83915 15.6706C7.0144 15.3543 6.29033 14.961 5.6621 14.5381C5.49952 14.4286 5.28141 14.4458 5.14283 14.5843L3.83018 15.897C3.53729 16.1899 3.06242 16.1899 2.76952 15.897C2.47663 15.6041 2.47663 15.1292 2.76952 14.8363L3.93106 13.6748C4.09033 13.5155 4.08616 13.2566 3.92825 13.096C3.49289 12.6531 3.15279 12.2344 2.90126 11.8914C2.71579 11.6385 2.57761 11.4256 2.48417 11.2727C2.43742 11.1963 2.40178 11.1347 2.37695 11.0906C2.36452 11.0685 2.3548 11.0508 2.34772 11.0377L2.33909 11.0216L2.33626 11.0162L2.33521 11.0142C2.33521 11.0142 2.33441 11.0127 2.99977 10.6667L2.33441 11.0127C2.14331 10.6452 2.28631 10.1923 2.65381 10.0012C3.02074 9.81044 3.47274 9.95271 3.66435 10.319L3.66722 10.3243C3.67038 10.3301 3.67603 10.3405 3.68422 10.355C3.70059 10.3841 3.72706 10.43 3.76391 10.4903C3.83765 10.6109 3.9526 10.7886 4.11087 11.0044C4.42796 11.4368 4.91547 12.0175 5.58952 12.5986C6.93141 13.7554 9.01343 14.9167 11.9998 14.9167C14.9862 14.9167 17.0682 13.7554 18.4101 12.5986C19.0842 12.0175 19.5717 11.4368 19.8888 11.0044C20.047 10.7886 20.162 10.6109 20.2357 10.4903C20.2726 10.43 20.299 10.3841 20.3154 10.355C20.3236 10.3405 20.3293 10.3301 20.3324 10.3243L20.3344 10.3206L20.335 10.3196C20.5266 9.95329 20.9789 9.81044 21.3458 10.0012Z"
+        fill="currentColor" />
+</svg>
\ No newline at end of file
diff --git a/weave-js/src/assets/icons/icon-pin-to-right.svg b/weave-js/src/assets/icons/icon-pin-to-right.svg
new file mode 100644
index 00000000000..1ae05ea52ae
--- /dev/null
+++ b/weave-js/src/assets/icons/icon-pin-to-right.svg
@@ -0,0 +1,5 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path fill-rule="evenodd" clip-rule="evenodd"
+        d="M3.01719 9.2399C2.06222 10.1952 2.52073 11.7288 3.65016 12.2008C5.16656 12.8346 6.60702 13.9748 7.45814 14.976C8.31598 15.9852 9.31862 17.6021 9.90729 19.2358C10.3377 20.4302 11.8963 20.8641 12.8429 19.9172L15.8021 16.9571L20.2201 21.3765C20.5129 21.6694 20.9876 21.6694 21.2804 21.3765C21.5732 21.0836 21.5732 20.6087 21.2804 20.3158L16.8624 15.8964L19.9117 12.8461C20.8582 11.8993 20.4245 10.3402 19.2305 9.90964C17.5973 9.32079 15.9809 8.31782 14.9721 7.45971C13.9712 6.60832 12.8313 5.16741 12.1977 3.65052C11.7259 2.52073 10.1927 2.06208 9.23776 3.01736L3.01719 9.2399ZM4.22824 10.8168C4.11611 10.7699 4.03833 10.6718 4.01204 10.5595C3.98693 10.4523 4.01377 10.3643 4.07752 10.3006L10.2981 4.07802C10.3618 4.01425 10.4498 3.9874 10.557 4.01252C10.6692 4.03882 10.7673 4.11663 10.8141 4.22879C11.5256 5.93216 12.4296 7.18928 14.0007 8.60243C15.5826 10.0253 16.9977 10.6991 18.722 11.3208C18.8244 11.3577 18.8927 11.4398 18.9167 11.5384C18.9399 11.634 18.9168 11.72 18.8514 11.7855L11.7825 18.8566C11.7171 18.922 11.6311 18.9451 11.5355 18.9219C11.437 18.8979 11.3549 18.8296 11.318 18.7272C10.6965 17.0023 10.0229 15.5868 8.60049 14.0043C7.18779 12.4327 5.93107 11.5284 4.22824 10.8168Z"
+        fill="currentColor" />
+</svg>
\ No newline at end of file
diff --git a/weave-js/src/components/Icon/Icon.tsx b/weave-js/src/components/Icon/Icon.tsx
index a87f41dfe56..9c6b8f1e054 100644
--- a/weave-js/src/components/Icon/Icon.tsx
+++ b/weave-js/src/components/Icon/Icon.tsx
@@ -141,6 +141,7 @@ import {ReactComponent as ImportMolecule} from '../../assets/icons/icon-molecule
 import {ReactComponent as ImportMusicAudio} from '../../assets/icons/icon-music-audio.svg';
 import {ReactComponent as ImportNewSectionAbove} from '../../assets/icons/icon-new-section-above.svg';
 import {ReactComponent as ImportNewSectionBelow} from '../../assets/icons/icon-new-section-below.svg';
+import {ReactComponent as ImportNotVisible} from '../../assets/icons/icon-not-visible.svg';
 import {ReactComponent as ImportNumber} from '../../assets/icons/icon-number.svg';
 import {ReactComponent as ImportOpenNewTab} from '../../assets/icons/icon-open-new-tab.svg';
 import {ReactComponent as ImportOpenaiLogo} from '../../assets/icons/icon-openai-logo.svg';
@@ -160,6 +161,7 @@ import {ReactComponent as ImportPaused} from '../../assets/icons/icon-paused.svg
 import {ReactComponent as ImportPencilEdit} from '../../assets/icons/icon-pencil-edit.svg';
 import {ReactComponent as ImportPhoto} from '../../assets/icons/icon-photo.svg';
 import {ReactComponent as ImportPin} from '../../assets/icons/icon-pin.svg';
+import {ReactComponent as ImportPinToRight} from '../../assets/icons/icon-pin-to-right.svg';
 import {ReactComponent as ImportPlay} from '../../assets/icons/icon-play.svg';
 import {ReactComponent as ImportPlotly} from '../../assets/icons/icon-plotly.svg';
 import {ReactComponent as ImportPriorityCritical} from '../../assets/icons/icon-priority-critical.svg';
@@ -698,6 +700,9 @@ export const IconNewSectionAbove = (props: SVGIconProps) => (
 export const IconNewSectionBelow = (props: SVGIconProps) => (
   <ImportNewSectionBelow {...updateIconProps(props)} />
 );
+export const IconNotVisible = (props: SVGIconProps) => (
+  <ImportNotVisible {...updateIconProps(props)} />
+);
 export const IconNumber = (props: SVGIconProps) => (
   <ImportNumber {...updateIconProps(props)} />
 );
@@ -755,6 +760,9 @@ export const IconPhoto = (props: SVGIconProps) => (
 export const IconPin = (props: SVGIconProps) => (
   <ImportPin {...updateIconProps(props)} />
 );
+export const IconPinToRight = (props: SVGIconProps) => (
+  <ImportPinToRight {...updateIconProps(props)} />
+);
 export const IconPlay = (props: SVGIconProps) => (
   <ImportPlay {...updateIconProps(props)} />
 );
@@ -1201,6 +1209,7 @@ const ICON_NAME_TO_ICON: Record<IconName, ElementType> = {
   'music-audio': IconMusicAudio,
   'new-section-above': IconNewSectionAbove,
   'new-section-below': IconNewSectionBelow,
+  'not-visible': IconNotVisible,
   number: IconNumber,
   'open-new-tab': IconOpenNewTab,
   'openai-logo': IconOpenaiLogo,
@@ -1220,6 +1229,7 @@ const ICON_NAME_TO_ICON: Record<IconName, ElementType> = {
   'pencil-edit': IconPencilEdit,
   photo: IconPhoto,
   pin: IconPin,
+  'pin-to-right': IconPinToRight,
   play: IconPlay,
   plotly: IconPlotly,
   'priority-critical': IconPriorityCritical,
diff --git a/weave-js/src/components/Icon/index.ts b/weave-js/src/components/Icon/index.ts
index aa92f74b31a..d061df41656 100644
--- a/weave-js/src/components/Icon/index.ts
+++ b/weave-js/src/components/Icon/index.ts
@@ -141,6 +141,7 @@ export {
   IconMusicAudio,
   IconNewSectionAbove,
   IconNewSectionBelow,
+  IconNotVisible,
   IconNumber,
   IconOpenaiLogo,
   IconOpenNewTab,
@@ -160,6 +161,7 @@ export {
   IconPencilEdit,
   IconPhoto,
   IconPin,
+  IconPinToRight,
   IconPlay,
   IconPlotly,
   IconPriorityCritical,
diff --git a/weave-js/src/components/Icon/types.ts b/weave-js/src/components/Icon/types.ts
index 8002b47eb9c..e9ea98d1c0e 100644
--- a/weave-js/src/components/Icon/types.ts
+++ b/weave-js/src/components/Icon/types.ts
@@ -140,6 +140,7 @@ export const IconNames = {
   MusicAudio: 'music-audio',
   NewSectionAbove: 'new-section-above',
   NewSectionBelow: 'new-section-below',
+  NotVisible: 'not-visible',
   Number: 'number',
   OpenNewTab: 'open-new-tab',
   OpenaiLogo: 'openai-logo',
@@ -159,6 +160,7 @@ export const IconNames = {
   PencilEdit: 'pencil-edit',
   Photo: 'photo',
   Pin: 'pin',
+  PinToRight: 'pin-to-right',
   Play: 'play',
   Plotly: 'plotly',
   PriorityCritical: 'priority-critical',
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx
index e6e62180f64..9f845ad0cd1 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx
@@ -29,7 +29,13 @@ import {
 import {MOON_200, TEAL_300} from '@wandb/weave/common/css/color.styles';
 import {Switch} from '@wandb/weave/components';
 import {Checkbox} from '@wandb/weave/components/Checkbox/Checkbox';
-import {Icon} from '@wandb/weave/components/Icon';
+import {
+  Icon,
+  IconNotVisible,
+  IconPinToRight,
+  IconSortAscending,
+  IconSortDescending,
+} from '@wandb/weave/components/Icon';
 import React, {
   FC,
   useCallback,
@@ -1069,6 +1075,13 @@ export const CallsTable: FC<{
           },
           columnMenu: CallsCustomColumnMenu,
           pagination: PaginationButtons,
+          columnMenuSortDescendingIcon: IconSortDescending,
+          columnMenuSortAscendingIcon: IconSortAscending,
+          columnMenuHideIcon: IconNotVisible,
+          columnMenuPinLeftIcon: () => (
+            <IconPinToRight style={{transform: 'scaleX(-1)'}} />
+          ),
+          columnMenuPinRightIcon: IconPinToRight,
         }}
       />
     </FilterLayoutTemplate>

From 563fe3a0cebc10ef949dc88bb40e6461e233584d Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Fri, 6 Dec 2024 13:14:40 -0600
Subject: [PATCH 04/52] chore(ui) add filled circle icon (#3163)

---
 weave-js/src/assets/icons/icon-filled-circle.svg | 3 +++
 weave-js/src/components/Icon/Icon.tsx            | 5 +++++
 weave-js/src/components/Icon/index.ts            | 1 +
 weave-js/src/components/Icon/types.ts            | 1 +
 4 files changed, 10 insertions(+)
 create mode 100644 weave-js/src/assets/icons/icon-filled-circle.svg

diff --git a/weave-js/src/assets/icons/icon-filled-circle.svg b/weave-js/src/assets/icons/icon-filled-circle.svg
new file mode 100644
index 00000000000..2b57c31558b
--- /dev/null
+++ b/weave-js/src/assets/icons/icon-filled-circle.svg
@@ -0,0 +1,3 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M19 12C19 15.866 15.866 19 12 19C8.13401 19 5 15.866 5 12C5 8.13401 8.13401 5 12 5C15.866 5 19 8.13401 19 12Z" fill="currentColor"/>
+</svg>
diff --git a/weave-js/src/components/Icon/Icon.tsx b/weave-js/src/components/Icon/Icon.tsx
index 9c6b8f1e054..948c8a456b4 100644
--- a/weave-js/src/components/Icon/Icon.tsx
+++ b/weave-js/src/components/Icon/Icon.tsx
@@ -74,6 +74,7 @@ import {ReactComponent as ImportExpandUncollapse} from '../../assets/icons/icon-
 import {ReactComponent as ImportExportShareUpload} from '../../assets/icons/icon-export-share-upload.svg';
 import {ReactComponent as ImportFacebookSocial} from '../../assets/icons/icon-facebook-social.svg';
 import {ReactComponent as ImportFailed} from '../../assets/icons/icon-failed.svg';
+import {ReactComponent as ImportFilledCircle} from '../../assets/icons/icon-filled-circle.svg';
 import {ReactComponent as ImportFilterAlt} from '../../assets/icons/icon-filter-alt.svg';
 import {ReactComponent as ImportFlashBolt} from '../../assets/icons/icon-flash-bolt.svg';
 import {ReactComponent as ImportFolderAlt} from '../../assets/icons/icon-folder-alt.svg';
@@ -499,6 +500,9 @@ export const IconFacebookSocial = (props: SVGIconProps) => (
 export const IconFailed = (props: SVGIconProps) => (
   <ImportFailed {...updateIconProps(props)} />
 );
+export const IconFilledCircle = (props: SVGIconProps) => (
+  <ImportFilledCircle {...updateIconProps(props)} />
+);
 export const IconFilterAlt = (props: SVGIconProps) => (
   <ImportFilterAlt {...updateIconProps(props)} />
 );
@@ -1142,6 +1146,7 @@ const ICON_NAME_TO_ICON: Record<IconName, ElementType> = {
   'export-share-upload': IconExportShareUpload,
   'facebook-social': IconFacebookSocial,
   failed: IconFailed,
+  'filled-circle': IconFilledCircle,
   'filter-alt': IconFilterAlt,
   'flash-bolt': IconFlashBolt,
   'folder-alt': IconFolderAlt,
diff --git a/weave-js/src/components/Icon/index.ts b/weave-js/src/components/Icon/index.ts
index d061df41656..81839a71019 100644
--- a/weave-js/src/components/Icon/index.ts
+++ b/weave-js/src/components/Icon/index.ts
@@ -74,6 +74,7 @@ export {
   IconExportShareUpload,
   IconFacebookSocial,
   IconFailed,
+  IconFilledCircle,
   IconFilterAlt,
   IconFlashBolt,
   IconFolderAlt,
diff --git a/weave-js/src/components/Icon/types.ts b/weave-js/src/components/Icon/types.ts
index e9ea98d1c0e..55f46c52833 100644
--- a/weave-js/src/components/Icon/types.ts
+++ b/weave-js/src/components/Icon/types.ts
@@ -73,6 +73,7 @@ export const IconNames = {
   ExportShareUpload: 'export-share-upload',
   FacebookSocial: 'facebook-social',
   Failed: 'failed',
+  FilledCircle: 'filled-circle',
   FilterAlt: 'filter-alt',
   FlashBolt: 'flash-bolt',
   FolderAlt: 'folder-alt',

From 3a94131f505869e4cd2d43152338050b6fd8b652 Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Fri, 6 Dec 2024 13:46:39 -0800
Subject: [PATCH 05/52] add batch processor (#3167)

---
 tests/trace/test_trace_server_common.py       | 24 +++++
 .../clickhouse_trace_server_batched.py        | 88 ++++++++-----------
 weave/trace_server/trace_server_common.py     | 28 ++++++
 3 files changed, 87 insertions(+), 53 deletions(-)

diff --git a/tests/trace/test_trace_server_common.py b/tests/trace/test_trace_server_common.py
index 9bc7495481f..d9170f83eee 100644
--- a/tests/trace/test_trace_server_common.py
+++ b/tests/trace/test_trace_server_common.py
@@ -1,4 +1,5 @@
 from weave.trace_server.trace_server_common import (
+    DynamicBatchProcessor,
     LRUCache,
     get_nested_key,
     set_nested_key,
@@ -54,3 +55,26 @@ def test_lru_cache():
     cache["c"] = 10
     assert cache["c"] == 10
     assert cache["d"] == 4
+
+
+def test_dynamic_batch_processor():
+    # Initialize processor with:
+    # - initial batch size of 2
+    # - max size of 8
+    # - growth factor of 2
+    processor = DynamicBatchProcessor(initial_size=2, max_size=8, growth_factor=2)
+
+    test_data = range(15)
+
+    batches = list(processor.make_batches(iter(test_data)))
+
+    # Expected batch sizes: 2, 4, 8, 1
+    assert batches[0] == [0, 1]
+    assert batches[1] == [2, 3, 4, 5]
+    assert batches[2] == [6, 7, 8, 9, 10, 11, 12, 13]
+    assert batches[3] == [14]
+    assert len(batches) == 4
+
+    # Verify all items were processed
+    flattened = [item for batch in batches for item in batch]
+    assert flattened == list(range(15))
diff --git a/weave/trace_server/clickhouse_trace_server_batched.py b/weave/trace_server/clickhouse_trace_server_batched.py
index 943ef31b0b8..77c61889899 100644
--- a/weave/trace_server/clickhouse_trace_server_batched.py
+++ b/weave/trace_server/clickhouse_trace_server_batched.py
@@ -95,6 +95,7 @@
     validate_cost_purge_req,
 )
 from weave.trace_server.trace_server_common import (
+    DynamicBatchProcessor,
     LRUCache,
     digest_is_version_like,
     empty_str_to_none,
@@ -343,68 +344,46 @@ def calls_query_stream(self, req: tsi.CallsQueryReq) -> Iterator[tsi.CallSchema]
         )
 
         select_columns = [c.field for c in cq.select_fields]
+        expand_columns = req.expand_columns or []
+        include_feedback = req.include_feedback or False
 
-        if not req.expand_columns and not req.include_feedback:
-            for row in raw_res:
-                yield tsi.CallSchema.model_validate(
-                    _ch_call_dict_to_call_schema_dict(dict(zip(select_columns, row)))
-                )
+        def row_to_call_schema_dict(row: tuple[Any, ...]) -> dict[str, Any]:
+            return _ch_call_dict_to_call_schema_dict(dict(zip(select_columns, row)))
 
-        else:
-            expand_columns = req.expand_columns or []
-            ref_cache = LRUCache(max_size=1000)
-
-            batch_size = 10
-            batch = []
+        if not expand_columns and not include_feedback:
             for row in raw_res:
-                call_dict = _ch_call_dict_to_call_schema_dict(
-                    dict(zip(select_columns, row))
+                yield tsi.CallSchema.model_validate(row_to_call_schema_dict(row))
+            return
+
+        ref_cache = LRUCache(max_size=1000)
+        batch_processor = DynamicBatchProcessor(
+            initial_size=10,
+            max_size=MAX_CALLS_STREAM_BATCH_SIZE,
+            growth_factor=10,
+        )
+
+        for batch in batch_processor.make_batches(raw_res):
+            call_dicts = [row_to_call_schema_dict(row) for row in batch]
+            if expand_columns:
+                self._expand_call_refs(
+                    req.project_id, call_dicts, expand_columns, ref_cache
                 )
-                batch.append(call_dict)
 
-                if len(batch) >= batch_size:
-                    hydrated_batch = self._hydrate_calls(
-                        req.project_id,
-                        batch,
-                        expand_columns,
-                        req.include_feedback or False,
-                        ref_cache,
-                    )
-                    for call in hydrated_batch:
-                        yield tsi.CallSchema.model_validate(call)
-
-                    # *** Dynamic increase from 10 to 500 ***
-                    batch_size = min(MAX_CALLS_STREAM_BATCH_SIZE, batch_size * 10)
-                    batch = []
-
-            hydrated_batch = self._hydrate_calls(
-                req.project_id,
-                batch,
-                expand_columns,
-                req.include_feedback or False,
-                ref_cache,
-            )
-            for call in hydrated_batch:
+            if include_feedback:
+                self._add_feedback_to_calls(req.project_id, call_dicts)
+
+            for call in call_dicts:
                 yield tsi.CallSchema.model_validate(call)
 
-    def _hydrate_calls(
-        self,
-        project_id: str,
-        calls: list[dict[str, Any]],
-        expand_columns: list[str],
-        include_feedback: bool,
-        ref_cache: LRUCache,
-    ) -> list[dict[str, Any]]:
+    def _add_feedback_to_calls(
+        self, project_id: str, calls: list[dict[str, Any]]
+    ) -> None:
         if len(calls) == 0:
-            return calls
-
-        self._expand_call_refs(project_id, calls, expand_columns, ref_cache)
-        if include_feedback:
-            feedback_query_req = make_feedback_query_req(project_id, calls)
-            feedback = self.feedback_query(feedback_query_req)
-            hydrate_calls_with_feedback(calls, feedback)
+            return
 
-        return calls
+        feedback_query_req = make_feedback_query_req(project_id, calls)
+        feedback = self.feedback_query(feedback_query_req)
+        hydrate_calls_with_feedback(calls, feedback)
 
     def _get_refs_to_resolve(
         self, calls: list[dict[str, Any]], expand_columns: list[str]
@@ -436,6 +415,9 @@ def _expand_call_refs(
         expand_columns: list[str],
         ref_cache: LRUCache,
     ) -> None:
+        if len(calls) == 0:
+            return
+
         # format expand columns by depth, iterate through each batch in order
         expand_column_by_depth = defaultdict(list)
         for col in expand_columns:
diff --git a/weave/trace_server/trace_server_common.py b/weave/trace_server/trace_server_common.py
index 0ff14d4396b..0691927bc47 100644
--- a/weave/trace_server/trace_server_common.py
+++ b/weave/trace_server/trace_server_common.py
@@ -1,6 +1,7 @@
 import copy
 import datetime
 from collections import OrderedDict, defaultdict
+from collections.abc import Iterator
 from typing import Any, Optional, cast
 
 from weave.trace_server import refs_internal as ri
@@ -170,6 +171,33 @@ def __setitem__(self, key: str, value: Any) -> None:
         super().__setitem__(key, value)
 
 
+class DynamicBatchProcessor:
+    """Helper class to handle dynamic batch processing with growing batch sizes."""
+
+    def __init__(self, initial_size: int, max_size: int, growth_factor: int):
+        self.batch_size = initial_size
+        self.max_size = max_size
+        self.growth_factor = growth_factor
+
+    def make_batches(self, iterator: Iterator[Any]) -> Iterator[list[Any]]:
+        batch = []
+
+        for item in iterator:
+            batch.append(item)
+
+            if len(batch) >= self.batch_size:
+                yield batch
+
+                batch = []
+                self.batch_size = self._compute_batch_size()
+
+        if batch:
+            yield batch
+
+    def _compute_batch_size(self) -> int:
+        return min(self.max_size, self.batch_size * self.growth_factor)
+
+
 def digest_is_version_like(digest: str) -> tuple[bool, int]:
     """
     Check if a digest is a version like string.

From 463e92daedfb718034c0e1ca1cb6a0067d4ee693 Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Fri, 6 Dec 2024 14:21:15 -0800
Subject: [PATCH 06/52] chore(weave): add with_new_client context for
 concurrent ch queries (#3168)

---
 .../clickhouse_trace_server_batched.py        | 31 ++++++++++++++++---
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/weave/trace_server/clickhouse_trace_server_batched.py b/weave/trace_server/clickhouse_trace_server_batched.py
index 77c61889899..0a8c4fcf079 100644
--- a/weave/trace_server/clickhouse_trace_server_batched.py
+++ b/weave/trace_server/clickhouse_trace_server_batched.py
@@ -382,7 +382,8 @@ def _add_feedback_to_calls(
             return
 
         feedback_query_req = make_feedback_query_req(project_id, calls)
-        feedback = self.feedback_query(feedback_query_req)
+        with self.with_new_client():
+            feedback = self.feedback_query(feedback_query_req)
         hydrate_calls_with_feedback(calls, feedback)
 
     def _get_refs_to_resolve(
@@ -430,9 +431,10 @@ def _expand_call_refs(
             if not refs_to_resolve:
                 continue
 
-            vals = self._refs_read_batch_within_project(
-                project_id, list(refs_to_resolve.values()), ref_cache
-            )
+            with self.with_new_client():
+                vals = self._refs_read_batch_within_project(
+                    project_id, list(refs_to_resolve.values()), ref_cache
+                )
             for ((i, col), ref), val in zip(refs_to_resolve.items(), vals):
                 if isinstance(val, dict) and "_ref" not in val:
                     val["_ref"] = ref.uri()
@@ -1503,6 +1505,7 @@ def completions_create(
     # Private Methods
     @property
     def ch_client(self) -> CHClient:
+        """Returns and creates (if necessary) the clickhouse client"""
         if not hasattr(self._thread_local, "ch_client"):
             self._thread_local.ch_client = self._mint_client()
         return self._thread_local.ch_client
@@ -1520,6 +1523,26 @@ def _mint_client(self) -> CHClient:
         client.database = self._database
         return client
 
+    @contextmanager
+    def with_new_client(self) -> Iterator[None]:
+        """Context manager to use a new client for operations.
+        Each call gets a fresh client with its own clickhouse session ID.
+
+        Usage:
+        ```
+        with self.with_new_client():
+            self.feedback_query(req)
+        ```
+        """
+        client = self._mint_client()
+        original_client = self.ch_client
+        self._thread_local.ch_client = client
+        try:
+            yield
+        finally:
+            self._thread_local.ch_client = original_client
+            client.close()
+
     # def __del__(self) -> None:
     #     self.ch_client.close()
 

From 30336d35c191dc0a9e97b29d356273829b26eaea Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Fri, 6 Dec 2024 16:26:41 -0600
Subject: [PATCH 07/52] chore(ui): fix warning about key attr spread (#3173)

---
 weave-js/src/components/FancyPage/FancyPageMenu.tsx | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/weave-js/src/components/FancyPage/FancyPageMenu.tsx b/weave-js/src/components/FancyPage/FancyPageMenu.tsx
index c5971756016..6829b70fc06 100644
--- a/weave-js/src/components/FancyPage/FancyPageMenu.tsx
+++ b/weave-js/src/components/FancyPage/FancyPageMenu.tsx
@@ -60,7 +60,6 @@ export const FancyPageMenu = ({
               return null;
             }
             const linkProps = {
-              key: menuItem.slug,
               to: menuItem.isDisabled
                 ? {}
                 : {
@@ -76,7 +75,7 @@ export const FancyPageMenu = ({
               },
             };
             return (
-              <Link {...linkProps}>
+              <Link key={menuItem.slug} {...linkProps}>
                 <DropdownMenu.Item {...menuItemProps}>
                   <Icon name={menuItem.iconName} />
                   {menuItem.nameTooltip || menuItem.name}

From b3766997391f5a403e70d06914aa93bcb022fa00 Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Fri, 6 Dec 2024 14:33:15 -0800
Subject: [PATCH 08/52] chore(weave): set default call stream batch size to 100
 (#3169)

---
 weave/trace_server/clickhouse_trace_server_batched.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/weave/trace_server/clickhouse_trace_server_batched.py b/weave/trace_server/clickhouse_trace_server_batched.py
index 0a8c4fcf079..d40d7bcc2a3 100644
--- a/weave/trace_server/clickhouse_trace_server_batched.py
+++ b/weave/trace_server/clickhouse_trace_server_batched.py
@@ -121,6 +121,7 @@
 FILE_CHUNK_SIZE = 100000
 
 MAX_DELETE_CALLS_COUNT = 100
+INITIAL_CALLS_STREAM_BATCH_SIZE = 100
 MAX_CALLS_STREAM_BATCH_SIZE = 500
 
 
@@ -357,7 +358,7 @@ def row_to_call_schema_dict(row: tuple[Any, ...]) -> dict[str, Any]:
 
         ref_cache = LRUCache(max_size=1000)
         batch_processor = DynamicBatchProcessor(
-            initial_size=10,
+            initial_size=INITIAL_CALLS_STREAM_BATCH_SIZE,
             max_size=MAX_CALLS_STREAM_BATCH_SIZE,
             growth_factor=10,
         )

From 059f38839e0aef48e34627728eb0a42275ced93e Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Fri, 6 Dec 2024 16:32:52 -0800
Subject: [PATCH 09/52] chore(weave): ui op selecter jump back to top fix
 (#3174)

---
 .../Browse3/pages/CallsPage/CallsTable.tsx    | 161 +++++++++++-------
 1 file changed, 95 insertions(+), 66 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx
index 9f845ad0cd1..18130d64341 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx
@@ -85,7 +85,10 @@ import {TraceCallSchema} from '../wfReactInterface/traceServerClientTypes';
 import {traceCallToUICallSchema} from '../wfReactInterface/tsDataModelHooks';
 import {EXPANDED_REF_REF_KEY} from '../wfReactInterface/tsDataModelHooksCallRefExpansion';
 import {objectVersionNiceString} from '../wfReactInterface/utilities';
-import {CallSchema} from '../wfReactInterface/wfDataModelHooksInterface';
+import {
+  CallSchema,
+  OpVersionSchema,
+} from '../wfReactInterface/wfDataModelHooksInterface';
 import {CallsCharts} from './CallsCharts';
 import {CallsCustomColumnMenu} from './CallsCustomColumnMenu';
 import {
@@ -743,71 +746,13 @@ export const CallsTable: FC<{
         <TailwindContents>
           <RefreshButton onClick={() => calls.refetch()} />
           {!hideOpSelector && (
-            <div className="flex-none">
-              <ListItem
-                sx={{minWidth: 190, width: 320, height: 32, padding: 0}}>
-                <FormControl fullWidth sx={{borderColor: MOON_200}}>
-                  <Autocomplete
-                    PaperComponent={paperProps => (
-                      <StyledPaper {...paperProps} />
-                    )}
-                    sx={{
-                      '& .MuiOutlinedInput-root': {
-                        height: '32px',
-                        '& fieldset': {
-                          borderColor: MOON_200,
-                        },
-                        '&:hover fieldset': {
-                          borderColor: `rgba(${TEAL_300}, 0.48)`,
-                        },
-                      },
-                      '& .MuiOutlinedInput-input': {
-                        height: '32px',
-                        padding: '0 14px',
-                        boxSizing: 'border-box',
-                      },
-                    }}
-                    size="small"
-                    // Temp disable multiple for simplicity - may want to re-enable
-                    // multiple
-                    limitTags={1}
-                    disabled={Object.keys(frozenFilter ?? {}).includes(
-                      'opVersions'
-                    )}
-                    value={selectedOpVersionOption}
-                    onChange={(event, newValue) => {
-                      if (newValue === ALL_TRACES_OR_CALLS_REF_KEY) {
-                        setFilter({
-                          ...filter,
-                          opVersionRefs: [],
-                        });
-                      } else {
-                        setFilter({
-                          ...filter,
-                          opVersionRefs: newValue ? [newValue] : [],
-                        });
-                      }
-                    }}
-                    renderInput={renderParams => (
-                      <StyledTextField
-                        {...renderParams}
-                        sx={{maxWidth: '350px'}}
-                      />
-                    )}
-                    getOptionLabel={option => {
-                      return opVersionOptions[option]?.title ?? 'loading...';
-                    }}
-                    disableClearable={
-                      selectedOpVersionOption === ALL_TRACES_OR_CALLS_REF_KEY
-                    }
-                    groupBy={option => opVersionOptions[option]?.group}
-                    options={Object.keys(opVersionOptions)}
-                    popupIcon={<Icon name="chevron-down" />}
-                    clearIcon={<Icon name="close" />}
-                  />
-                </FormControl>
-              </ListItem>
-            </div>
+            <OpSelector
+              frozenFilter={frozenFilter}
+              filter={filter}
+              setFilter={setFilter}
+              selectedOpVersionOption={selectedOpVersionOption}
+              opVersionOptions={opVersionOptions}
+            />
           )}
           {filterModel && setFilterModel && (
             <FilterPanel
@@ -1088,6 +1033,90 @@ export const CallsTable: FC<{
   );
 };
 
+const OpSelector = ({
+  frozenFilter,
+  filter,
+  setFilter,
+  selectedOpVersionOption,
+  opVersionOptions,
+}: {
+  frozenFilter: WFHighLevelCallFilter | undefined;
+  filter: WFHighLevelCallFilter;
+  setFilter: (state: WFHighLevelCallFilter) => void;
+  selectedOpVersionOption: string;
+  opVersionOptions: Record<
+    string,
+    {
+      title: string;
+      ref: string;
+      group: string;
+      objectVersion?: OpVersionSchema;
+    }
+  >;
+}) => {
+  const frozenOpFilter = Object.keys(frozenFilter ?? {}).includes('opVersions');
+  const handleChange = useCallback(
+    (event: any, newValue: string | null) => {
+      if (newValue === ALL_TRACES_OR_CALLS_REF_KEY) {
+        setFilter({
+          ...filter,
+          opVersionRefs: [],
+        });
+      } else {
+        setFilter({
+          ...filter,
+          opVersionRefs: newValue ? [newValue] : [],
+        });
+      }
+    },
+    [filter, setFilter]
+  );
+
+  return (
+    <div className="flex-none">
+      <ListItem sx={{minWidth: 190, width: 320, height: 32, padding: 0}}>
+        <FormControl fullWidth sx={{borderColor: MOON_200}}>
+          <Autocomplete
+            PaperComponent={paperProps => <StyledPaper {...paperProps} />}
+            sx={{
+              '& .MuiOutlinedInput-root': {
+                height: '32px',
+                '& fieldset': {
+                  borderColor: MOON_200,
+                },
+                '&:hover fieldset': {
+                  borderColor: `rgba(${TEAL_300}, 0.48)`,
+                },
+              },
+              '& .MuiOutlinedInput-input': {
+                height: '32px',
+                padding: '0 14px',
+                boxSizing: 'border-box',
+              },
+            }}
+            size="small"
+            limitTags={1}
+            disabled={frozenOpFilter}
+            value={selectedOpVersionOption}
+            onChange={handleChange}
+            renderInput={renderParams => (
+              <StyledTextField {...renderParams} sx={{maxWidth: '350px'}} />
+            )}
+            getOptionLabel={option => opVersionOptions[option]?.title ?? ''}
+            disableClearable={
+              selectedOpVersionOption === ALL_TRACES_OR_CALLS_REF_KEY
+            }
+            groupBy={option => opVersionOptions[option]?.group}
+            options={Object.keys(opVersionOptions)}
+            popupIcon={<Icon name="chevron-down" />}
+            clearIcon={<Icon name="close" />}
+          />
+        </FormControl>
+      </ListItem>
+    </div>
+  );
+};
+
 const ButtonDivider = () => (
   <div className="h-24 flex-none border-l-[1px] border-moon-250"></div>
 );

From 58de5178369ae305b1f12da75ae2b8bf47a63c01 Mon Sep 17 00:00:00 2001
From: Andrew Truong <itsandrewtruong@gmail.com>
Date: Sun, 8 Dec 2024 21:35:28 -0500
Subject: [PATCH 10/52] feat(weave): Add option to name weave evals (and give a
 memorable name if not specified) (#3135)

---
 tests/trace/test_evaluations.py | 22 ++++++++-
 tests/trace/util.py             |  7 +++
 weave/flow/eval.py              | 26 ++++++++--
 weave/flow/util.py              | 87 +++++++++++++++++++++++++++++++++
 weave/trace/op.py               | 24 ++++-----
 5 files changed, 150 insertions(+), 16 deletions(-)

diff --git a/tests/trace/test_evaluations.py b/tests/trace/test_evaluations.py
index f4993e9227d..e5c38ef0140 100644
--- a/tests/trace/test_evaluations.py
+++ b/tests/trace/test_evaluations.py
@@ -7,7 +7,7 @@
 from PIL import Image
 
 import weave
-from tests.trace.util import AnyIntMatcher
+from tests.trace.util import AnyIntMatcher, AnyStrMatcher
 from weave import Evaluation, Model
 from weave.scorers import Scorer
 from weave.trace.refs import CallRef
@@ -504,8 +504,8 @@ async def test_evaluation_data_topology(client):
             }
         },
         "weave": {
+            "display_name": AnyStrMatcher(),
             "latency_ms": AnyIntMatcher(),
-            "trace_name": "Evaluation.evaluate",
             "status": "success",
         },
     }
@@ -1029,3 +1029,21 @@ def my_second_scorer(text, output, model_output):
 
     with pytest.raises(ValueError, match="Both 'output' and 'model_output'"):
         evaluation = weave.Evaluation(dataset=ds, scorers=[my_second_scorer])
+
+
+@pytest.mark.asyncio
+async def test_evaluation_with_custom_name(client):
+    dataset = weave.Dataset(rows=[{"input": "hi", "output": "hello"}])
+    evaluation = weave.Evaluation(dataset=dataset, evaluation_name="wow-custom!")
+
+    @weave.op()
+    def model(input: str) -> str:
+        return "hmmm"
+
+    await evaluation.evaluate(model)
+
+    calls = list(client.get_calls(filter=tsi.CallsFilter(trace_roots_only=True)))
+    assert len(calls) == 1
+
+    call = calls[0]
+    assert call.display_name == "wow-custom!"
diff --git a/tests/trace/util.py b/tests/trace/util.py
index eb4c6002beb..beb651722e3 100644
--- a/tests/trace/util.py
+++ b/tests/trace/util.py
@@ -8,6 +8,13 @@ def client_is_sqlite(client):
     return isinstance(client.server._internal_trace_server, SqliteTraceServer)
 
 
+class AnyStrMatcher:
+    """Matches any string."""
+
+    def __eq__(self, other):
+        return isinstance(other, str)
+
+
 class AnyIntMatcher:
     """Matches any integer."""
 
diff --git a/weave/flow/eval.py b/weave/flow/eval.py
index 5f4a961f904..bf78dc06d85 100644
--- a/weave/flow/eval.py
+++ b/weave/flow/eval.py
@@ -5,9 +5,10 @@
 import time
 import traceback
 from collections.abc import Coroutine
+from datetime import datetime
 from typing import Any, Callable, Literal, Optional, Union, cast
 
-from pydantic import PrivateAttr
+from pydantic import PrivateAttr, model_validator
 from rich import print
 from rich.console import Console
 
@@ -16,6 +17,7 @@
 from weave.flow.dataset import Dataset
 from weave.flow.model import Model, get_infer_method
 from weave.flow.obj import Object
+from weave.flow.util import make_memorable_name
 from weave.scorers import (
     Scorer,
     _has_oldstyle_scorers,
@@ -28,7 +30,7 @@
 from weave.trace.env import get_weave_parallelism
 from weave.trace.errors import OpCallError
 from weave.trace.isinstance import weave_isinstance
-from weave.trace.op import Op, as_op, is_op
+from weave.trace.op import CallDisplayNameFunc, Op, as_op, is_op
 from weave.trace.vals import WeaveObject
 from weave.trace.weave_client import Call, get_ref
 
@@ -41,6 +43,12 @@
 )
 
 
+def default_evaluation_display_name(call: Call) -> str:
+    date = datetime.now().strftime("%Y-%m-%d")
+    unique_name = make_memorable_name()
+    return f"eval-{date}-{unique_name}"
+
+
 def async_call(func: Union[Callable, Op], *args: Any, **kwargs: Any) -> Coroutine:
     is_async = False
     if is_op(func):
@@ -116,9 +124,21 @@ def function_to_evaluate(question: str):
     preprocess_model_input: Optional[Callable] = None
     trials: int = 1
 
+    # Custom evaluation name for display in the UI.  This is the same API as passing a
+    # custom `call_display_name` to `weave.op` (see that for more details).
+    evaluation_name: Optional[Union[str, CallDisplayNameFunc]] = None
+
     # internal attr to track whether to use the new `output` or old `model_output` key for outputs
     _output_key: Literal["output", "model_output"] = PrivateAttr("output")
 
+    @model_validator(mode="after")
+    def _update_display_name(self) -> "Evaluation":
+        if self.evaluation_name:
+            # Treat user-specified `evaluation_name` as the name for `Evaluation.evaluate`
+            eval_op = as_op(self.evaluate)
+            eval_op.call_display_name = self.evaluation_name
+        return self
+
     def model_post_init(self, __context: Any) -> None:
         scorers: list[Union[Callable, Scorer, Op]] = []
         for scorer in self.scorers or []:
@@ -486,7 +506,7 @@ async def eval_example(example: dict) -> dict:
             eval_rows.append(eval_row)
         return EvaluationResults(rows=weave.Table(eval_rows))
 
-    @weave.op()
+    @weave.op(call_display_name=default_evaluation_display_name)
     async def evaluate(self, model: Union[Callable, Model]) -> dict:
         # The need for this pattern is quite unfortunate and highlights a gap in our
         # data model. As a user, I just want to pass a list of data `eval_rows` to
diff --git a/weave/flow/util.py b/weave/flow/util.py
index 4d89e777d88..ba35d5ebe4a 100644
--- a/weave/flow/util.py
+++ b/weave/flow/util.py
@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import multiprocessing
+import random
 from collections.abc import AsyncIterator, Awaitable, Iterable
 from typing import Any, Callable, TypeVar
 
@@ -81,3 +82,89 @@ def warn_once(logger: logging.Logger, message: str) -> None:
     if message not in _shown_warnings:
         logger.warning(message)
         _shown_warnings.add(message)
+
+
+def make_memorable_name() -> str:
+    adjectives = [
+        "brave",
+        "bright",
+        "calm",
+        "charming",
+        "clever",
+        "daring",
+        "dazzling",
+        "eager",
+        "elegant",
+        "eloquent",
+        "fierce",
+        "friendly",
+        "gentle",
+        "graceful",
+        "happy",
+        "honest",
+        "imaginative",
+        "innocent",
+        "joyful",
+        "jubilant",
+        "keen",
+        "kind",
+        "lively",
+        "loyal",
+        "merry",
+        "nice",
+        "noble",
+        "optimistic",
+        "proud",
+        "quiet",
+        "rich",
+        "sweet",
+        "tender",
+        "unique",
+        "wise",
+        "zealous",
+    ]
+
+    nouns = [
+        "bear",
+        "bird",
+        "breeze",
+        "cedar",
+        "cloud",
+        "daisy",
+        "dawn",
+        "dolphin",
+        "dusk",
+        "eagle",
+        "fish",
+        "flower",
+        "forest",
+        "hill",
+        "horizon",
+        "island",
+        "lake",
+        "lion",
+        "maple",
+        "meadow",
+        "moon",
+        "mountain",
+        "oak",
+        "ocean",
+        "pine",
+        "plateau",
+        "rain",
+        "river",
+        "rose",
+        "star",
+        "stream",
+        "sun",
+        "tiger",
+        "tree",
+        "valley",
+        "whale",
+        "wind",
+        "wolf",
+    ]
+
+    adj = random.choice(adjectives)
+    noun = random.choice(nouns)
+    return f"{adj}-{noun}"
diff --git a/weave/trace/op.py b/weave/trace/op.py
index 5e33e8bdbf8..2b5835474d8 100644
--- a/weave/trace/op.py
+++ b/weave/trace/op.py
@@ -107,16 +107,15 @@ def _apply_fn_defaults_to_inputs(
 ) -> dict[str, Any]:
     inputs = {**inputs}
     sig = inspect.signature(fn)
-    for param_name, param in sig.parameters.items():
-        if param_name not in inputs:
-            if param.default != inspect.Parameter.empty and not _value_is_sentinel(
-                param
-            ):
-                inputs[param_name] = param.default
-            if param.kind == inspect.Parameter.VAR_POSITIONAL:
-                inputs[param_name] = ()
-            elif param.kind == inspect.Parameter.VAR_KEYWORD:
-                inputs[param_name] = {}
+    for name, param in sig.parameters.items():
+        if name in inputs:
+            continue
+        if param.default != inspect.Parameter.empty and not _value_is_sentinel(param):
+            inputs[name] = param.default
+        if param.kind == inspect.Parameter.VAR_POSITIONAL:
+            inputs[name] = ()
+        if param.kind == inspect.Parameter.VAR_KEYWORD:
+            inputs[name] = {}
     return inputs
 
 
@@ -216,6 +215,7 @@ def _default_on_input_handler(func: Op, args: tuple, kwargs: dict) -> ProcessedI
         inputs = sig.bind(*args, **kwargs).arguments
     except TypeError as e:
         raise OpCallError(f"Error calling {func.name}: {e}")
+
     inputs_with_defaults = _apply_fn_defaults_to_inputs(func, inputs)
     return ProcessedInputs(
         original_args=args,
@@ -736,7 +736,9 @@ def as_op(fn: Callable) -> Op:
     if not is_op(fn):
         raise ValueError("fn must be a weave.op() decorated function")
 
-    return cast(Op, fn)
+    # The unbinding is necessary for methods because `MethodType` is applied after the
+    # func is decorated into an Op.
+    return maybe_unbind_method(cast(Op, fn))
 
 
 __docspec__ = [call, calls]

From 505bdfdec1291ade698b15431d3741fbd4b76c85 Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Mon, 9 Dec 2024 09:32:51 -0600
Subject: [PATCH 11/52] chore(ui): Add TooltipDeprecated (#3176)

---
 weave-js/src/components/TooltipDeprecated.tsx | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 weave-js/src/components/TooltipDeprecated.tsx

diff --git a/weave-js/src/components/TooltipDeprecated.tsx b/weave-js/src/components/TooltipDeprecated.tsx
new file mode 100644
index 00000000000..1b600a6589c
--- /dev/null
+++ b/weave-js/src/components/TooltipDeprecated.tsx
@@ -0,0 +1,36 @@
+/**
+ * @deprecated Don't use this in any new code, we're trying to get rid of semantic-ui.
+ */
+import {Popup} from 'semantic-ui-react';
+import styled from 'styled-components';
+
+import {
+  hexToRGB,
+  MOON_650,
+  MOON_800,
+  OBLIVION,
+  WHITE,
+} from '../common/css/globals.styles';
+
+export const TooltipDeprecated = styled(Popup).attrs({
+  basic: true, // This removes the pointing arrow.
+  mouseEnterDelay: 500,
+  popperModifiers: {
+    preventOverflow: {
+      // Prevent popper from erroneously constraining the popup.
+      // Without this, tooltips in single row table cells get positioned under the cursor,
+      // causing them to immediately close.
+      boundariesElement: 'viewport',
+    },
+  },
+})`
+  && {
+    color: ${WHITE};
+    background: ${MOON_800};
+    border-color: ${MOON_650};
+    box-shadow: 0px 4px 6px ${hexToRGB(OBLIVION, 0.2)};
+    font-size: 14px;
+    line-height: 140%;
+    max-width: 300px;
+  }
+`;

From 78007556ff606650f1dc65f0bcf4d09b7f74d65e Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Mon, 9 Dec 2024 10:02:15 -0600
Subject: [PATCH 12/52] chore(ui): pluralize calls link (#3178)

---
 .../PagePanelComponents/Home/Browse3/pages/common/Links.tsx   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
index 8a11252bc33..be78bb51367 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
@@ -9,6 +9,7 @@ import {Link as LinkComp, useHistory} from 'react-router-dom';
 import styled, {css} from 'styled-components';
 
 import {TargetBlank} from '../../../../../../common/util/links';
+import {maybePluralizeWord} from '../../../../../../core/util/string';
 import {
   FEEDBACK_EXPAND_PARAM,
   PATH_PARAM,
@@ -414,7 +415,8 @@ export const CallsLink: React.FC<{
       $variant={props.variant}
       to={router.callsUIUrl(props.entity, props.project, props.filter)}>
       {props.callCount}
-      {props.countIsLimited ? '+' : ''} calls
+      {props.countIsLimited ? '+' : ''}{' '}
+      {maybePluralizeWord(props.callCount, 'call')}
     </Link>
   );
 };

From 0176050f5b01f44fa3c667a0adc853c4bce727f1 Mon Sep 17 00:00:00 2001
From: Martin Mark <martin.mark@wandb.com>
Date: Mon, 9 Dec 2024 12:17:15 -0500
Subject: [PATCH 13/52] chore(ui): Playground design tweaks (header, buttons)
 (#3172)

* White background header bar

* Message options: alignment

* Hide headerContent if not shown - fixes 16px of dead space

* Pointer events / positioning

* Padding fixes

* MessagePanel sizing issue

* Response sizing and style tweaks

* Lint

* Header spacing

* Message card spacing and buttons

* Lint

* Simplified showing and hiding the buttons

* Lint
---
 .../Home/Browse3/pages/ChatView/ChatView.tsx  |   6 +-
 .../Browse3/pages/ChatView/MessagePanel.tsx   | 220 +++++++++---------
 .../PlaygroundMessagePanelButtons.tsx         |   2 +-
 .../ChatView/PlaygroundMessagePanelEditor.tsx |   9 +-
 .../PlaygroundChat/PlaygroundChat.tsx         |  14 +-
 .../PlaygroundChat/PlaygroundChatTopBar.tsx   |   1 -
 .../Browse3/pages/common/SimplePageLayout.tsx |  26 ++-
 7 files changed, 143 insertions(+), 135 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChatView.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChatView.tsx
index 90e6e591e5b..10818aa020e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChatView.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChatView.tsx
@@ -27,16 +27,14 @@ export const ChatView = ({chat}: ChatViewProps) => {
 
   return (
     <div className="flex flex-col pb-32">
-      <span className="mb-[-16px] text-sm font-semibold text-moon-800">
-        Messages
-      </span>
+      <p className="mb-[8px] text-sm font-semibold text-moon-800">Messages</p>
       <MessageList
         messages={chat.request?.messages || []}
         scrollLastMessage={scrollLastMessage}
       />
       {chatResult?.choices && chatResult.choices.length > 0 && (
         <>
-          <span className="-mb-16 mt-24 text-sm font-semibold text-moon-800">
+          <span className="mb-[8px] text-sm font-semibold text-moon-800">
             Response
           </span>
           <div ref={outputRef}>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
index c9ed9313f59..cc1911b60d4 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
@@ -33,7 +33,6 @@ export const MessagePanel = ({
 }: MessagePanelProps) => {
   const [isShowingMore, setIsShowingMore] = useState(false);
   const [isOverflowing, setIsOverflowing] = useState(false);
-  const [isHovering, setIsHovering] = useState(false);
   const [editorHeight, setEditorHeight] = useState<number | null>(
     pendingToolResponseId ? 100 : null
   );
@@ -62,126 +61,127 @@ export const MessagePanel = ({
     : undefined;
 
   return (
-    <div className={classNames('flex gap-8', {'mt-24': !isTool})}>
-      {!isNested && !isSystemPrompt && (
-        <div className="w-32 flex-shrink-0">
-          {!isUser && !isTool && (
-            <Callout
-              size="x-small"
-              icon="robot-service-member"
-              color="moon"
-              className="h-32 w-32"
-            />
-          )}
-        </div>
-      )}
+    <div
+      className={classNames('group', {
+        'mb-[24px]': !isNested,
+        'mb-[0]': isNested,
+      })}>
+      <div className="flex gap-[16px]">
+        {!isNested && !isSystemPrompt && (
+          <div className="w-32 flex-shrink-0">
+            {!isUser && !isTool && (
+              <Callout
+                size="x-small"
+                icon="robot-service-member"
+                color="moon"
+                className="mt-[4px] h-32 w-32 bg-moon-100"
+              />
+            )}
+          </div>
+        )}
 
-      <div
-        className={classNames('relative w-full overflow-visible', {
-          'rounded-lg': !isNested,
-          'border-t border-moon-250': isTool,
-          'bg-moon-100': isSystemPrompt || hasToolCalls,
-          'bg-cactus-300/[0.24]': isUser,
-          'max-w-full': !isUser,
-          'max-w-[768px]': isUser,
-          'ml-auto py-8': isUser,
-          'mr-auto pb-8 pt-4': !isUser,
-          'py-8': hasContent,
-        })}
-        onMouseEnter={() => setIsHovering(true)}
-        onMouseLeave={() => setIsHovering(false)}>
-        <div>
-          {isSystemPrompt && (
-            <div className="flex justify-between px-16">
-              <div className="text-moon-500">
-                {message.role.charAt(0).toUpperCase() + message.role.slice(1)}
+        <div
+          className={classNames('relative w-full overflow-visible', {
+            'rounded-lg': !isNested,
+            'border-t border-moon-250': isTool,
+            'bg-moon-100': isSystemPrompt || hasToolCalls,
+            'bg-cactus-300/[0.24]': isUser,
+            'max-w-full': !isUser,
+            'max-w-[768px]': isUser,
+            'ml-auto': isUser,
+            'mr-auto': !isUser,
+            'py-[16px]': hasContent,
+            'pb-[4px] pt-[8px]': !isUser && !isTool && !isSystemPrompt,
+          })}>
+          <div>
+            {isSystemPrompt && (
+              <div className="flex justify-between px-[16px]">
+                <div className="text-sm text-moon-500">
+                  {message.role.charAt(0).toUpperCase() + message.role.slice(1)}
+                </div>
               </div>
-            </div>
-          )}
+            )}
 
-          {isTool && (
-            <div className={classNames('pb-8')}>
-              <div className="text-[14px] font-semibold text-moon-500">
-                Response
+            {isTool && (
+              <div className={classNames('pb-8')}>
+                <div className="text-[14px] font-semibold text-moon-500">
+                  Response
+                </div>
               </div>
+            )}
+
+            <div
+              ref={contentRef}
+              className={classNames('w-full overflow-y-hidden', {
+                'max-h-[400px]': !isShowingMore,
+                'max-h-full': isShowingMore,
+              })}>
+              {isPlayground && editorHeight ? (
+                <PlaygroundMessagePanelEditor
+                  message={message}
+                  index={index}
+                  isChoice={isChoice ?? false}
+                  editorHeight={editorHeight}
+                  isNested={isNested ?? false}
+                  pendingToolResponseId={pendingToolResponseId}
+                  setEditorHeight={setEditorHeight}
+                />
+              ) : (
+                <>
+                  {hasContent && (
+                    <div
+                      className={classNames(hasToolCalls ? 'pb-8' : '', {
+                        'px-16': isSystemPrompt || isUser,
+                      })}>
+                      {_.isString(message.content) ? (
+                        <MessagePanelPart
+                          value={message.content}
+                          isStructuredOutput={isStructuredOutput}
+                        />
+                      ) : (
+                        message.content!.map((p, i) => (
+                          <MessagePanelPart key={i} value={p} />
+                        ))
+                      )}
+                    </div>
+                  )}
+                  {hasToolCalls && (
+                    <div
+                      className={classNames({
+                        'border-t border-moon-250 pt-8': hasContent,
+                      })}>
+                      <ToolCalls toolCalls={message.tool_calls!} />
+                    </div>
+                  )}
+                </>
+              )}
             </div>
-          )}
 
-          <div
-            ref={contentRef}
-            className={classNames('w-full overflow-y-hidden', {
-              'max-h-[400px]': !isShowingMore,
-              'max-h-full': isShowingMore,
-            })}>
-            {isPlayground && editorHeight ? (
-              <PlaygroundMessagePanelEditor
-                message={message}
-                index={index}
-                isChoice={isChoice ?? false}
-                editorHeight={editorHeight}
-                isNested={isNested ?? false}
-                pendingToolResponseId={pendingToolResponseId}
-                setEditorHeight={setEditorHeight}
+            {isOverflowing && !editorHeight && (
+              <ShowMoreButton
+                isUser={isUser}
+                isShowingMore={isShowingMore}
+                setIsShowingMore={setIsShowingMore}
               />
-            ) : (
-              <>
-                {hasContent && (
-                  <div
-                    className={classNames(hasToolCalls ? 'pb-8' : '', {
-                      'px-16': isSystemPrompt || isUser,
-                    })}>
-                    {_.isString(message.content) ? (
-                      <MessagePanelPart
-                        value={message.content}
-                        isStructuredOutput={isStructuredOutput}
-                      />
-                    ) : (
-                      message.content!.map((p, i) => (
-                        <MessagePanelPart key={i} value={p} />
-                      ))
-                    )}
-                  </div>
-                )}
-                {hasToolCalls && (
-                  <div
-                    className={classNames({
-                      'border-t border-moon-250 pt-8': hasContent,
-                    })}>
-                    <ToolCalls toolCalls={message.tool_calls!} />
-                  </div>
-                )}
-              </>
             )}
           </div>
-
-          {isOverflowing && !editorHeight && (
-            <ShowMoreButton
-              isUser={isUser}
-              isShowingMore={isShowingMore}
-              setIsShowingMore={setIsShowingMore}
-            />
-          )}
-
-          {/* Playground buttons (retry, edit, delete) */}
-          {isPlayground && isHovering && !editorHeight && (
-            <div
-              className={classNames(
-                'absolute -bottom-[24px] right-[4px] flex w-full items-center justify-start',
-                isNested ? 'bottom-0' : 'bottom-[-32px]'
-              )}>
-              <PlaygroundMessagePanelButtons
-                index={message.original_index ?? index}
-                isChoice={isChoice ?? false}
-                isTool={isTool}
-                hasContent={hasContent}
-                contentRef={contentRef}
-                setEditorHeight={setEditorHeight}
-                responseIndexes={responseIndexes}
-              />
-            </div>
-          )}
         </div>
       </div>
+
+      {/* Playground buttons (retry, edit, delete) - using group and group-hover to control opacity. */}
+      {isPlayground && !editorHeight ? (
+        <div className="flex w-full items-center justify-start opacity-0 group-hover:opacity-100">
+          <PlaygroundMessagePanelButtons
+            index={message.original_index ?? index}
+            isChoice={isChoice ?? false}
+            isTool={isTool}
+            hasContent={hasContent}
+            contentRef={contentRef}
+            setEditorHeight={setEditorHeight}
+            responseIndexes={responseIndexes}
+          />
+        </div>
+      ) : null}
     </div>
   );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelButtons.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelButtons.tsx
index e491c38ede7..771a9d4075f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelButtons.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelButtons.tsx
@@ -27,7 +27,7 @@ export const PlaygroundMessagePanelButtons: React.FC<
   const {deleteMessage, deleteChoice, retry} = usePlaygroundContext();
 
   return (
-    <div className="z-10 ml-auto flex gap-4 rounded-lg border border-moon-250 bg-white p-4">
+    <div className="ml-auto flex gap-4 rounded-lg pt-[8px]">
       <Button
         variant="quiet"
         size="small"
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx
index aa12cacb027..746b033579a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx
@@ -65,14 +65,19 @@ export const PlaygroundMessagePanelEditor: React.FC<
   };
 
   return (
-    <div className={classNames('w-full pt-16', isNested ? 'px-2' : 'px-16')}>
+    <div
+      className={classNames(
+        'w-full pt-[6px]',
+        isNested ? 'px-[4px]' : 'px-[8px]'
+      )}>
       <StyledTextArea
         value={editedContent}
         onChange={e => setEditedContent(e.target.value)}
         autoGrow
         maxHeight={160}
       />
-      <div className="z-100 mt-8 flex justify-end gap-8">
+      {/* 6px vs. 8px to make up for extra padding from textarea field */}
+      <div className="z-100 mt-[6px] flex justify-end gap-[8px]">
         <Button variant="quiet" size="medium" onClick={handleCancel}>
           Cancel
         </Button>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
index b5f675633fe..94cd3c17644 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
@@ -115,11 +115,15 @@ export const PlaygroundChat = ({
               }}>
               <Box
                 sx={{
+                  backgroundColor: 'white',
+                  borderBottom: `1px solid ${MOON_200}`,
                   position: 'absolute',
-                  top: '8px',
-                  width: 'calc(100% - 32px)',
-                  left: '16px',
-                  right: '16px',
+                  top: '0',
+                  width: '100%',
+                  paddingTop: '8px',
+                  paddingBottom: '8px',
+                  paddingLeft: '16px',
+                  paddingRight: '16px',
                   zIndex: 10,
                 }}>
                 <PlaygroundChatTopBar
@@ -143,7 +147,7 @@ export const PlaygroundChat = ({
                   flexGrow: 1,
                 }}>
                 <Tailwind>
-                  <div className=" mx-auto h-full min-w-[400px] max-w-[800px] pb-8">
+                  <div className=" mx-auto mt-[32px] h-full min-w-[400px] max-w-[800px] pb-8">
                     {state.traceCall && (
                       <PlaygroundContext.Provider
                         value={{
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChatTopBar.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChatTopBar.tsx
index 8b49483a4ed..e00f1a02b2c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChatTopBar.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChatTopBar.tsx
@@ -110,7 +110,6 @@ export const PlaygroundChatTopBar: React.FC<PlaygroundChatTopBarProps> = ({
         width: '100%',
         display: 'flex',
         justifyContent: 'space-between',
-        paddingBottom: '8px',
       }}>
       <Box
         sx={{
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx
index e8965d3f4a2..9d2c6ab9718 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx
@@ -313,18 +313,20 @@ const SimpleTabView: FC<{
         height: '100%',
         overflow: 'hidden',
       }}>
-      <Box
-        sx={{
-          maxHeight: '50%',
-          flex: '0 0 auto',
-          width: '100%',
-          overflow: 'auto',
-          pt: 1,
-          px: 2,
-          alignContent: 'center',
-        }}>
-        {props.headerContent}
-      </Box>
+      {props.headerContent && (
+        <Box
+          sx={{
+            maxHeight: '50%',
+            flex: '0 0 auto',
+            width: '100%',
+            overflow: 'auto',
+            pt: 1,
+            px: 2,
+            alignContent: 'center',
+          }}>
+          {props.headerContent}
+        </Box>
+      )}
       {(!props.hideTabsIfSingle || props.tabs.length > 1) && (
         <Tabs.Root
           style={{margin: '12px 16px 0 16px'}}

From aeb29fcbf9d6abfaa81ab362c86e1963bc35e689 Mon Sep 17 00:00:00 2001
From: Martin Mark <martin.mark@wandb.com>
Date: Mon, 9 Dec 2024 12:44:29 -0500
Subject: [PATCH 14/52] chore(ui): Update metadata header styling (#3049)

* Object metadata to Tailwind / refactor

* Updated versions link and styling

* Updated OpsVersionLink

* Updated CallsLink to include next icon

* Lint

* Better flow for column sizes

* Lint

* No more colons
---
 .../Home/Browse3/pages/ObjectVersionPage.tsx  | 83 ++++++++--------
 .../Home/Browse3/pages/OpVersionPage.tsx      | 98 ++++++++++++-------
 .../Home/Browse3/pages/common/Links.tsx       | 22 +++--
 3 files changed, 119 insertions(+), 84 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx
index 6108ed5407e..085587a64d8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx
@@ -211,47 +211,50 @@ const ObjectVersionPageInner: React.FC<{
         </Tailwind>
       }
       headerContent={
-        <SimpleKeyValueTable
-          data={{
-            [refExtra ? 'Parent Object' : 'Name']: (
-              <>
-                {objectName}{' '}
-                {objectVersions.loading ? (
-                  <LoadingDots />
-                ) : (
-                  <>
-                    [
-                    <ObjectVersionsLink
-                      entity={entityName}
-                      project={projectName}
-                      filter={{
-                        objectName,
-                      }}
-                      versionCount={objectVersionCount}
-                      neverPeek
-                      variant="secondary"
+        <Tailwind>
+          <div className="grid w-full auto-cols-max grid-flow-col gap-[16px] text-[14px]">
+            <div className="block">
+              <p className="text-moon-500">Name</p>
+              <div className="flex items-center">
+                <ObjectVersionsLink
+                  entity={entityName}
+                  project={projectName}
+                  filter={{objectName}}
+                  versionCount={objectVersionCount}
+                  neverPeek
+                  variant="secondary">
+                  <div className="group flex items-center font-semibold">
+                    <span>{objectName}</span>
+                    {objectVersions.loading ? (
+                      <LoadingDots />
+                    ) : (
+                      <span className="ml-[4px]">
+                        ({objectVersionCount} version
+                        {objectVersionCount !== 1 ? 's' : ''})
+                      </span>
+                    )}
+                    <Icon
+                      name="forward-next"
+                      width={16}
+                      height={16}
+                      className="ml-[2px] opacity-0 group-hover:opacity-100"
                     />
-                    ]
-                  </>
-                )}
-              </>
-            ),
-            Version: <>{objectVersionIndex}</>,
-            ...(refExtra
-              ? {
-                  Subpath: refExtra,
-                }
-              : {}),
-            // 'Type Version': (
-            //   <TypeVersionLink
-            //     entityName={entityName}
-            //     projectName={projectName}
-            //     typeName={typeName}
-            //     version={typeVersionHash}
-            //   />
-            // ),
-          }}
-        />
+                  </div>
+                </ObjectVersionsLink>
+              </div>
+            </div>
+            <div className="block">
+              <p className="text-moon-500">Version</p>
+              <p>{objectVersionIndex}</p>
+            </div>
+            {refExtra && (
+              <div className="block">
+                <p className="text-moon-500">Subpath</p>
+                <p>{refExtra}</p>
+              </div>
+            )}
+          </div>
+        </Tailwind>
       }
       // menuItems={[
       //   {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/OpVersionPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/OpVersionPage.tsx
index 1a6e4afc577..36f4e44afc5 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/OpVersionPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/OpVersionPage.tsx
@@ -1,5 +1,6 @@
 import React, {useMemo} from 'react';
 
+import {Icon} from '../../../../Icon';
 import {LoadingDots} from '../../../../LoadingDots';
 import {Tailwind} from '../../../../Tailwind';
 import {NotFoundPanel} from '../NotFoundPanel';
@@ -13,7 +14,6 @@ import {
 import {CenteredAnimatedLoader} from './common/Loader';
 import {
   ScrollableTabContent,
-  SimpleKeyValueTable,
   SimplePageLayoutWithHeader,
 } from './common/SimplePageLayout';
 import {TabUseOp} from './TabUseOp';
@@ -75,49 +75,71 @@ const OpVersionPageInner: React.FC<{
     <SimplePageLayoutWithHeader
       title={opVersionText(opId, versionIndex)}
       headerContent={
-        <SimpleKeyValueTable
-          data={{
-            Name: (
-              <>
-                {opId}{' '}
-                {opVersions.loading ? (
-                  <LoadingDots />
-                ) : (
-                  <>
-                    [
-                    <OpVersionsLink
-                      entity={entity}
-                      project={project}
-                      filter={{
-                        opName: opId,
-                      }}
-                      versionCount={opVersionCount}
-                      neverPeek
-                      variant="secondary"
-                    />
-                    ]
-                  </>
-                )}
-              </>
-            ),
-            Version: <>{versionIndex}</>,
-            Calls:
-              !callsStats.loading || opVersionCallCount > 0 ? (
-                <CallsLink
+        <Tailwind>
+          <div className="grid w-full auto-cols-max grid-flow-col gap-[16px] text-[14px]">
+            <div className="block">
+              <p className="text-moon-500">Name</p>
+              <div className="flex items-center">
+                <OpVersionsLink
                   entity={entity}
                   project={project}
-                  callCount={opVersionCallCount}
                   filter={{
-                    opVersionRefs: [uri],
+                    opName: opId,
                   }}
+                  versionCount={opVersionCount}
                   neverPeek
-                  variant="secondary"
-                />
+                  variant="secondary">
+                  <div className="group flex items-center font-semibold">
+                    <span>{opId}</span>
+                    {opVersions.loading ? (
+                      <LoadingDots />
+                    ) : (
+                      <span className="ml-[4px]">
+                        ({opVersionCount} version
+                        {opVersionCount !== 1 ? 's' : ''})
+                      </span>
+                    )}
+                    <Icon
+                      name="forward-next"
+                      width={16}
+                      height={16}
+                      className="ml-[2px] opacity-0 group-hover:opacity-100"
+                    />
+                  </div>
+                </OpVersionsLink>
+              </div>
+            </div>
+            <div className="block">
+              <p className="text-moon-500">Version</p>
+              <p>{versionIndex}</p>
+            </div>
+            <div className="block">
+              <p className="text-moon-500">Calls:</p>
+              {!callsStats.loading || opVersionCallCount > 0 ? (
+                <div className="group flex w-max items-center">
+                  <CallsLink
+                    entity={entity}
+                    project={project}
+                    callCount={opVersionCallCount}
+                    filter={{
+                      opVersionRefs: [uri],
+                    }}
+                    neverPeek
+                    variant="secondary"
+                  />
+                  <Icon
+                    name="forward-next"
+                    width={16}
+                    height={16}
+                    className="ml-[2px] text-teal-500 opacity-0 hover:hidden group-hover:opacity-100"
+                  />
+                </div>
               ) : (
-                <></>
-              ),
-          }}
-        />
+                <p>-</p>
+              )}
+            </div>
+          </div>
+        </Tailwind>
       }
       tabs={[
         {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
index be78bb51367..4060735cc67 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
@@ -429,6 +429,7 @@ export const ObjectVersionsLink: React.FC<{
   filter?: WFHighLevelObjectVersionFilter;
   neverPeek?: boolean;
   variant?: LinkVariant;
+  children?: React.ReactNode;
 }> = props => {
   const {peekingRouter, baseRouter} = useWeaveflowRouteContext();
   const router = props.neverPeek ? baseRouter : peekingRouter;
@@ -440,9 +441,13 @@ export const ObjectVersionsLink: React.FC<{
         props.project,
         props.filter
       )}>
-      {props.versionCount}
-      {props.countIsLimited ? '+' : ''} version
-      {props.versionCount !== 1 ? 's' : ''}
+      {props.children ?? (
+        <>
+          {props.versionCount}
+          {props.countIsLimited ? '+' : ''} version
+          {props.versionCount !== 1 ? 's' : ''}
+        </>
+      )}
     </Link>
   );
 };
@@ -455,6 +460,7 @@ export const OpVersionsLink: React.FC<{
   filter?: WFHighLevelOpVersionFilter;
   neverPeek?: boolean;
   variant?: LinkVariant;
+  children?: React.ReactNode;
 }> = props => {
   const {peekingRouter, baseRouter} = useWeaveflowRouteContext();
   const router = props.neverPeek ? baseRouter : peekingRouter;
@@ -462,9 +468,13 @@ export const OpVersionsLink: React.FC<{
     <Link
       $variant={props.variant}
       to={router.opVersionsUIUrl(props.entity, props.project, props.filter)}>
-      {props.versionCount}
-      {props.countIsLimited ? '+' : ''} version
-      {props.versionCount !== 1 ? 's' : ''}
+      {props.children ?? (
+        <>
+          {props.versionCount}
+          {props.countIsLimited ? '+' : ''} version
+          {props.versionCount !== 1 ? 's' : ''}
+        </>
+      )}
     </Link>
   );
 };

From 7320ca9a1fcc51808103403d61923e00361a02d9 Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Mon, 9 Dec 2024 10:35:15 -0800
Subject: [PATCH 15/52] chore(ui): disable refresh while refreshing (#3179)

---
 .../Home/Browse3/pages/CallsPage/CallsTable.tsx              | 5 ++++-
 .../Home/Browse3/pages/CallsPage/CallsTableButtons.tsx       | 4 +++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx
index 18130d64341..3632664aaf0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTable.tsx
@@ -744,7 +744,10 @@ export const CallsTable: FC<{
       }}
       filterListItems={
         <TailwindContents>
-          <RefreshButton onClick={() => calls.refetch()} />
+          <RefreshButton
+            onClick={() => calls.refetch()}
+            disabled={callsLoading}
+          />
           {!hideOpSelector && (
             <OpSelector
               frozenFilter={frozenFilter}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTableButtons.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTableButtons.tsx
index 2d5322df584..1892c085396 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTableButtons.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallsPage/CallsTableButtons.tsx
@@ -471,7 +471,8 @@ export const BulkDeleteButton: FC<{
 
 export const RefreshButton: FC<{
   onClick: () => void;
-}> = ({onClick}) => {
+  disabled?: boolean;
+}> = ({onClick, disabled}) => {
   return (
     <Box
       sx={{
@@ -483,6 +484,7 @@ export const RefreshButton: FC<{
         variant="outline"
         size="medium"
         onClick={onClick}
+        disabled={disabled}
         tooltip="Refresh"
         icon="reload-refresh"
       />

From 0cfe058b709b77c96b901a0975f113e4d42d85a3 Mon Sep 17 00:00:00 2001
From: Connie Lee <connie.lee0317@gmail.com>
Date: Mon, 9 Dec 2024 11:39:23 -0800
Subject: [PATCH 16/52] style(app): Update tailwind shadow configs (#3180)

---
 weave-js/tailwind.config.cjs | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/weave-js/tailwind.config.cjs b/weave-js/tailwind.config.cjs
index b234cc3cf1a..d8a580c928c 100644
--- a/weave-js/tailwind.config.cjs
+++ b/weave-js/tailwind.config.cjs
@@ -13,8 +13,13 @@ module.exports = {
      */
     boxShadow: {
       none: 'none',
-      md: '0px 12px 24px 0px #15181F29',
-      lg: '0px 24px 48px 0px #15181F29',
+      flat: '0px 4px 8px 0px #0D0F120a', // oblivion 4%
+      medium: '0px 12px 24px 0px #0D0F1229', // oblivion 16%
+      deep: '0px 24px 48px 0px #0D0F123d', // oblivion 24%
+
+      // deprecated shadow configs
+      md: '0px 12px 24px 0px #15181F29', // use shadow-medium instead
+      lg: '0px 24px 48px 0px #15181F29', // use shadow-deep instead
     },
     spacing: {
       0: '0rem',
@@ -189,17 +194,17 @@ module.exports = {
     },
     extend: {
       animation: {
-        'wave': 'wave 3s linear  infinite'
+        wave: 'wave 3s linear  infinite',
       },
       keyframes: {
-        "wave": {
-          "0%, 30%, 100%": {
-            transform: "initial"
+        wave: {
+          '0%, 30%, 100%': {
+            transform: 'initial',
           },
-          "15%": {
-            transform: "translateY(-10px)"
-          }
-        }
+          '15%': {
+            transform: 'translateY(-10px)',
+          },
+        },
       },
       opacity: {
         35: '.35',
@@ -221,6 +226,6 @@ module.exports = {
      in their parent hierarchy */
   important: '.tw-style',
   experimental: {
-    optimizeUniversalDefaults: true
+    optimizeUniversalDefaults: true,
   },
 };

From e9b73f7c44479cc783ddc12d59518b5e6e40f1b4 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 9 Dec 2024 12:13:06 -0800
Subject: [PATCH 17/52] feat(weave): Add AzureOpenAI support for Scorers
 (#3171)

* init

* init
---
 weave/scorers/llm_utils.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/weave/scorers/llm_utils.py b/weave/scorers/llm_utils.py
index 8e2672b8ee7..68ae2ccb366 100644
--- a/weave/scorers/llm_utils.py
+++ b/weave/scorers/llm_utils.py
@@ -23,10 +23,17 @@
     from google.generativeai import GenerativeModel
     from instructor.patch import InstructorChatCompletionCreate
     from mistralai import Mistral
-    from openai import AsyncOpenAI, OpenAI
+    from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 
     _LLM_CLIENTS = Union[
-        OpenAI, AsyncOpenAI, Anthropic, AsyncAnthropic, Mistral, GenerativeModel
+        OpenAI,
+        AsyncOpenAI,
+        AzureOpenAI,
+        AsyncAzureOpenAI,
+        Anthropic,
+        AsyncAnthropic,
+        Mistral,
+        GenerativeModel,
     ]
 else:
     _LLM_CLIENTS = object
@@ -34,6 +41,8 @@
 _LLM_CLIENTS_NAMES = (
     "OpenAI",
     "AsyncOpenAI",
+    "AzureOpenAI",
+    "AsyncAzureOpenAI",
     "Anthropic",
     "AsyncAnthropic",
     "Mistral",

From 2351ecf4f867a7834262f9b68ed026ba77723b24 Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Mon, 9 Dec 2024 15:15:14 -0600
Subject: [PATCH 18/52] chore(weave): add sun and moon icons (#3182)

---
 weave-js/src/assets/icons/icon-moon.svg         |  3 +++
 weave-js/src/assets/icons/icon-not-visible.svg  |  2 +-
 weave-js/src/assets/icons/icon-pin-to-right.svg |  2 +-
 weave-js/src/assets/icons/icon-sun.svg          | 11 +++++++++++
 weave-js/src/components/Icon/Icon.tsx           | 10 ++++++++++
 weave-js/src/components/Icon/index.ts           |  2 ++
 weave-js/src/components/Icon/types.ts           |  2 ++
 7 files changed, 30 insertions(+), 2 deletions(-)
 create mode 100644 weave-js/src/assets/icons/icon-moon.svg
 create mode 100644 weave-js/src/assets/icons/icon-sun.svg

diff --git a/weave-js/src/assets/icons/icon-moon.svg b/weave-js/src/assets/icons/icon-moon.svg
new file mode 100644
index 00000000000..e448eab96c3
--- /dev/null
+++ b/weave-js/src/assets/icons/icon-moon.svg
@@ -0,0 +1,3 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M10.3603 3.06624C10.545 3.32685 10.5444 3.67587 10.3588 3.93584C9.54573 5.07441 9.06752 6.46746 9.06752 7.97442C9.06752 11.8172 12.1827 14.9325 16.0256 14.9325C17.5325 14.9325 18.9256 14.4543 20.0641 13.6412C20.3241 13.4556 20.6731 13.455 20.9337 13.6397C21.1943 13.8245 21.3093 14.154 21.2202 14.4608C20.0812 18.3824 16.4622 21.25 12.1715 21.25C6.96817 21.25 2.75 17.0318 2.75 11.8284C2.75 7.53776 5.61757 3.91878 9.53923 2.77975C9.84599 2.69065 10.1755 2.80564 10.3603 3.06624ZM8.09192 5.03657C5.78955 6.42258 4.25 8.94635 4.25 11.8284C4.25 16.2034 7.7966 19.75 12.1715 19.75C15.0536 19.75 17.5774 18.2104 18.9634 15.9081C18.048 16.2472 17.0581 16.4325 16.0256 16.4325C11.3543 16.4325 7.56752 12.6457 7.56752 7.97442C7.56752 6.94192 7.75281 5.95198 8.09192 5.03657Z" fill="currentColor"/>
+</svg>
diff --git a/weave-js/src/assets/icons/icon-not-visible.svg b/weave-js/src/assets/icons/icon-not-visible.svg
index 766810c7811..b2782d987b9 100644
--- a/weave-js/src/assets/icons/icon-not-visible.svg
+++ b/weave-js/src/assets/icons/icon-not-visible.svg
@@ -2,4 +2,4 @@
     <path
         d="M21.3458 10.0012C21.7133 10.1923 21.8563 10.6452 21.6652 11.0127L20.9998 10.6667C21.6652 11.0127 21.6652 11.0127 21.6652 11.0127L21.6644 11.0142L21.6634 11.0162L21.6605 11.0216L21.6519 11.0377C21.6448 11.0508 21.6351 11.0685 21.6227 11.0906C21.5979 11.1347 21.5622 11.1963 21.5155 11.2727C21.422 11.4256 21.2838 11.6385 21.0984 11.8914C20.8469 12.2344 20.5068 12.653 20.0715 13.0959C19.9136 13.2565 19.9094 13.5154 20.0687 13.6747L21.2303 14.8363C21.5232 15.1292 21.5232 15.6041 21.2303 15.897C20.9374 16.1899 20.4625 16.1899 20.1696 15.897L18.8569 14.5843C18.7183 14.4457 18.5002 14.4286 18.3376 14.538C17.7094 14.961 16.9854 15.3543 16.1606 15.6706C15.9444 15.7535 15.8335 15.9974 15.9226 16.2112L16.4922 17.5782C16.6515 17.9605 16.4707 18.3997 16.0883 18.559C15.706 18.7183 15.2669 18.5375 15.1076 18.1551L14.4192 16.5031C14.3463 16.328 14.1606 16.229 13.9735 16.2596C13.3566 16.3605 12.6991 16.4167 11.9998 16.4167C11.3006 16.4167 10.6432 16.3605 10.0263 16.2596C9.8392 16.229 9.65352 16.3281 9.58058 16.5031L8.89224 18.1551C8.73293 18.5375 8.29382 18.7183 7.91147 18.559C7.52912 18.3997 7.34831 17.9605 7.50763 17.5782L8.07719 16.2112C8.16627 15.9975 8.0554 15.7536 7.83915 15.6706C7.0144 15.3543 6.29033 14.961 5.6621 14.5381C5.49952 14.4286 5.28141 14.4458 5.14283 14.5843L3.83018 15.897C3.53729 16.1899 3.06242 16.1899 2.76952 15.897C2.47663 15.6041 2.47663 15.1292 2.76952 14.8363L3.93106 13.6748C4.09033 13.5155 4.08616 13.2566 3.92825 13.096C3.49289 12.6531 3.15279 12.2344 2.90126 11.8914C2.71579 11.6385 2.57761 11.4256 2.48417 11.2727C2.43742 11.1963 2.40178 11.1347 2.37695 11.0906C2.36452 11.0685 2.3548 11.0508 2.34772 11.0377L2.33909 11.0216L2.33626 11.0162L2.33521 11.0142C2.33521 11.0142 2.33441 11.0127 2.99977 10.6667L2.33441 11.0127C2.14331 10.6452 2.28631 10.1923 2.65381 10.0012C3.02074 9.81044 3.47274 9.95271 3.66435 10.319L3.66722 10.3243C3.67038 10.3301 3.67603 10.3405 3.68422 10.355C3.70059 10.3841 3.72706 10.43 3.76391 10.4903C3.83765 10.6109 3.9526 10.7886 4.11087 11.0044C4.42796 11.4368 4.91547 12.0175 5.58952 12.5986C6.93141 13.7554 9.01343 14.9167 11.9998 14.9167C14.9862 14.9167 17.0682 13.7554 18.4101 12.5986C19.0842 12.0175 19.5717 11.4368 19.8888 11.0044C20.047 10.7886 20.162 10.6109 20.2357 10.4903C20.2726 10.43 20.299 10.3841 20.3154 10.355C20.3236 10.3405 20.3293 10.3301 20.3324 10.3243L20.3344 10.3206L20.335 10.3196C20.5266 9.95329 20.9789 9.81044 21.3458 10.0012Z"
         fill="currentColor" />
-</svg>
\ No newline at end of file
+</svg>
diff --git a/weave-js/src/assets/icons/icon-pin-to-right.svg b/weave-js/src/assets/icons/icon-pin-to-right.svg
index 1ae05ea52ae..46a9c0bf114 100644
--- a/weave-js/src/assets/icons/icon-pin-to-right.svg
+++ b/weave-js/src/assets/icons/icon-pin-to-right.svg
@@ -2,4 +2,4 @@
     <path fill-rule="evenodd" clip-rule="evenodd"
         d="M3.01719 9.2399C2.06222 10.1952 2.52073 11.7288 3.65016 12.2008C5.16656 12.8346 6.60702 13.9748 7.45814 14.976C8.31598 15.9852 9.31862 17.6021 9.90729 19.2358C10.3377 20.4302 11.8963 20.8641 12.8429 19.9172L15.8021 16.9571L20.2201 21.3765C20.5129 21.6694 20.9876 21.6694 21.2804 21.3765C21.5732 21.0836 21.5732 20.6087 21.2804 20.3158L16.8624 15.8964L19.9117 12.8461C20.8582 11.8993 20.4245 10.3402 19.2305 9.90964C17.5973 9.32079 15.9809 8.31782 14.9721 7.45971C13.9712 6.60832 12.8313 5.16741 12.1977 3.65052C11.7259 2.52073 10.1927 2.06208 9.23776 3.01736L3.01719 9.2399ZM4.22824 10.8168C4.11611 10.7699 4.03833 10.6718 4.01204 10.5595C3.98693 10.4523 4.01377 10.3643 4.07752 10.3006L10.2981 4.07802C10.3618 4.01425 10.4498 3.9874 10.557 4.01252C10.6692 4.03882 10.7673 4.11663 10.8141 4.22879C11.5256 5.93216 12.4296 7.18928 14.0007 8.60243C15.5826 10.0253 16.9977 10.6991 18.722 11.3208C18.8244 11.3577 18.8927 11.4398 18.9167 11.5384C18.9399 11.634 18.9168 11.72 18.8514 11.7855L11.7825 18.8566C11.7171 18.922 11.6311 18.9451 11.5355 18.9219C11.437 18.8979 11.3549 18.8296 11.318 18.7272C10.6965 17.0023 10.0229 15.5868 8.60049 14.0043C7.18779 12.4327 5.93107 11.5284 4.22824 10.8168Z"
         fill="currentColor" />
-</svg>
\ No newline at end of file
+</svg>
diff --git a/weave-js/src/assets/icons/icon-sun.svg b/weave-js/src/assets/icons/icon-sun.svg
new file mode 100644
index 00000000000..bb0c57891b0
--- /dev/null
+++ b/weave-js/src/assets/icons/icon-sun.svg
@@ -0,0 +1,11 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M12 7.75C9.65279 7.75 7.75 9.65279 7.75 12C7.75 14.3472 9.65279 16.25 12 16.25C14.3472 16.25 16.25 14.3472 16.25 12C16.25 9.65279 14.3472 7.75 12 7.75ZM6.25 12C6.25 8.82436 8.82436 6.25 12 6.25C15.1756 6.25 17.75 8.82436 17.75 12C17.75 15.1756 15.1756 17.75 12 17.75C8.82436 17.75 6.25 15.1756 6.25 12Z" fill="currentColor"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M12 1.25C12.4142 1.25 12.75 1.58579 12.75 2V4C12.75 4.41421 12.4142 4.75 12 4.75C11.5858 4.75 11.25 4.41421 11.25 4V2C11.25 1.58579 11.5858 1.25 12 1.25Z" fill="currentColor"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M12 19.25C12.4142 19.25 12.75 19.5858 12.75 20V22C12.75 22.4142 12.4142 22.75 12 22.75C11.5858 22.75 11.25 22.4142 11.25 22V20C11.25 19.5858 11.5858 19.25 12 19.25Z" fill="currentColor"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M1.25 12C1.25 11.5858 1.58579 11.25 2 11.25H4C4.41421 11.25 4.75 11.5858 4.75 12C4.75 12.4142 4.41421 12.75 4 12.75H2C1.58579 12.75 1.25 12.4142 1.25 12Z" fill="currentColor"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M19.25 12C19.25 11.5858 19.5858 11.25 20 11.25H22C22.4142 11.25 22.75 11.5858 22.75 12C22.75 12.4142 22.4142 12.75 22 12.75H20C19.5858 12.75 19.25 12.4142 19.25 12Z" fill="currentColor"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M4.3986 4.3986C4.6915 4.10571 5.16637 4.10571 5.45926 4.3986L6.87348 5.81282C7.16637 6.10571 7.16637 6.58058 6.87348 6.87348C6.58058 7.16637 6.10571 7.16637 5.81282 6.87348L4.3986 5.45926C4.10571 5.16637 4.10571 4.6915 4.3986 4.3986Z" fill="currentColor"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M17.1265 17.1265C17.4194 16.8336 17.8943 16.8336 18.1872 17.1265L19.6014 18.5407C19.8943 18.8336 19.8943 19.3085 19.6014 19.6014C19.3085 19.8943 18.8336 19.8943 18.5407 19.6014L17.1265 18.1872C16.8336 17.8943 16.8336 17.4194 17.1265 17.1265Z" fill="currentColor"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M4.3986 19.6014C4.10571 19.3085 4.10571 18.8336 4.3986 18.5407L5.81282 17.1265C6.10571 16.8336 6.58058 16.8336 6.87348 17.1265C7.16637 17.4194 7.16637 17.8943 6.87348 18.1872L5.45926 19.6014C5.16637 19.8943 4.6915 19.8943 4.3986 19.6014Z" fill="currentColor"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M17.1265 6.87348C16.8336 6.58058 16.8336 6.10571 17.1265 5.81282L18.5407 4.3986C18.8336 4.10571 19.3085 4.10571 19.6014 4.3986C19.8943 4.6915 19.8943 5.16637 19.6014 5.45926L18.1872 6.87348C17.8943 7.16637 17.4194 7.16637 17.1265 6.87348Z" fill="currentColor"/>
+</svg>
diff --git a/weave-js/src/components/Icon/Icon.tsx b/weave-js/src/components/Icon/Icon.tsx
index 948c8a456b4..9c020b1bab9 100644
--- a/weave-js/src/components/Icon/Icon.tsx
+++ b/weave-js/src/components/Icon/Icon.tsx
@@ -139,6 +139,7 @@ import {ReactComponent as ImportMinimizeMode} from '../../assets/icons/icon-mini
 import {ReactComponent as ImportModel} from '../../assets/icons/icon-model.svg';
 import {ReactComponent as ImportModelOnDark} from '../../assets/icons/icon-model-on-dark.svg';
 import {ReactComponent as ImportMolecule} from '../../assets/icons/icon-molecule.svg';
+import {ReactComponent as ImportMoon} from '../../assets/icons/icon-moon.svg';
 import {ReactComponent as ImportMusicAudio} from '../../assets/icons/icon-music-audio.svg';
 import {ReactComponent as ImportNewSectionAbove} from '../../assets/icons/icon-new-section-above.svg';
 import {ReactComponent as ImportNewSectionBelow} from '../../assets/icons/icon-new-section-below.svg';
@@ -216,6 +217,7 @@ import {ReactComponent as ImportStar} from '../../assets/icons/icon-star.svg';
 import {ReactComponent as ImportStarFilled} from '../../assets/icons/icon-star-filled.svg';
 import {ReactComponent as ImportStop} from '../../assets/icons/icon-stop.svg';
 import {ReactComponent as ImportStopped} from '../../assets/icons/icon-stopped.svg';
+import {ReactComponent as ImportSun} from '../../assets/icons/icon-sun.svg';
 import {ReactComponent as ImportSwap} from '../../assets/icons/icon-swap.svg';
 import {ReactComponent as ImportSweepBayes} from '../../assets/icons/icon-sweep-bayes.svg';
 import {ReactComponent as ImportSweepGrid} from '../../assets/icons/icon-sweep-grid.svg';
@@ -695,6 +697,9 @@ export const IconModelOnDark = (props: SVGIconProps) => (
 export const IconMolecule = (props: SVGIconProps) => (
   <ImportMolecule {...updateIconProps(props)} />
 );
+export const IconMoon = (props: SVGIconProps) => (
+  <ImportMoon {...updateIconProps(props)} />
+);
 export const IconMusicAudio = (props: SVGIconProps) => (
   <ImportMusicAudio {...updateIconProps(props)} />
 );
@@ -926,6 +931,9 @@ export const IconStop = (props: SVGIconProps) => (
 export const IconStopped = (props: SVGIconProps) => (
   <ImportStopped {...updateIconProps(props)} />
 );
+export const IconSun = (props: SVGIconProps) => (
+  <ImportSun {...updateIconProps(props)} />
+);
 export const IconSwap = (props: SVGIconProps) => (
   <ImportSwap {...updateIconProps(props)} />
 );
@@ -1211,6 +1219,7 @@ const ICON_NAME_TO_ICON: Record<IconName, ElementType> = {
   model: IconModel,
   'model-on-dark': IconModelOnDark,
   molecule: IconMolecule,
+  moon: IconMoon,
   'music-audio': IconMusicAudio,
   'new-section-above': IconNewSectionAbove,
   'new-section-below': IconNewSectionBelow,
@@ -1288,6 +1297,7 @@ const ICON_NAME_TO_ICON: Record<IconName, ElementType> = {
   'star-filled': IconStarFilled,
   stop: IconStop,
   stopped: IconStopped,
+  sun: IconSun,
   swap: IconSwap,
   'sweep-bayes': IconSweepBayes,
   'sweep-grid': IconSweepGrid,
diff --git a/weave-js/src/components/Icon/index.ts b/weave-js/src/components/Icon/index.ts
index 81839a71019..85ea5332649 100644
--- a/weave-js/src/components/Icon/index.ts
+++ b/weave-js/src/components/Icon/index.ts
@@ -139,6 +139,7 @@ export {
   IconModel,
   IconModelOnDark,
   IconMolecule,
+  IconMoon,
   IconMusicAudio,
   IconNewSectionAbove,
   IconNewSectionBelow,
@@ -216,6 +217,7 @@ export {
   IconStarFilled,
   IconStop,
   IconStopped,
+  IconSun,
   IconSwap,
   IconSweepBayes,
   IconSweepGrid,
diff --git a/weave-js/src/components/Icon/types.ts b/weave-js/src/components/Icon/types.ts
index 55f46c52833..87a1207bc85 100644
--- a/weave-js/src/components/Icon/types.ts
+++ b/weave-js/src/components/Icon/types.ts
@@ -138,6 +138,7 @@ export const IconNames = {
   Model: 'model',
   ModelOnDark: 'model-on-dark',
   Molecule: 'molecule',
+  Moon: 'moon',
   MusicAudio: 'music-audio',
   NewSectionAbove: 'new-section-above',
   NewSectionBelow: 'new-section-below',
@@ -215,6 +216,7 @@ export const IconNames = {
   StarFilled: 'star-filled',
   Stop: 'stop',
   Stopped: 'stopped',
+  Sun: 'sun',
   Swap: 'swap',
   SweepBayes: 'sweep-bayes',
   SweepGrid: 'sweep-grid',

From 8d0f3f7a6d84108fe52d335251d4080078dff0f0 Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Mon, 9 Dec 2024 16:08:01 -0600
Subject: [PATCH 19/52] chore(ui): fix typo (#3183)

---
 .../Home/Browse3/pages/CallPage/CallPage.tsx                | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx
index 3e4a74a4885..484b038c193 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx
@@ -312,7 +312,7 @@ const CallPageInnerVertical: FC<{
 
   const {rowIdsConfigured} = useContext(TableRowSelectionContext);
   const {isPeeking} = useContext(WeaveflowPeekContext);
-  const showPaginationContols = isPeeking && rowIdsConfigured;
+  const showPaginationControls = isPeeking && rowIdsConfigured;
 
   const callTabs = useCallTabs(currentCall);
 
@@ -330,10 +330,10 @@ const CallPageInnerVertical: FC<{
             justifyContent: 'space-between',
             alignItems: 'center',
           }}>
-          {showPaginationContols && (
+          {showPaginationControls && (
             <PaginationControls call={call} path={path} />
           )}
-          <Box sx={{marginLeft: showPaginationContols ? 0 : 'auto'}}>
+          <Box sx={{marginLeft: showPaginationControls ? 0 : 'auto'}}>
             <Button
               icon="layout-tabs"
               tooltip={`${showTraceTree ? 'Hide' : 'Show'} trace tree`}

From a4df17003fcfa312aba136983e4d9890ad5efe22 Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Mon, 9 Dec 2024 15:50:59 -0800
Subject: [PATCH 20/52] chore(ui): split up and page metrics in eval compare
 summary section (#2672)

---
 .../ComparisonDefinitionSection.tsx           |   4 +-
 .../SummaryPlotsSection/MetricsSelector.tsx   | 173 +++++
 .../SummaryPlotsSection/PlotlyBarPlot.tsx     |  48 +-
 .../SummaryPlotsSection/PlotlyRadarPlot.tsx   |   8 +-
 .../SummaryPlotsSection.tsx                   | 595 ++++++++++--------
 5 files changed, 555 insertions(+), 273 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/MetricsSelector.tsx

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx
index b5c1a4bf96c..66ce56c4cb0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx
@@ -105,7 +105,7 @@ const AddEvaluationButton: React.FC<{
   );
   const expandedRefCols = useMemo(() => new Set<string>(), []);
   // Don't query for output here, re-queried in tsDataModelHooksEvaluationComparison.ts
-  const columns = useMemo(() => ['inputs'], []);
+  const columns = useMemo(() => ['inputs', 'display_name'], []);
   const calls = useCallsForQuery(
     props.state.data.entity,
     props.state.data.project,
@@ -137,7 +137,7 @@ const AddEvaluationButton: React.FC<{
       return;
     }
 
-    const filteredOptions = calls.result.filter(call => {
+    const filteredOptions = evalsNotComparing.filter(call => {
       if (
         (call.displayName ?? call.spanName)
           .toLowerCase()
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/MetricsSelector.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/MetricsSelector.tsx
new file mode 100644
index 00000000000..3e6dfdd30a1
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/MetricsSelector.tsx
@@ -0,0 +1,173 @@
+import {Popover} from '@mui/material';
+import {Switch} from '@wandb/weave/components';
+import {Button} from '@wandb/weave/components/Button';
+import {
+  DraggableGrow,
+  DraggableHandle,
+} from '@wandb/weave/components/DraggablePopups';
+import {TextField} from '@wandb/weave/components/Form/TextField';
+import {Tailwind} from '@wandb/weave/components/Tailwind';
+import {maybePluralize} from '@wandb/weave/core/util/string';
+import classNames from 'classnames';
+import React, {useRef, useState} from 'react';
+
+export const MetricsSelector: React.FC<{
+  setSelectedMetrics: (newModel: Record<string, boolean>) => void;
+  selectedMetrics: Record<string, boolean> | undefined;
+  allMetrics: string[];
+}> = ({setSelectedMetrics, selectedMetrics, allMetrics}) => {
+  const [search, setSearch] = useState('');
+
+  const ref = useRef<HTMLDivElement>(null);
+  const [anchorEl, setAnchorEl] = useState<null | HTMLElement>(null);
+  const onClick = (event: React.MouseEvent<HTMLElement>) => {
+    setAnchorEl(anchorEl ? null : ref.current);
+    setSearch('');
+  };
+  const open = Boolean(anchorEl);
+  const id = open ? 'simple-popper' : undefined;
+
+  const filteredCols = search
+    ? allMetrics.filter(col => col.toLowerCase().includes(search.toLowerCase()))
+    : allMetrics;
+
+  const shownMetrics = Object.values(selectedMetrics ?? {}).filter(Boolean);
+
+  const numHidden = allMetrics.length - shownMetrics.length;
+  const buttonSuffix = search ? `(${filteredCols.length})` : 'all';
+
+  return (
+    <>
+      <span ref={ref}>
+        <Button
+          variant="ghost"
+          icon="column"
+          tooltip="Manage metrics"
+          onClick={onClick}
+        />
+      </span>
+      <Popover
+        id={id}
+        open={open}
+        anchorEl={anchorEl}
+        anchorOrigin={{
+          vertical: 'bottom',
+          horizontal: 'center',
+        }}
+        transformOrigin={{
+          vertical: 'top',
+          horizontal: 'center',
+        }}
+        slotProps={{
+          paper: {
+            sx: {
+              overflow: 'visible',
+            },
+          },
+        }}
+        onClose={() => setAnchorEl(null)}
+        TransitionComponent={DraggableGrow}>
+        <Tailwind>
+          <div className="min-w-[360px] p-12">
+            <DraggableHandle>
+              <div className="flex items-center pb-8">
+                <div className="flex-auto text-xl font-semibold">
+                  Manage metrics
+                </div>
+                <div className="ml-16 text-moon-500">
+                  {maybePluralize(numHidden, 'hidden column', 's')}
+                </div>
+              </div>
+            </DraggableHandle>
+            <div className="mb-8">
+              <TextField
+                placeholder="Filter columns"
+                autoFocus
+                value={search}
+                onChange={setSearch}
+              />
+            </div>
+            <div className="max-h-[300px] overflow-auto">
+              {Array.from(allMetrics).map((metric: string) => {
+                const value = metric;
+                const idSwitch = `toggle-vis_${value}`;
+                const checked = selectedMetrics?.[metric] ?? false;
+                const label = metric;
+                const disabled = false;
+                if (
+                  search &&
+                  !label.toLowerCase().includes(search.toLowerCase())
+                ) {
+                  return null;
+                }
+                return (
+                  <div key={value}>
+                    <div
+                      className={classNames(
+                        'flex items-center py-2',
+                        disabled ? 'opacity-40' : ''
+                      )}>
+                      <Switch.Root
+                        id={idSwitch}
+                        size="small"
+                        checked={checked}
+                        onCheckedChange={isOn => {
+                          setSelectedMetrics(
+                            isOn
+                              ? {...selectedMetrics, [metric]: true}
+                              : {...selectedMetrics, [metric]: false}
+                          );
+                        }}
+                        disabled={disabled}>
+                        <Switch.Thumb size="small" checked={checked} />
+                      </Switch.Root>
+                      <label
+                        htmlFor={idSwitch}
+                        className={classNames(
+                          'ml-6',
+                          disabled ? '' : 'cursor-pointer'
+                        )}>
+                        {label}
+                      </label>
+                    </div>
+                  </div>
+                );
+              })}
+            </div>
+            <div className="mt-8 flex items-center">
+              <Button
+                size="small"
+                variant="quiet"
+                icon="hide-hidden"
+                disabled={filteredCols.length === 0}
+                onClick={() => {
+                  const newModel = {...selectedMetrics};
+                  for (const metric of filteredCols) {
+                    newModel[metric] = false;
+                  }
+                  setSelectedMetrics(newModel);
+                }}>
+                {`Hide ${buttonSuffix}`}
+              </Button>
+              <div className="flex-auto" />
+              <Button
+                size="small"
+                variant="quiet"
+                icon="show-visible"
+                disabled={filteredCols.length === 0}
+                onClick={() => {
+                  const newModel = {...selectedMetrics};
+                  for (const metric of filteredCols) {
+                    newModel[metric] = true;
+                  }
+                  setSelectedMetrics(newModel);
+                }}>
+                {`Show ${buttonSuffix}`}
+              </Button>
+            </div>
+          </div>
+        </Tailwind>
+      </Popover>
+    </>
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/PlotlyBarPlot.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/PlotlyBarPlot.tsx
index 9706ac09567..7942aea195e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/PlotlyBarPlot.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/PlotlyBarPlot.tsx
@@ -2,50 +2,54 @@ import * as Plotly from 'plotly.js';
 import React, {useEffect, useMemo, useRef} from 'react';
 
 import {PLOT_GRID_COLOR} from '../../ecpConstants';
-import {RadarPlotData} from './PlotlyRadarPlot';
 
 export const PlotlyBarPlot: React.FC<{
   height: number;
-  data: RadarPlotData;
+  yRange: [number, number];
+  plotlyData: Plotly.Data;
 }> = props => {
   const divRef = useRef<HTMLDivElement>(null);
-  const plotlyData: Plotly.Data[] = useMemo(() => {
-    return Object.keys(props.data).map((key, i) => {
-      const {metrics, name, color} = props.data[key];
-      return {
-        type: 'bar',
-        y: Object.values(metrics),
-        x: Object.keys(metrics),
-        name,
-        marker: {color},
-      };
-    });
-  }, [props.data]);
-
   const plotlyLayout: Partial<Plotly.Layout> = useMemo(() => {
     return {
-      height: props.height - 40,
+      height: props.height - 30,
       showlegend: false,
       margin: {
-        l: 0,
+        l: 20,
         r: 0,
         b: 20,
-        t: 0,
-        pad: 0,
+        t: 26,
       },
+      bargap: 0.1,
       xaxis: {
         automargin: true,
         fixedrange: true,
         gridcolor: PLOT_GRID_COLOR,
         linecolor: PLOT_GRID_COLOR,
+        showticklabels: false,
       },
       yaxis: {
         fixedrange: true,
+        range: props.yRange,
         gridcolor: PLOT_GRID_COLOR,
         linecolor: PLOT_GRID_COLOR,
+        showticklabels: true,
+        tickfont: {
+          size: 10,
+        },
+      },
+      title: {
+        multiline: true,
+        text: props.plotlyData.name ?? '',
+        font: {size: 12},
+        xref: 'paper',
+        x: 0.5,
+        y: 1,
+        yanchor: 'top',
+        pad: {t: 2},
       },
     };
-  }, [props.height]);
+  }, [props.height, props.plotlyData, props.yRange]);
+
   const plotlyConfig = useMemo(() => {
     return {
       displayModeBar: false,
@@ -57,11 +61,11 @@ export const PlotlyBarPlot: React.FC<{
   useEffect(() => {
     Plotly.newPlot(
       divRef.current as any,
-      plotlyData,
+      [props.plotlyData],
       plotlyLayout,
       plotlyConfig
     );
-  }, [plotlyConfig, plotlyData, plotlyLayout]);
+  }, [plotlyConfig, props.plotlyData, plotlyLayout]);
 
   return <div ref={divRef}></div>;
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/PlotlyRadarPlot.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/PlotlyRadarPlot.tsx
index d459d1354f1..47d0fa3f10c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/PlotlyRadarPlot.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/PlotlyRadarPlot.tsx
@@ -31,13 +31,13 @@ export const PlotlyRadarPlot: React.FC<{
   }, [props.data]);
   const plotlyLayout: Partial<Plotly.Layout> = useMemo(() => {
     return {
-      height: props.height,
+      height: props.height - 40,
       showlegend: false,
       margin: {
-        l: 60,
-        r: 0,
+        l: 20,
+        r: 20,
         b: 30,
-        t: 30,
+        t: 20,
         pad: 0,
       },
       polar: {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/SummaryPlotsSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/SummaryPlotsSection.tsx
index 5bfaa8fcb04..c23ffcce04d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/SummaryPlotsSection.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/SummaryPlotsSection.tsx
@@ -1,15 +1,6 @@
 import {Box} from '@material-ui/core';
-import {Popover} from '@mui/material';
-import {Switch} from '@wandb/weave/components';
 import {Button} from '@wandb/weave/components/Button';
-import {
-  DraggableGrow,
-  DraggableHandle,
-} from '@wandb/weave/components/DraggablePopups';
-import {TextField} from '@wandb/weave/components/Form/TextField';
 import {Tailwind} from '@wandb/weave/components/Tailwind';
-import {maybePluralize} from '@wandb/weave/core/util/string';
-import classNames from 'classnames';
 import React, {useEffect, useMemo, useRef, useState} from 'react';
 
 import {buildCompositeMetricsMap} from '../../compositeMetricsUtil';
@@ -27,6 +18,7 @@ import {
   resolveSummaryMetricValueForEvaluateCall,
 } from '../../ecpUtil';
 import {HorizontalBox, VerticalBox} from '../../Layout';
+import {MetricsSelector} from './MetricsSelector';
 import {PlotlyBarPlot} from './PlotlyBarPlot';
 import {PlotlyRadarPlot, RadarPlotData} from './PlotlyRadarPlot';
 
@@ -36,15 +28,12 @@ import {PlotlyRadarPlot, RadarPlotData} from './PlotlyRadarPlot';
 export const SummaryPlots: React.FC<{
   state: EvaluationComparisonState;
   setSelectedMetrics: (newModel: Record<string, boolean>) => void;
-}> = props => {
-  const {radarData, allMetricNames} = useNormalizedPlotDataFromMetrics(
-    props.state
-  );
-  const {selectedMetrics} = props.state;
-  const setSelectedMetrics = props.setSelectedMetrics;
+}> = ({state, setSelectedMetrics}) => {
+  const {radarData, allMetricNames} = useNormalizedPlotDataFromMetrics(state);
+  const {selectedMetrics} = state;
 
+  // Initialize selectedMetrics if null
   useEffect(() => {
-    // If selectedMetrics is null, we should show all metrics
     if (selectedMetrics == null) {
       setSelectedMetrics(
         Object.fromEntries(Array.from(allMetricNames).map(m => [m, true]))
@@ -52,10 +41,184 @@ export const SummaryPlots: React.FC<{
     }
   }, [selectedMetrics, setSelectedMetrics, allMetricNames]);
 
-  // filter down the plotlyRadarData to only include the selected metrics, after
-  // computation, to allow quick addition/removal of metrics
-  const filteredPlotlyRadarData = useMemo(() => {
-    const filteredData: RadarPlotData = {};
+  const filteredData = useFilteredData(radarData, selectedMetrics);
+  const normalizedRadarData = normalizeDataForRadarPlot(filteredData);
+  const barPlotData = useBarPlotData(filteredData);
+
+  const {
+    containerRef,
+    isInitialRender,
+    plotsPerPage,
+    currentPage,
+    setCurrentPage,
+  } = useContainerDimensions();
+
+  const {plotsToShow, totalPlots, startIndex, endIndex, totalPages} =
+    usePaginatedPlots(
+      normalizedRadarData,
+      barPlotData,
+      plotsPerPage,
+      currentPage
+    );
+
+  // Render placeholder during initial render
+  if (isInitialRender) {
+    return <div ref={containerRef} style={{width: '100%', height: '400px'}} />;
+  }
+
+  return (
+    <VerticalBox
+      sx={{
+        paddingLeft: STANDARD_PADDING,
+        paddingRight: STANDARD_PADDING,
+        flex: '1 1 auto',
+        width: '100%',
+      }}>
+      <SectionHeader
+        selectedMetrics={selectedMetrics}
+        setSelectedMetrics={setSelectedMetrics}
+        allMetrics={Array.from(allMetricNames)}
+      />
+      <div ref={containerRef} style={{width: '100%', display: 'flex'}}>
+        <HorizontalBox>{plotsToShow}</HorizontalBox>
+      </div>
+      <PaginationControls
+        currentPage={currentPage}
+        totalPages={totalPages}
+        startIndex={startIndex}
+        endIndex={endIndex}
+        totalPlots={totalPlots}
+        onPrevPage={() => setCurrentPage(prev => Math.max(prev - 1, 0))}
+        onNextPage={() =>
+          setCurrentPage(prev => Math.min(prev + 1, totalPages - 1))
+        }
+      />
+    </VerticalBox>
+  );
+};
+
+const SectionHeader: React.FC<{
+  selectedMetrics: Record<string, boolean> | undefined;
+  setSelectedMetrics: (newModel: Record<string, boolean>) => void;
+  allMetrics: string[];
+}> = ({selectedMetrics, setSelectedMetrics, allMetrics}) => (
+  <HorizontalBox
+    sx={{
+      width: '100%',
+      alignItems: 'center',
+      justifyContent: 'flex-start',
+    }}>
+    <Box
+      sx={{
+        fontSize: '1.5em',
+        fontWeight: 'bold',
+      }}>
+      Summary Metrics
+    </Box>
+    <Box sx={{marginLeft: 'auto'}}>
+      <div style={{display: 'flex', alignItems: 'center'}}>
+        <div style={{marginRight: '4px'}}>Configure displayed metrics</div>
+        <MetricsSelector
+          selectedMetrics={selectedMetrics}
+          setSelectedMetrics={setSelectedMetrics}
+          allMetrics={allMetrics}
+        />
+      </div>
+    </Box>
+  </HorizontalBox>
+);
+
+const RadarPlotBox: React.FC<{data: RadarPlotData}> = ({data}) => (
+  <Box
+    sx={{
+      height: PLOT_HEIGHT,
+      width: PLOT_HEIGHT * 2,
+      borderRadius: BOX_RADIUS,
+      border: STANDARD_BORDER,
+      padding: PLOT_PADDING,
+    }}>
+    <PlotlyRadarPlot height={PLOT_HEIGHT} data={data} />
+  </Box>
+);
+
+const BarPlotBox: React.FC<{
+  plot: {plotlyData: Plotly.Data; yRange: [number, number]};
+}> = ({plot}) => (
+  <Box
+    sx={{
+      height: PLOT_HEIGHT,
+      width: PLOT_HEIGHT,
+      borderRadius: BOX_RADIUS,
+      border: STANDARD_BORDER,
+      // make a bit more space for the title
+      paddingTop: PLOT_PADDING - 10,
+      paddingBottom: PLOT_PADDING,
+      paddingLeft: PLOT_PADDING,
+      paddingRight: PLOT_PADDING,
+    }}>
+    <PlotlyBarPlot
+      height={PLOT_HEIGHT}
+      plotlyData={plot.plotlyData}
+      yRange={plot.yRange}
+    />
+  </Box>
+);
+
+const PaginationControls: React.FC<{
+  currentPage: number;
+  totalPages: number;
+  startIndex: number;
+  endIndex: number;
+  totalPlots: number;
+  onPrevPage: () => void;
+  onNextPage: () => void;
+}> = ({
+  currentPage,
+  totalPages,
+  startIndex,
+  endIndex,
+  totalPlots,
+  onPrevPage,
+  onNextPage,
+}) => (
+  <HorizontalBox sx={{width: '100%'}}>
+    <Box
+      sx={{
+        marginLeft: 'auto',
+        display: 'flex',
+        alignItems: 'center',
+        justifyContent: 'center',
+      }}>
+      <Tailwind>
+        <div className="flex items-center">
+          <Button
+            variant="quiet"
+            onClick={onPrevPage}
+            disabled={currentPage === 0}
+            icon="chevron-next"
+            className="rotate-180"
+          />
+          <span className="mx-2 pb-2 text-sm text-moon-500">
+            {startIndex}-{endIndex} of {totalPlots}
+          </span>
+          <Button
+            variant="quiet"
+            onClick={onNextPage}
+            disabled={currentPage === totalPages - 1}
+            icon="chevron-next"
+          />
+        </div>
+      </Tailwind>
+    </Box>
+  </HorizontalBox>
+);
+
+const useFilteredData = (
+  radarData: RadarPlotData,
+  selectedMetrics: Record<string, boolean> | undefined
+) =>
+  useMemo(() => {
+    const data: RadarPlotData = {};
     for (const [callId, metricBin] of Object.entries(radarData)) {
       const metrics: {[metric: string]: number} = {};
       for (const [metric, value] of Object.entries(metricBin.metrics)) {
@@ -64,253 +227,195 @@ export const SummaryPlots: React.FC<{
         }
       }
       if (Object.keys(metrics).length > 0) {
-        filteredData[callId] = {
+        data[callId] = {
           metrics,
           name: metricBin.name,
           color: metricBin.color,
         };
       }
     }
-    return filteredData;
+    return data;
   }, [radarData, selectedMetrics]);
 
-  return (
-    <VerticalBox
-      sx={{
-        paddingLeft: STANDARD_PADDING,
-        paddingRight: STANDARD_PADDING,
-        flex: '1 1 auto',
-        width: '100%',
-      }}>
-      <HorizontalBox
-        sx={{
-          width: '100%',
-          alignItems: 'center',
-          justifyContent: 'flex-start',
-        }}>
-        <Box
-          sx={{
-            fontSize: '1.5em',
-            fontWeight: 'bold',
-          }}>
-          Summary Metrics
-        </Box>
-        <Box sx={{marginLeft: 'auto'}}>
-          <div style={{display: 'flex', alignItems: 'center'}}>
-            <div style={{marginRight: '4px'}}>Configure displayed metrics</div>
-            <MetricsSelector
-              selectedMetrics={selectedMetrics}
-              setSelectedMetrics={setSelectedMetrics}
-              allMetrics={Array.from(allMetricNames)}
-            />
-          </div>
-        </Box>
-      </HorizontalBox>
-      <HorizontalBox
-        sx={{
-          flexWrap: 'wrap',
-        }}>
-        <Box
-          sx={{
-            flex: '1 2 ' + PLOT_HEIGHT + 'px',
-            height: PLOT_HEIGHT,
-            borderRadius: BOX_RADIUS,
-            border: STANDARD_BORDER,
-            overflow: 'hidden',
-            alignContent: 'center',
-            width: PLOT_HEIGHT,
-          }}>
-          <PlotlyRadarPlot
-            height={PLOT_HEIGHT}
-            data={filteredPlotlyRadarData}
-          />
-        </Box>
-        <Box
-          sx={{
-            flex: '2 1 ' + PLOT_HEIGHT + 'px',
-            height: PLOT_HEIGHT,
-            overflow: 'hidden',
-            borderRadius: BOX_RADIUS,
-            border: STANDARD_BORDER,
-            padding: PLOT_PADDING,
-            width: PLOT_HEIGHT,
-          }}>
-          <PlotlyBarPlot height={PLOT_HEIGHT} data={filteredPlotlyRadarData} />
-        </Box>
-      </HorizontalBox>
-    </VerticalBox>
-  );
-};
+function getMetricValuesFromRadarData(radarData: RadarPlotData): {
+  [metric: string]: number[];
+} {
+  const metricValues: {[metric: string]: number[]} = {};
+  // Gather all values for each metric
+  Object.values(radarData).forEach(callData => {
+    Object.entries(callData.metrics).forEach(([metric, value]) => {
+      if (!metricValues[metric]) {
+        metricValues[metric] = [];
+      }
+      metricValues[metric].push(value);
+    });
+  });
+  return metricValues;
+}
 
-const MetricsSelector: React.FC<{
-  setSelectedMetrics: (newModel: Record<string, boolean>) => void;
-  selectedMetrics: Record<string, boolean> | undefined;
-  allMetrics: string[];
-}> = ({setSelectedMetrics, selectedMetrics, allMetrics}) => {
-  const [search, setSearch] = useState('');
-
-  const ref = useRef<HTMLDivElement>(null);
-  const [anchorEl, setAnchorEl] = useState<null | HTMLElement>(null);
-  const onClick = (event: React.MouseEvent<HTMLElement>) => {
-    setAnchorEl(anchorEl ? null : ref.current);
-    setSearch('');
+function getMetricMinsFromRadarData(radarData: RadarPlotData): {
+  [metric: string]: number;
+} {
+  const metricValues = getMetricValuesFromRadarData(radarData);
+  const metricMins: {[metric: string]: number} = {};
+  Object.entries(metricValues).forEach(([metric, values]) => {
+    metricMins[metric] = Math.min(...values);
+  });
+  return metricMins;
+}
+
+function normalizeDataForRadarPlot(radarData: RadarPlotData): RadarPlotData {
+  const metricMins = getMetricMinsFromRadarData(radarData);
+
+  const normalizedData: RadarPlotData = {};
+  Object.entries(radarData).forEach(([callId, callData]) => {
+    normalizedData[callId] = {
+      name: callData.name,
+      color: callData.color,
+      metrics: {},
+    };
+
+    Object.entries(callData.metrics).forEach(([metric, value]) => {
+      const min = metricMins[metric];
+      // Only shift values if there are negative values
+      const normalizedValue = min < 0 ? value - min : value;
+      normalizedData[callId].metrics[metric] = normalizedValue;
+    });
+  });
+
+  return normalizedData;
+}
+
+const useBarPlotData = (filteredData: RadarPlotData) =>
+  useMemo(() => {
+    const metrics: {
+      [metric: string]: {
+        callIds: string[];
+        values: number[];
+        name: string;
+        colors: string[];
+      };
+    } = {};
+
+    // Reorganize data by metric instead of by call
+    for (const [callId, metricBin] of Object.entries(filteredData)) {
+      for (const [metric, value] of Object.entries(metricBin.metrics)) {
+        if (!metrics[metric]) {
+          metrics[metric] = {callIds: [], values: [], name: metric, colors: []};
+        }
+        metrics[metric].callIds.push(callId);
+        metrics[metric].values.push(value);
+        metrics[metric].colors.push(metricBin.color);
+      }
+    }
+
+    // Convert metrics object to Plotly data format
+    return Object.entries(metrics).map(([metric, metricBin]) => {
+      const maxY = Math.max(...metricBin.values) * 1.1;
+      const minY = Math.min(...metricBin.values, 0);
+      const plotlyData: Plotly.Data = {
+        type: 'bar',
+        y: metricBin.values,
+        x: metricBin.callIds,
+        text: metricBin.values.map(value => value.toFixed(3)),
+        textposition: 'outside',
+        textfont: {size: 14, color: 'black'},
+        name: metric,
+        marker: {color: metricBin.colors},
+      };
+      return {plotlyData, yRange: [minY, maxY] as [number, number]};
+    });
+  }, [filteredData]);
+
+const useContainerDimensions = () => {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [containerWidth, setContainerWidth] = useState(0);
+  const [isInitialRender, setIsInitialRender] = useState(true);
+  const [currentPage, setCurrentPage] = useState(0);
+
+  useEffect(() => {
+    const updateWidth = () => {
+      if (containerRef.current) {
+        setContainerWidth(containerRef.current.offsetWidth);
+      }
+    };
+
+    updateWidth();
+    setIsInitialRender(false);
+
+    window.addEventListener('resize', updateWidth);
+    return () => window.removeEventListener('resize', updateWidth);
+  }, []);
+
+  const plotsPerPage = useMemo(() => {
+    return Math.max(1, Math.floor(containerWidth / PLOT_HEIGHT));
+  }, [containerWidth]);
+
+  return {
+    containerRef,
+    isInitialRender,
+    plotsPerPage,
+    currentPage,
+    setCurrentPage,
   };
-  const open = Boolean(anchorEl);
-  const id = open ? 'simple-popper' : undefined;
+};
 
-  const filteredCols = search
-    ? allMetrics.filter(col => col.toLowerCase().includes(search.toLowerCase()))
-    : allMetrics;
+const usePaginatedPlots = (
+  filteredData: RadarPlotData,
+  barPlotData: Array<{plotlyData: Plotly.Data; yRange: [number, number]}>,
+  plotsPerPage: number,
+  currentPage: number
+) => {
+  const radarPlotWidth = 2;
+  const totalBarPlots = barPlotData.length;
+  const totalPlotWidth = radarPlotWidth + totalBarPlots;
+  const totalPages = Math.ceil(totalPlotWidth / plotsPerPage);
 
-  const shownMetrics = Object.values(selectedMetrics ?? {}).filter(Boolean);
+  const plotsToShow = useMemo(() => {
+    // First page always shows radar plot
+    if (currentPage === 0) {
+      const availableSpace = plotsPerPage - radarPlotWidth;
+      return [
+        <RadarPlotBox key="radar" data={filteredData} />,
+        ...barPlotData
+          .slice(0, availableSpace)
+          .map((plot, index) => (
+            <BarPlotBox key={`bar-${index}`} plot={plot} />
+          )),
+      ];
+    } else {
+      // Subsequent pages show only bar plots
+      const startIdx =
+        (currentPage - 1) * plotsPerPage + (plotsPerPage - radarPlotWidth);
+      const endIdx = startIdx + plotsPerPage;
+      return barPlotData
+        .slice(startIdx, endIdx)
+        .map((plot, index) => (
+          <BarPlotBox key={`bar-${startIdx + index}`} plot={plot} />
+        ));
+    }
+  }, [currentPage, plotsPerPage, filteredData, barPlotData]);
 
-  const numHidden = allMetrics.length - shownMetrics.length;
-  const buttonSuffix = search ? `(${filteredCols.length})` : 'all';
+  // Calculate pagination details
+  const totalPlots = barPlotData.length + 1; // +1 for the radar plot
+  const startIndex =
+    currentPage === 0 ? 1 : Math.min(plotsPerPage + 1, totalPlots);
+  const endIndex =
+    currentPage === 0
+      ? Math.min(plotsToShow.length, totalPlots)
+      : Math.min(startIndex + plotsToShow.length - 1, totalPlots);
 
-  return (
-    <>
-      <span ref={ref}>
-        <Button
-          variant="ghost"
-          icon="column"
-          tooltip="Manage metrics"
-          onClick={onClick}
-        />
-      </span>
-      <Popover
-        id={id}
-        open={open}
-        anchorEl={anchorEl}
-        anchorOrigin={{
-          vertical: 'bottom',
-          horizontal: 'center',
-        }}
-        transformOrigin={{
-          vertical: 'top',
-          horizontal: 'center',
-        }}
-        slotProps={{
-          paper: {
-            sx: {
-              overflow: 'visible',
-            },
-          },
-        }}
-        onClose={() => setAnchorEl(null)}
-        TransitionComponent={DraggableGrow}>
-        <Tailwind>
-          <div className="min-w-[360px] p-12">
-            <DraggableHandle>
-              <div className="flex items-center pb-8">
-                <div className="flex-auto text-xl font-semibold">
-                  Manage metrics
-                </div>
-                <div className="ml-16 text-moon-500">
-                  {maybePluralize(numHidden, 'hidden column', 's')}
-                </div>
-              </div>
-            </DraggableHandle>
-            <div className="mb-8">
-              <TextField
-                placeholder="Filter columns"
-                autoFocus
-                value={search}
-                onChange={setSearch}
-              />
-            </div>
-            <div className="max-h-[300px] overflow-auto">
-              {Array.from(allMetrics).map((metric: string) => {
-                const value = metric;
-                const idSwitch = `toggle-vis_${value}`;
-                const checked = selectedMetrics?.[metric] ?? false;
-                const label = metric;
-                const disabled = false;
-                if (
-                  search &&
-                  !label.toLowerCase().includes(search.toLowerCase())
-                ) {
-                  return null;
-                }
-                return (
-                  <div key={value}>
-                    <div
-                      className={classNames(
-                        'flex items-center py-2',
-                        disabled ? 'opacity-40' : ''
-                      )}>
-                      <Switch.Root
-                        id={idSwitch}
-                        size="small"
-                        checked={checked}
-                        onCheckedChange={isOn => {
-                          setSelectedMetrics(
-                            isOn
-                              ? {...selectedMetrics, [metric]: true}
-                              : {...selectedMetrics, [metric]: false}
-                          );
-                        }}
-                        disabled={disabled}>
-                        <Switch.Thumb size="small" checked={checked} />
-                      </Switch.Root>
-                      <label
-                        htmlFor={idSwitch}
-                        className={classNames(
-                          'ml-6',
-                          disabled ? '' : 'cursor-pointer'
-                        )}>
-                        {label}
-                      </label>
-                    </div>
-                  </div>
-                );
-              })}
-            </div>
-            <div className="mt-8 flex items-center">
-              <Button
-                size="small"
-                variant="quiet"
-                icon="hide-hidden"
-                disabled={filteredCols.length === 0}
-                onClick={() => {
-                  const newModel = {...selectedMetrics};
-                  for (const metric of filteredCols) {
-                    newModel[metric] = false;
-                  }
-                  setSelectedMetrics(newModel);
-                }}>
-                {`Hide ${buttonSuffix}`}
-              </Button>
-              <div className="flex-auto" />
-              <Button
-                size="small"
-                variant="quiet"
-                icon="show-visible"
-                disabled={filteredCols.length === 0}
-                onClick={() => {
-                  const newModel = {...selectedMetrics};
-                  for (const metric of filteredCols) {
-                    newModel[metric] = true;
-                  }
-                  setSelectedMetrics(newModel);
-                }}>
-                {`Show ${buttonSuffix}`}
-              </Button>
-            </div>
-          </div>
-        </Tailwind>
-      </Popover>
-    </>
-  );
+  return {plotsToShow, totalPlots, startIndex, endIndex, totalPages};
 };
 
-const normalizeValues = (values: Array<number | undefined>): number[] => {
+function normalizeValues(values: Array<number | undefined>): number[] {
   // find the max value
   // find the power of 2 that is greater than the max value
   // divide all values by that power of 2
   const maxVal = Math.max(...(values.filter(v => v !== undefined) as number[]));
   const maxPower = Math.ceil(Math.log2(maxVal));
   return values.map(val => (val ? val / 2 ** maxPower : 0));
-};
+}
 
 const useNormalizedPlotDataFromMetrics = (
   state: EvaluationComparisonState

From a7170b4c1e6dbca9b279865b588d4edfee34468f Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Tue, 10 Dec 2024 11:00:13 -0600
Subject: [PATCH 21/52] feat(ui): Support Anthropic calls in Chat View (#3187)

---
 .../Home/Browse3/pages/ChatView/hooks.ts      | 88 +++++++++++++++++++
 1 file changed, 88 insertions(+)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/hooks.ts
index 38b5c820195..33ced58ec49 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/hooks.ts
@@ -246,6 +246,63 @@ export const isTraceCallChatFormatGemini = (call: TraceCallSchema): boolean => {
   );
 };
 
+export const isAnthropicContentBlock = (block: any): boolean => {
+  if (!_.isPlainObject(block)) {
+    return false;
+  }
+  // TODO: Are there other types?
+  if (block.type !== 'text') {
+    return false;
+  }
+  if (!hasStringProp(block, 'text')) {
+    return false;
+  }
+  return true;
+};
+
+export const isAnthropicCompletionFormat = (output: any): boolean => {
+  if (output !== null) {
+    // TODO: Could have additional checks here on things like usage
+    if (
+      _.isPlainObject(output) &&
+      output.type === 'message' &&
+      output.role === 'assistant' &&
+      hasStringProp(output, 'model') &&
+      _.isArray(output.content) &&
+      output.content.every((c: any) => isAnthropicContentBlock(c))
+    ) {
+      return true;
+    }
+    return false;
+  }
+  return true;
+};
+
+type AnthropicContentBlock = {
+  type: 'text';
+  text: string;
+};
+
+export const anthropicContentBlocksToChoices = (
+  blocks: AnthropicContentBlock[],
+  stopReason: string
+): Choice[] => {
+  const choices: Choice[] = [];
+  for (let i = 0; i < blocks.length; i++) {
+    const block = blocks[i];
+    choices.push({
+      index: i,
+      message: {
+        role: 'assistant',
+        content: block.text,
+      },
+      // TODO: What is correct way to map this?
+      finish_reason: stopReason,
+    });
+  }
+  return choices;
+};
+
 export const isTraceCallChatFormatOpenAI = (call: TraceCallSchema): boolean => {
   if (!('messages' in call.inputs)) {
     return false;
@@ -336,6 +393,19 @@ export const normalizeChatRequest = (request: any): ChatRequest => {
       ],
     };
   }
+  // Anthropic has system message as a top-level request field
+  if (hasStringProp(request, 'system')) {
+    return {
+      ...request,
+      messages: [
+        {
+          role: 'system',
+          content: request.system,
+        },
+        ...request.messages,
+      ],
+    };
+  }
   return request as ChatRequest;
 };
 
@@ -360,6 +430,24 @@ export const normalizeChatCompletion = (
       },
     };
   }
+  if (isAnthropicCompletionFormat(completion)) {
+    return {
+      id: completion.id,
+      choices: anthropicContentBlocksToChoices(
+        completion.content,
+        completion.stop_reason
+      ),
+      created: 0,
+      model: completion.model,
+      system_fingerprint: '',
+      usage: {
+        prompt_tokens: completion.usage.input_tokens,
+        completion_tokens: completion.usage.output_tokens,
+        total_tokens:
+          completion.usage.input_tokens + completion.usage.output_tokens,
+      },
+    };
+  }
   return completion as ChatCompletion;
 };
 

From 935f68cbfdd55a2f65a476b201d2227167416097 Mon Sep 17 00:00:00 2001
From: Connie Lee <connie.lee0317@gmail.com>
Date: Tue, 10 Dec 2024 09:28:08 -0800
Subject: [PATCH 22/52] feat(ui): Add ExpandingPill component (#3184)

---
 weave-js/src/components/Tag/Pill.tsx | 39 ++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/weave-js/src/components/Tag/Pill.tsx b/weave-js/src/components/Tag/Pill.tsx
index 6f734f9cbfc..ed958a0c543 100644
--- a/weave-js/src/components/Tag/Pill.tsx
+++ b/weave-js/src/components/Tag/Pill.tsx
@@ -59,3 +59,42 @@ export const IconOnlyPill: FC<IconOnlyPillProps> = ({
     </Tailwind>
   );
 };
+
+export type ExpandingPillProps = {
+  className?: string;
+  color?: TagColorName;
+  icon: IconName;
+  label: string;
+};
+export const ExpandingPill = ({
+  className,
+  color,
+  icon,
+  label,
+}: ExpandingPillProps) => {
+  const classes = useTagClasses({color, isInteractive: true});
+  return (
+    <Tailwind>
+      <div
+        className={twMerge(
+          classes,
+          'rounded-2xl p-4',
+          'max-w-[22px] hover:max-w-[25ch]',
+          // note: transition delay + duration = 500ms, which matches our default tooltip delay
+          // so that when pairing a pill with a tooltip, the tooltip doesn't lag and appear later
+          'transition-[max-width] delay-200 duration-300',
+          '[&:hover_span]:opacity-100 [&_span]:opacity-0',
+          className
+        )}>
+        <Icon className="h-14 w-14 shrink-0" name={icon} role="presentation" />
+        <span
+          className={twMerge(
+            'max-w-[24ch] overflow-hidden text-ellipsis whitespace-nowrap px-4',
+            'transition-opacity delay-200 duration-300'
+          )}>
+          {label}
+        </span>
+      </div>
+    </Tailwind>
+  );
+};

From a14edbe6f6c37f3e9094b50ffdd2adc5bd3ecb65 Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Tue, 10 Dec 2024 11:52:04 -0600
Subject: [PATCH 23/52] chore(weave): ui_url error gives a hint about the
 problem (#3190)

---
 weave/trace/weave_client.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/weave/trace/weave_client.py b/weave/trace/weave_client.py
index 86ba7b8653e..0eca3fcbedb 100644
--- a/weave/trace/weave_client.py
+++ b/weave/trace/weave_client.py
@@ -239,7 +239,9 @@ def func_name(self) -> str:
     @property
     def feedback(self) -> RefFeedbackQuery:
         if not self.id:
-            raise ValueError("Can't get feedback for call without ID")
+            raise ValueError(
+                "Can't get feedback for call without ID, was `weave.init` called?"
+            )
 
         if self._feedback is None:
             try:
@@ -253,7 +255,9 @@ def feedback(self) -> RefFeedbackQuery:
     @property
     def ui_url(self) -> str:
         if not self.id:
-            raise ValueError("Can't get URL for call without ID")
+            raise ValueError(
+                "Can't get URL for call without ID, was `weave.init` called?"
+            )
 
         try:
             entity, project = self.project_id.split("/")
@@ -265,7 +269,9 @@ def ui_url(self) -> str:
     def ref(self) -> CallRef:
         entity, project = self.project_id.split("/")
         if not self.id:
-            raise ValueError("Can't get ref for call without ID")
+            raise ValueError(
+                "Can't get ref for call without ID, was `weave.init` called?"
+            )
 
         return CallRef(entity, project, self.id)
 
@@ -273,7 +279,9 @@ def ref(self) -> CallRef:
     def children(self) -> CallsIter:
         client = weave_client_context.require_weave_client()
         if not self.id:
-            raise ValueError("Can't get children of call without ID")
+            raise ValueError(
+                "Can't get children of call without ID, was `weave.init` called?"
+            )
 
         client = weave_client_context.require_weave_client()
         return CallsIter(

From c19fe6bf246c058a3cbaa11bd544eb2e2fbbd24c Mon Sep 17 00:00:00 2001
From: brianlund-wandb <brian.lund@wandb.com>
Date: Tue, 10 Dec 2024 10:36:51 -0800
Subject: [PATCH 24/52] fix(app): render bounding boxes on media panel without
 points (#3158)

* WB-16623 coercing empty point to default, targeting camera on bounding box when point cloud empty

* code style cleanup
---
 .../src/common/util/SdkPointCloudToBabylon.test.ts | 14 ++++++++++++++
 weave-js/src/common/util/SdkPointCloudToBabylon.ts |  2 +-
 weave-js/src/common/util/render_babylon.ts         | 13 +++++++++++--
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/weave-js/src/common/util/SdkPointCloudToBabylon.test.ts b/weave-js/src/common/util/SdkPointCloudToBabylon.test.ts
index 95c7639dc4a..df5eca57b46 100644
--- a/weave-js/src/common/util/SdkPointCloudToBabylon.test.ts
+++ b/weave-js/src/common/util/SdkPointCloudToBabylon.test.ts
@@ -4,6 +4,7 @@ import {
   DEFAULT_POINT_COLOR,
   getFilteringOptionsForPointCloud,
   getVertexCompatiblePositionsAndColors,
+  loadPointCloud,
   MAX_BOUNDING_BOX_LABELS_FOR_DISPLAY,
   MaxAlphaValue,
 } from './SdkPointCloudToBabylon';
@@ -174,3 +175,16 @@ describe('getFilteringOptionsForPointCloud', () => {
     expect(newClassIdToLabel.get(49)).toEqual('label49');
   });
 });
+describe('loadPointCloud', () => {
+  it('appropriate defaults set when loading point cloud from file', () => {
+    const fileContents = JSON.stringify({
+      boxes: [],
+      points: [[]],
+      type: 'lidar/beta',
+      vectors: [],
+    });
+    const babylonPointCloud = loadPointCloud(fileContents);
+    expect(babylonPointCloud.points).toHaveLength(1);
+    expect(babylonPointCloud.points[0].position).toEqual([0, 0, 0]);
+  });
+});
diff --git a/weave-js/src/common/util/SdkPointCloudToBabylon.ts b/weave-js/src/common/util/SdkPointCloudToBabylon.ts
index 274e1676be4..d52682743ee 100644
--- a/weave-js/src/common/util/SdkPointCloudToBabylon.ts
+++ b/weave-js/src/common/util/SdkPointCloudToBabylon.ts
@@ -160,7 +160,7 @@ export const handlePoints = (object3D: Object3DScene): ScenePoint[] => {
   // Draw Points
   return truncatedPoints.map(point => {
     const [x, y, z, r, g, b] = point;
-    const position: Position = [x, y, z];
+    const position: Position = [x ?? 0, y ?? 0, z ?? 0];
     const category = r;
 
     if (r !== undefined && g !== undefined && b !== undefined) {
diff --git a/weave-js/src/common/util/render_babylon.ts b/weave-js/src/common/util/render_babylon.ts
index 10aee3f6c51..ebd213c2677 100644
--- a/weave-js/src/common/util/render_babylon.ts
+++ b/weave-js/src/common/util/render_babylon.ts
@@ -394,6 +394,15 @@ const pointCloudScene = (
   // Apply vertexData to custom mesh
   vertexData.applyToMesh(pcMesh);
 
+  // A file without any points defined still includes a single, empty "point".
+  // In order to play nice with Babylon, we position this empty point at 0,0,0.
+  // Hence, a pointCloud with a single point at 0,0,0 is likely empty.
+  const isEmpty =
+    pointCloud.points.length === 1 &&
+    pointCloud.points[0].position[0] === 0 &&
+    pointCloud.points[0].position[1] === 0 &&
+    pointCloud.points[0].position[2] === 0;
+
   camera.parent = pcMesh;
 
   const pcMaterial = new Babylon.StandardMaterial('mat', scene);
@@ -472,8 +481,8 @@ const pointCloudScene = (
         new Vector3(edges.length * 2, edges.length * 2, edges.length * 2)
       );
 
-    // If we are iterating over camera, target a box
-    if (index === meta?.cameraIndex) {
+    // If we are iterating over camera or the cloud is empty, target a box
+    if (index === meta?.cameraIndex || (index === 0 && isEmpty)) {
       camera.position = center.add(new Vector3(0, 0, 1000));
       camera.target = center;
       camera.zoomOn([lines]);

From 0966c9fb21d00fa6b3ed9019de2c4c471fc6ec30 Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Tue, 10 Dec 2024 12:08:50 -0800
Subject: [PATCH 25/52] fix(ui): feedback header when empty (#3191)

---
 .../Browse3/feedback/StructuredFeedback/FeedbackSidebar.tsx     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/FeedbackSidebar.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/FeedbackSidebar.tsx
index ef6bcbd69ff..0b3c9603fef 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/FeedbackSidebar.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/FeedbackSidebar.tsx
@@ -96,7 +96,7 @@ export const FeedbackSidebar = ({
         <div className="text-lg font-semibold">Feedback</div>
         <div className="flex-grow" />
       </div>
-      <div className="min-h-1 mb-8 h-1 flex-grow overflow-auto bg-moon-300" />
+      <div className="min-h-1 mb-8 h-1 overflow-auto bg-moon-300" />
       {humanAnnotationSpecs.length > 0 ? (
         <>
           <div className="ml-6 h-full flex-grow overflow-auto">

From 35a78d6e83006ea4462d48485651213a4afbd381 Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Tue, 10 Dec 2024 16:20:17 -0800
Subject: [PATCH 26/52] chore(ui): make eval section header draggable (#2637)

---
 .../CompareEvaluationsPage.tsx                |  14 +--
 .../compareEvaluationsContext.tsx             |  25 ++--
 .../pages/CompareEvaluationsPage/ecpState.ts  |  63 +++++++---
 .../pages/CompareEvaluationsPage/ecpTypes.ts  |   1 +
 .../ComparisonDefinitionSection.tsx           | 110 ++++++++++++------
 .../EvaluationDefinition.tsx                  |  83 +------------
 .../DraggableSection/DraggableItem.tsx        | 103 ++++++++++++++++
 .../DraggableSection/DraggableSection.tsx     |  34 ++++++
 .../ExampleCompareSection.tsx                 |   2 +-
 .../ExampleFilterSection.tsx                  |   4 +-
 .../ScorecardSection/ScorecardSection.tsx     |   3 +-
 11 files changed, 280 insertions(+), 162 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/DraggableSection/DraggableItem.tsx
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/DraggableSection/DraggableSection.tsx

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx
index b302b0262ae..478c4887546 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage.tsx
@@ -77,8 +77,6 @@ export const CompareEvaluationsPage: React.FC<
 export const CompareEvaluationsPageContent: React.FC<
   CompareEvaluationsPageProps
 > = props => {
-  const [baselineEvaluationCallId, setBaselineEvaluationCallId] =
-    React.useState<string | null>(null);
   const [comparisonDimensions, setComparisonDimensions] =
     React.useState<ComparisonDimensionsType | null>(null);
 
@@ -104,14 +102,6 @@ export const CompareEvaluationsPageContent: React.FC<
     [comparisonDimensions]
   );
 
-  React.useEffect(() => {
-    // Only update the baseline if we are switching evaluations, if there
-    // is more than 1, we are in the compare view and baseline is auto set
-    if (props.evaluationCallIds.length === 1) {
-      setBaselineEvaluationCallId(props.evaluationCallIds[0]);
-    }
-  }, [props.evaluationCallIds]);
-
   if (props.evaluationCallIds.length === 0) {
     return <div>No evaluations to compare</div>;
   }
@@ -120,13 +110,11 @@ export const CompareEvaluationsPageContent: React.FC<
     <CompareEvaluationsProvider
       entity={props.entity}
       project={props.project}
+      initialEvaluationCallIds={props.evaluationCallIds}
       selectedMetrics={props.selectedMetrics}
       setSelectedMetrics={props.setSelectedMetrics}
-      initialEvaluationCallIds={props.evaluationCallIds}
-      baselineEvaluationCallId={baselineEvaluationCallId ?? undefined}
       comparisonDimensions={comparisonDimensions ?? undefined}
       onEvaluationCallIdsUpdate={props.onEvaluationCallIdsUpdate}
-      setBaselineEvaluationCallId={setBaselineEvaluationCallId}
       setComparisonDimensions={setComparisonDimensionsAndClearInputDigest}
       selectedInputDigest={selectedInputDigest ?? undefined}
       setSelectedInputDigest={setSelectedInputDigest}>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/compareEvaluationsContext.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/compareEvaluationsContext.tsx
index b65658a890a..638565e8ad6 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/compareEvaluationsContext.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/compareEvaluationsContext.tsx
@@ -10,9 +10,6 @@ import {ComparisonDimensionsType} from './ecpState';
 
 const CompareEvaluationsContext = React.createContext<{
   state: EvaluationComparisonState;
-  setBaselineEvaluationCallId: React.Dispatch<
-    React.SetStateAction<string | null>
-  >;
   setComparisonDimensions: React.Dispatch<
     React.SetStateAction<ComparisonDimensionsType | null>
   >;
@@ -20,6 +17,7 @@ const CompareEvaluationsContext = React.createContext<{
   setSelectedMetrics: (newModel: Record<string, boolean>) => void;
   addEvaluationCall: (newCallId: string) => void;
   removeEvaluationCall: (callId: string) => void;
+  setEvaluationCallOrder: (newCallIdOrder: string[]) => void;
 } | null>(null);
 
 export const useCompareEvaluationsState = () => {
@@ -33,34 +31,26 @@ export const useCompareEvaluationsState = () => {
 export const CompareEvaluationsProvider: React.FC<{
   entity: string;
   project: string;
+  initialEvaluationCallIds: string[];
   selectedMetrics: Record<string, boolean> | null;
   setSelectedMetrics: (newModel: Record<string, boolean>) => void;
 
-  initialEvaluationCallIds: string[];
   onEvaluationCallIdsUpdate: (newEvaluationCallIds: string[]) => void;
-  setBaselineEvaluationCallId: React.Dispatch<
-    React.SetStateAction<string | null>
-  >;
   setComparisonDimensions: React.Dispatch<
     React.SetStateAction<ComparisonDimensionsType | null>
   >;
   setSelectedInputDigest: React.Dispatch<React.SetStateAction<string | null>>;
-  baselineEvaluationCallId?: string;
   comparisonDimensions?: ComparisonDimensionsType;
   selectedInputDigest?: string;
 }> = ({
   entity,
   project,
+  initialEvaluationCallIds,
   selectedMetrics,
   setSelectedMetrics,
-
-  initialEvaluationCallIds,
   onEvaluationCallIdsUpdate,
-  setBaselineEvaluationCallId,
   setComparisonDimensions,
-
   setSelectedInputDigest,
-  baselineEvaluationCallId,
   comparisonDimensions,
   selectedInputDigest,
   children,
@@ -77,7 +67,6 @@ export const CompareEvaluationsProvider: React.FC<{
     entity,
     project,
     evaluationCallIds,
-    baselineEvaluationCallId,
     comparisonDimensions,
     selectedInputDigest,
     selectedMetrics ?? undefined
@@ -89,7 +78,6 @@ export const CompareEvaluationsProvider: React.FC<{
     }
     return {
       state: initialState.result,
-      setBaselineEvaluationCallId,
       setComparisonDimensions,
       setSelectedInputDigest,
       setSelectedMetrics,
@@ -105,14 +93,17 @@ export const CompareEvaluationsProvider: React.FC<{
         setEvaluationCallIds(newEvaluationCallIds);
         onEvaluationCallIdsUpdate(newEvaluationCallIds);
       },
+      setEvaluationCallOrder: (newCallIdOrder: string[]) => {
+        setEvaluationCallIds(newCallIdOrder);
+        onEvaluationCallIdsUpdate(newCallIdOrder);
+      },
     };
   }, [
     initialState.loading,
     initialState.result,
+    setEvaluationCallIds,
     evaluationCallIds,
     onEvaluationCallIdsUpdate,
-    setEvaluationCallIds,
-    setBaselineEvaluationCallId,
     setComparisonDimensions,
     setSelectedInputDigest,
     setSelectedMetrics,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/ecpState.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/ecpState.ts
index 35f95dbf14f..e5c1b03d60a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/ecpState.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/ecpState.ts
@@ -18,14 +18,14 @@ import {getMetricIds} from './ecpUtil';
 export type EvaluationComparisonState = {
   // The normalized data for the evaluations
   data: EvaluationComparisonData;
-  // The evaluation call id of the baseline model
-  baselineEvaluationCallId: string;
   // The dimensions to compare & filter results
   comparisonDimensions?: ComparisonDimensionsType;
   // The current digest which is in view
   selectedInputDigest?: string;
   // The selected metrics to display
   selectedMetrics?: Record<string, boolean>;
+  // Ordered call Ids
+  evaluationCallIdsOrdered: string[];
 };
 
 export type ComparisonDimensionsType = Array<{
@@ -43,12 +43,14 @@ export const useEvaluationComparisonState = (
   entity: string,
   project: string,
   evaluationCallIds: string[],
-  baselineEvaluationCallId?: string,
   comparisonDimensions?: ComparisonDimensionsType,
   selectedInputDigest?: string,
   selectedMetrics?: Record<string, boolean>
 ): Loadable<EvaluationComparisonState> => {
-  const data = useEvaluationComparisonData(entity, project, evaluationCallIds);
+  const orderedCallIds = useMemo(() => {
+    return getCallIdsOrderedForQuery(evaluationCallIds);
+  }, [evaluationCallIds]);
+  const data = useEvaluationComparisonData(entity, project, orderedCallIds);
 
   const value = useMemo(() => {
     if (data.result == null || data.loading) {
@@ -92,42 +94,45 @@ export const useEvaluationComparisonState = (
       loading: false,
       result: {
         data: data.result,
-        baselineEvaluationCallId:
-          baselineEvaluationCallId ?? evaluationCallIds[0],
         comparisonDimensions: newComparisonDimensions,
         selectedInputDigest,
         selectedMetrics,
+        evaluationCallIdsOrdered: evaluationCallIds,
       },
     };
   }, [
     data.result,
     data.loading,
-    baselineEvaluationCallId,
-    evaluationCallIds,
     comparisonDimensions,
     selectedInputDigest,
     selectedMetrics,
+    evaluationCallIds,
   ]);
 
   return value;
 };
 
+export const getOrderedCallIds = (state: EvaluationComparisonState) => {
+  return Array.from(state.evaluationCallIdsOrdered);
+};
+
+export const getBaselineCallId = (state: EvaluationComparisonState) => {
+  return getOrderedCallIds(state)[0];
+};
+
 /**
- * Should use this over keys of `state.data.evaluationCalls` because it ensures the baseline
- * evaluation call is first.
+ * Sort call IDs to ensure consistent order for memoized query params
  */
-export const getOrderedCallIds = (state: EvaluationComparisonState) => {
-  const initial = Object.keys(state.data.evaluationCalls);
-  moveItemToFront(initial, state.baselineEvaluationCallId);
-  return initial;
+const getCallIdsOrderedForQuery = (callIds: string[]) => {
+  return Array.from(callIds).sort();
 };
 
 /**
  * Should use this over keys of `state.data.models` because it ensures the baseline model is first.
  */
 export const getOrderedModelRefs = (state: EvaluationComparisonState) => {
-  const baselineRef =
-    state.data.evaluationCalls[state.baselineEvaluationCallId].modelRef;
+  const baselineCallId = getBaselineCallId(state);
+  const baselineRef = state.data.evaluationCalls[baselineCallId].modelRef;
   const refs = Object.keys(state.data.models);
   // Make sure the baseline model is first
   moveItemToFront(refs, baselineRef);
@@ -145,3 +150,29 @@ const moveItemToFront = <T>(arr: T[], item: T): T[] => {
   }
   return arr;
 };
+
+export const getOrderedEvalsWithNewBaseline = (
+  evaluationCallIds: string[],
+  newBaselineCallId: string
+) => {
+  return moveItemToFront(evaluationCallIds, newBaselineCallId);
+};
+
+export const swapEvaluationCalls = (
+  evaluationCallIds: string[],
+  ndx1: number,
+  ndx2: number
+): string[] => {
+  return swapArrayItems(evaluationCallIds, ndx1, ndx2);
+};
+
+const swapArrayItems = <T>(arr: T[], ndx1: number, ndx2: number): T[] => {
+  if (ndx1 < 0 || ndx2 < 0 || ndx1 >= arr.length || ndx2 >= arr.length) {
+    throw new Error('Index out of bounds');
+  }
+  const newArr = [...arr];
+  const from = newArr[ndx1];
+  newArr[ndx1] = newArr[ndx2];
+  newArr[ndx2] = from;
+  return newArr;
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/ecpTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/ecpTypes.ts
index 7454e0707b4..b4642fae240 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/ecpTypes.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/ecpTypes.ts
@@ -18,6 +18,7 @@ export type EvaluationComparisonData = {
   };
 
   // EvaluationCalls are the specific calls of an evaluation.
+  // The visual order of the evaluation calls is determined by the order of the keys.
   evaluationCalls: {
     [callId: string]: EvaluationCall;
   };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx
index 66ce56c4cb0..2704a66cbea 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/ComparisonDefinitionSection.tsx
@@ -12,44 +12,81 @@ import {
 import {useCallsForQuery} from '../../../CallsPage/callsTableQuery';
 import {useEvaluationsFilter} from '../../../CallsPage/evaluationsFilter';
 import {Id} from '../../../common/Id';
+import {opNiceName} from '../../../common/Links';
 import {useWFHooks} from '../../../wfReactInterface/context';
 import {
   CallSchema,
   ObjectVersionKey,
 } from '../../../wfReactInterface/wfDataModelHooksInterface';
 import {useCompareEvaluationsState} from '../../compareEvaluationsContext';
-import {STANDARD_PADDING} from '../../ecpConstants';
-import {getOrderedCallIds} from '../../ecpState';
-import {EvaluationComparisonState} from '../../ecpState';
+import {
+  EvaluationComparisonState,
+  getOrderedCallIds,
+  getOrderedEvalsWithNewBaseline,
+  swapEvaluationCalls,
+} from '../../ecpState';
 import {HorizontalBox} from '../../Layout';
-import {EvaluationDefinition, VerticalBar} from './EvaluationDefinition';
+import {ItemDef} from '../DraggableSection/DraggableItem';
+import {DraggableSection} from '../DraggableSection/DraggableSection';
+import {VerticalBar} from './EvaluationDefinition';
 
 export const ComparisonDefinitionSection: React.FC<{
   state: EvaluationComparisonState;
 }> = props => {
-  const evalCallIds = useMemo(
-    () => getOrderedCallIds(props.state),
-    [props.state]
-  );
+  const {setEvaluationCallOrder, removeEvaluationCall} =
+    useCompareEvaluationsState();
+
+  const callIds = useMemo(() => {
+    return getOrderedCallIds(props.state);
+  }, [props.state]);
+
+  const items: ItemDef[] = useMemo(() => {
+    return callIds.map(callId => ({
+      key: 'evaluations',
+      value: callId,
+      label: props.state.data.evaluationCalls[callId]?.name ?? callId,
+    }));
+  }, [callIds, props.state.data.evaluationCalls]);
+
+  const onSetBaseline = (value: string | null) => {
+    if (!value) {
+      return;
+    }
+    const newSortOrder = getOrderedEvalsWithNewBaseline(callIds, value);
+    setEvaluationCallOrder(newSortOrder);
+  };
+  const onRemoveItem = (value: string) => removeEvaluationCall(value);
+  const onSortEnd = ({
+    oldIndex,
+    newIndex,
+  }: {
+    oldIndex: number;
+    newIndex: number;
+  }) => {
+    const newSortOrder = swapEvaluationCalls(callIds, oldIndex, newIndex);
+    setEvaluationCallOrder(newSortOrder);
+  };
 
   return (
-    <HorizontalBox
-      sx={{
-        alignItems: 'center',
-        paddingLeft: STANDARD_PADDING,
-        paddingRight: STANDARD_PADDING,
-        width: '100%',
-        overflow: 'auto',
-      }}>
-      {evalCallIds.map((key, ndx) => {
-        return (
-          <React.Fragment key={key}>
-            <EvaluationDefinition state={props.state} callId={key} />
-          </React.Fragment>
-        );
-      })}
-      <AddEvaluationButton state={props.state} />
-    </HorizontalBox>
+    <Tailwind>
+      <div className="flex w-full items-center gap-4 overflow-x-auto px-16">
+        <HorizontalBox>
+          <DraggableSection
+            useDragHandle
+            axis="x"
+            state={props.state}
+            items={items}
+            onSetBaseline={onSetBaseline}
+            onRemoveItem={onRemoveItem}
+            onSortEnd={onSortEnd}
+            useWindowAsScrollContainer
+          />
+        </HorizontalBox>
+        <HorizontalBox>
+          <AddEvaluationButton state={props.state} />
+        </HorizontalBox>
+      </div>
+    </Tailwind>
   );
 };
 
@@ -81,7 +118,7 @@ const ModelRefLabel: React.FC<{modelRef: string}> = props => {
   const objectVersion = useObjectVersion(objVersionKey);
   return (
     <span className="ml-2">
-      {objectVersion.result?.objectId}:{objectVersion.result?.versionIndex}
+      {objectVersion.result?.objectId}:v{objectVersion.result?.versionIndex}
     </span>
   );
 };
@@ -119,10 +156,9 @@ const AddEvaluationButton: React.FC<{
 
   const evalsNotComparing = useMemo(() => {
     return calls.result.filter(
-      call =>
-        !Object.keys(props.state.data.evaluationCalls).includes(call.callId)
+      call => !getOrderedCallIds(props.state).includes(call.callId)
     );
-  }, [calls.result, props.state.data.evaluationCalls]);
+  }, [calls.result, props.state]);
 
   const [menuOptions, setMenuOptions] =
     useState<CallSchema[]>(evalsNotComparing);
@@ -222,12 +258,18 @@ const AddEvaluationButton: React.FC<{
                     variant="ghost"
                     size="small"
                     className="pb-8 pt-8 font-['Source_Sans_Pro'] text-base font-normal text-moon-800"
-                    onClick={() => {
-                      addEvaluationCall(call.callId);
-                    }}>
+                    onClick={() => addEvaluationCall(call.callId)}>
                     <>
-                      <span>{call.displayName ?? call.spanName}</span>
-                      <Id id={call.callId} type="Call" className="ml-0 mr-4" />
+                      <span className="max-w-[250px] flex-shrink flex-grow overflow-hidden text-ellipsis whitespace-nowrap">
+                        {call.displayName ?? opNiceName(call.spanName)}
+                      </span>
+                      <span className="flex-shrink-0">
+                        <Id
+                          id={call.callId}
+                          type="Call"
+                          className="ml-0 mr-4"
+                        />
+                      </span>
                       <VerticalBar />
                       <ModelRefLabel modelRef={call.traceCall?.inputs.model} />
                     </>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx
index adc80d044f6..5dcf835e378 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ComparisonDefinitionSection/EvaluationDefinition.tsx
@@ -1,8 +1,5 @@
 import {Box} from '@material-ui/core';
 import {Circle} from '@mui/icons-material';
-import {PopupDropdown} from '@wandb/weave/common/components/PopupDropdown';
-import {Button} from '@wandb/weave/components/Button';
-import {Pill} from '@wandb/weave/components/Tag';
 import React, {useMemo} from 'react';
 
 import {
@@ -17,87 +14,17 @@ import {SmallRef} from '../../../../../Browse2/SmallRef';
 import {CallLink, ObjectVersionLink} from '../../../common/Links';
 import {useWFHooks} from '../../../wfReactInterface/context';
 import {ObjectVersionKey} from '../../../wfReactInterface/wfDataModelHooksInterface';
-import {useCompareEvaluationsState} from '../../compareEvaluationsContext';
-import {
-  BOX_RADIUS,
-  CIRCLE_SIZE,
-  EVAL_DEF_HEIGHT,
-  STANDARD_BORDER,
-} from '../../ecpConstants';
+import {CIRCLE_SIZE} from '../../ecpConstants';
 import {EvaluationComparisonState} from '../../ecpState';
-import {HorizontalBox} from '../../Layout';
-
-export const EvaluationDefinition: React.FC<{
-  state: EvaluationComparisonState;
-  callId: string;
-}> = props => {
-  const {removeEvaluationCall, setBaselineEvaluationCallId} =
-    useCompareEvaluationsState();
-
-  const menuOptions = useMemo(() => {
-    return [
-      {
-        key: 'add-to-baseline',
-        content: 'Set as baseline',
-        onClick: () => {
-          setBaselineEvaluationCallId(props.callId);
-        },
-        disabled: props.callId === props.state.baselineEvaluationCallId,
-      },
-      {
-        key: 'remove',
-        content: 'Remove',
-        onClick: () => {
-          removeEvaluationCall(props.callId);
-        },
-        disabled: Object.keys(props.state.data.evaluationCalls).length === 1,
-      },
-    ];
-  }, [
-    props.callId,
-    props.state.baselineEvaluationCallId,
-    props.state.data.evaluationCalls,
-    removeEvaluationCall,
-    setBaselineEvaluationCallId,
-  ]);
-
-  return (
-    <HorizontalBox
-      sx={{
-        height: EVAL_DEF_HEIGHT,
-        borderRadius: BOX_RADIUS,
-        border: STANDARD_BORDER,
-        padding: '12px',
-        alignItems: 'center',
-        justifyContent: 'space-between',
-      }}>
-      <EvaluationCallLink {...props} />
-      {props.callId === props.state.baselineEvaluationCallId && (
-        <Pill label="Baseline" color="teal" />
-      )}
-      <div style={{marginLeft: '-14px'}}>
-        <PopupDropdown
-          sections={[menuOptions]}
-          trigger={
-            <Button
-              className="rotate-90"
-              icon="overflow-horizontal"
-              size="small"
-              variant="ghost"
-              style={{marginLeft: '4px'}}
-            />
-          }
-        />
-      </div>
-    </HorizontalBox>
-  );
-};
 
 export const EvaluationCallLink: React.FC<{
   callId: string;
   state: EvaluationComparisonState;
 }> = props => {
-  const evaluationCall = props.state.data.evaluationCalls[props.callId];
+  const evaluationCall = props.state.data.evaluationCalls?.[props.callId];
+  if (!evaluationCall) {
+    return null;
+  }
   const {entity, project} = props.state.data;
 
   return (
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/DraggableSection/DraggableItem.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/DraggableSection/DraggableItem.tsx
new file mode 100644
index 00000000000..1510c502b99
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/DraggableSection/DraggableItem.tsx
@@ -0,0 +1,103 @@
+import {Button} from '@wandb/weave/components/Button';
+import * as DropdownMenu from '@wandb/weave/components/DropdownMenu';
+import {Icon} from '@wandb/weave/components/Icon';
+import {Pill} from '@wandb/weave/components/Tag/Pill';
+import {Tailwind} from '@wandb/weave/components/Tailwind';
+import classNames from 'classnames';
+import React, {useState} from 'react';
+import {SortableElement, SortableHandle} from 'react-sortable-hoc';
+
+import {EvaluationComparisonState} from '../../ecpState';
+import {EvaluationCallLink} from '../ComparisonDefinitionSection/EvaluationDefinition';
+
+export type ItemDef = {
+  key: string;
+  value: string;
+  label?: string;
+};
+
+type DraggableItemProps = {
+  state: EvaluationComparisonState;
+  item: ItemDef;
+  numItems: number;
+  idx: number;
+  onRemoveItem: (value: string) => void;
+  onSetBaseline: (value: string | null) => void;
+};
+
+export const DraggableItem = SortableElement(
+  ({
+    state,
+    item,
+    numItems,
+    idx,
+    onRemoveItem,
+    onSetBaseline,
+  }: DraggableItemProps) => {
+    const isDeletable = numItems > 1;
+    const isBaseline = idx === 0;
+    const [isOpen, setIsOpen] = useState(false);
+
+    const onMakeBaselinePropagated = (e: React.MouseEvent) => {
+      e.stopPropagation();
+      onSetBaseline(item.value);
+    };
+
+    const onRemoveItemPropagated = (e: React.MouseEvent) => {
+      e.stopPropagation();
+      onRemoveItem(item.value);
+    };
+
+    return (
+      <Tailwind>
+        <div
+          className={classNames(
+            'flex select-none items-center gap-4 whitespace-nowrap rounded-[4px] border-[1px] border-moon-250 bg-white px-4 py-8 text-sm font-semibold hover:border-moon-350 hover:bg-moon-100'
+          )}>
+          <DragHandle />
+          <div className="flex items-center">
+            <EvaluationCallLink callId={item.value} state={state} />
+            {isBaseline && (
+              <Pill color="teal" label="Baseline" className="ml-8" />
+            )}
+          </div>
+          <DropdownMenu.Root open={isOpen} onOpenChange={setIsOpen}>
+            <DropdownMenu.Trigger>
+              <Button
+                variant="ghost"
+                icon="overflow-vertical"
+                active={isOpen}
+                size="small"
+              />
+            </DropdownMenu.Trigger>
+            <DropdownMenu.Portal>
+              <DropdownMenu.Content align="start">
+                <DropdownMenu.Item
+                  onClick={onMakeBaselinePropagated}
+                  disabled={isBaseline}>
+                  <Icon name="baseline-alt" />
+                  Make baseline
+                </DropdownMenu.Item>
+                {isDeletable && (
+                  <>
+                    <DropdownMenu.Separator />
+                    <DropdownMenu.Item onClick={onRemoveItemPropagated}>
+                      <Icon name="delete" />
+                      Remove from comparison
+                    </DropdownMenu.Item>
+                  </>
+                )}
+              </DropdownMenu.Content>
+            </DropdownMenu.Portal>
+          </DropdownMenu.Root>
+        </div>
+      </Tailwind>
+    );
+  }
+);
+
+const DragHandle = SortableHandle(() => (
+  <div className="cursor-grab">
+    <Icon name="drag-grip" />
+  </div>
+));
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/DraggableSection/DraggableSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/DraggableSection/DraggableSection.tsx
new file mode 100644
index 00000000000..23a03ceb5b3
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/DraggableSection/DraggableSection.tsx
@@ -0,0 +1,34 @@
+import React from 'react';
+import {SortableContainer} from 'react-sortable-hoc';
+
+import {EvaluationComparisonState} from '../../ecpState';
+import {DraggableItem} from './DraggableItem';
+import {ItemDef} from './DraggableItem';
+
+type DraggableSectionProps = {
+  state: EvaluationComparisonState;
+  items: ItemDef[];
+  onSetBaseline: (value: string | null) => void;
+  onRemoveItem: (value: string) => void;
+};
+
+export const DraggableSection = SortableContainer(
+  ({state, items, onSetBaseline, onRemoveItem}: DraggableSectionProps) => {
+    return (
+      <div className="flex flex-wrap items-center gap-16">
+        {items.map((item, index) => (
+          <DraggableItem
+            key={`item-${item.value}`}
+            numItems={items.length}
+            index={index}
+            idx={index}
+            item={item}
+            state={state}
+            onRemoveItem={onRemoveItem}
+            onSetBaseline={onSetBaseline}
+          />
+        ))}
+      </div>
+    );
+  }
+);
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ExampleCompareSection/ExampleCompareSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ExampleCompareSection/ExampleCompareSection.tsx
index 6041492b5c5..398f65ecd45 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ExampleCompareSection/ExampleCompareSection.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ExampleCompareSection/ExampleCompareSection.tsx
@@ -149,7 +149,7 @@ const stickySidebarHeaderMixin: React.CSSProperties = {
 
 /**
  * This component will occupy the entire space provided by the parent container.
- * It is intended to be used in teh CompareEvaluations page, as it depends on
+ * It is intended to be used in the CompareEvaluations page, as it depends on
  * the EvaluationComparisonState. However, in principle, it is a general purpose
  * model-output comparison tool. It allows the user to view inputs, then compare
  * model outputs and evaluation metrics across multiple trials.
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ExampleFilterSection/ExampleFilterSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ExampleFilterSection/ExampleFilterSection.tsx
index b2c8773a7d1..1146c8ea960 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ExampleFilterSection/ExampleFilterSection.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ExampleFilterSection/ExampleFilterSection.tsx
@@ -13,7 +13,7 @@ import {
 } from '../../compositeMetricsUtil';
 import {PLOT_HEIGHT, STANDARD_PADDING} from '../../ecpConstants';
 import {MAX_PLOT_DOT_SIZE, MIN_PLOT_DOT_SIZE} from '../../ecpConstants';
-import {EvaluationComparisonState} from '../../ecpState';
+import {EvaluationComparisonState, getBaselineCallId} from '../../ecpState';
 import {metricDefinitionId} from '../../ecpUtil';
 import {
   flattenedDimensionPath,
@@ -103,7 +103,7 @@ const SingleDimensionFilter: React.FC<{
   }, [props.state.data]);
 
   const {setComparisonDimensions} = useCompareEvaluationsState();
-  const baselineCallId = props.state.baselineEvaluationCallId;
+  const baselineCallId = getBaselineCallId(props.state);
   const compareCallId = Object.keys(props.state.data.evaluationCalls).find(
     callId => callId !== baselineCallId
   )!;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ScorecardSection/ScorecardSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ScorecardSection/ScorecardSection.tsx
index 4b319150fa8..303f640f47e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ScorecardSection/ScorecardSection.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/ScorecardSection/ScorecardSection.tsx
@@ -32,6 +32,7 @@ import {
 } from '../../ecpConstants';
 import {
   EvaluationComparisonState,
+  getBaselineCallId,
   getOrderedCallIds,
   getOrderedModelRefs,
 } from '../../ecpState';
@@ -414,7 +415,7 @@ export const ScorecardSection: React.FC<{
                     </GridCell>
                     {evalCallIds.map((evalCallId, mNdx) => {
                       const baseline = resolveSummaryMetricResult(
-                        props.state.baselineEvaluationCallId,
+                        getBaselineCallId(props.state),
                         groupName,
                         metricKey,
                         compositeSummaryMetrics,

From f967984c8bf49bf49e1e5a500b338381f2481e4e Mon Sep 17 00:00:00 2001
From: Erica Diaz <156136421+ericakdiaz@users.noreply.github.com>
Date: Tue, 10 Dec 2024 16:23:39 -0800
Subject: [PATCH 27/52] chore(weave): Update ToggleButtonGroup to allow
 disabling individual options (#3194)

---
 weave-js/src/components/ToggleButtonGroup.tsx | 67 +++++++++++--------
 1 file changed, 38 insertions(+), 29 deletions(-)

diff --git a/weave-js/src/components/ToggleButtonGroup.tsx b/weave-js/src/components/ToggleButtonGroup.tsx
index 65b2c538975..eca93e95657 100644
--- a/weave-js/src/components/ToggleButtonGroup.tsx
+++ b/weave-js/src/components/ToggleButtonGroup.tsx
@@ -9,6 +9,7 @@ import {Tailwind} from './Tailwind';
 export type ToggleOption = {
   value: string;
   icon?: IconName;
+  isDisabled?: boolean;
 };
 
 export type ToggleButtonGroupProps = {
@@ -37,7 +38,10 @@ export const ToggleButtonGroup = React.forwardRef<
   }
 
   const handleValueChange = (newValue: string) => {
-    if (newValue !== value) {
+    if (
+      newValue !== value &&
+      options.find(option => option.value === newValue)?.isDisabled !== true
+    ) {
       onValueChange(newValue);
     }
   };
@@ -49,34 +53,39 @@ export const ToggleButtonGroup = React.forwardRef<
         onValueChange={handleValueChange}
         className="flex gap-px"
         ref={ref}>
-        {options.map(({value: optionValue, icon}) => (
-          <ToggleGroup.Item
-            key={optionValue}
-            value={optionValue}
-            disabled={isDisabled || value === optionValue}>
-            <Button
-              icon={icon}
-              size={size}
-              className={twMerge(
-                size === 'small' &&
-                  (icon ? 'gap-4 px-4 py-3 text-sm' : 'px-8 py-3 text-sm'),
-                size === 'medium' &&
-                  (icon ? 'gap-5 px-7 py-4 text-base' : 'px-10 py-4 text-base'),
-                size === 'large' &&
-                  (icon
-                    ? 'gap-6 px-10 py-8 text-base'
-                    : 'px-12 py-8 text-base'),
-                isDisabled && 'pointer-events-none opacity-35',
-                value === optionValue
-                  ? 'bg-teal-300/[0.48] text-teal-600 hover:bg-teal-300/[0.48]'
-                  : 'hover:bg-oblivion/7 bg-oblivion/5 text-moon-600 hover:text-moon-800',
-                'first:rounded-l-sm', // First button rounded left
-                'last:rounded-r-sm' // Last button rounded right
-              )}>
-              {optionValue}
-            </Button>
-          </ToggleGroup.Item>
-        ))}
+        {options.map(
+          ({value: optionValue, icon, isDisabled: optionIsDisabled}) => (
+            <ToggleGroup.Item
+              key={optionValue}
+              value={optionValue}
+              disabled={isDisabled}
+              asChild>
+              <Button
+                icon={icon}
+                size={size}
+                className={twMerge(
+                  size === 'small' &&
+                    (icon ? 'gap-4 px-4 py-3 text-sm' : 'px-8 py-3 text-sm'),
+                  size === 'medium' &&
+                    (icon
+                      ? 'gap-5 px-7 py-4 text-base'
+                      : 'px-10 py-4 text-base'),
+                  size === 'large' &&
+                    (icon
+                      ? 'gap-6 px-10 py-8 text-base'
+                      : 'px-12 py-8 text-base'),
+                  (isDisabled || optionIsDisabled) && 'cursor-auto opacity-35',
+                  value === optionValue
+                    ? 'bg-teal-300/[0.48] text-teal-600 hover:bg-teal-300/[0.48]'
+                    : 'hover:bg-oblivion/7 bg-oblivion/5 text-moon-600 hover:text-moon-800',
+                  'first:rounded-l-sm', // First button rounded left
+                  'last:rounded-r-sm' // Last button rounded right
+                )}>
+                {optionValue}
+              </Button>
+            </ToggleGroup.Item>
+          )
+        )}
       </ToggleGroup.Root>
     </Tailwind>
   );

From 4e1e2c9a6da47ceff2fbd113053d959b43b4ce6f Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 10 Dec 2024 16:30:13 -0800
Subject: [PATCH 28/52] chore(weave): Fix predict detection heuristic for evals
 (#3196)

* init

* lint
---
 .../tsDataModelHooksEvaluationComparison.ts          | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooksEvaluationComparison.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooksEvaluationComparison.ts
index 8418f2976ad..847421464e3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooksEvaluationComparison.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooksEvaluationComparison.ts
@@ -69,7 +69,6 @@
  *  across different datasets.
  */
 
-import _ from 'lodash';
 import {sum} from 'lodash';
 import {useEffect, useMemo, useRef, useState} from 'react';
 
@@ -479,17 +478,8 @@ const fetchEvaluationComparisonData = async (
           const maybeDigest = parts[1];
           if (maybeDigest != null && !maybeDigest.includes('/')) {
             const rowDigest = maybeDigest;
-            const possiblePredictNames = [
-              'predict',
-              'infer',
-              'forward',
-              'invoke',
-            ];
             const isProbablyPredictCall =
-              (_.some(possiblePredictNames, name =>
-                traceCall.op_name.includes(`.${name}:`)
-              ) &&
-                modelRefs.includes(traceCall.inputs.self)) ||
+              modelRefs.includes(traceCall.inputs.self) ||
               modelRefs.includes(traceCall.op_name);
 
             const isProbablyScoreCall = scorerRefs.has(traceCall.op_name);

From bf39669bf3ebba977e9b6cd9224c9abce22a3a62 Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Tue, 10 Dec 2024 16:36:47 -0800
Subject: [PATCH 29/52] fix(ui): Evaluations -- normalize radar data, show real
 values for bar charts (#3199)

---
 .../SummaryPlotsSection.tsx                   | 94 ++++++++++---------
 1 file changed, 50 insertions(+), 44 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/SummaryPlotsSection.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/SummaryPlotsSection.tsx
index c23ffcce04d..02c456df850 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/SummaryPlotsSection.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/sections/SummaryPlotsSection/SummaryPlotsSection.tsx
@@ -29,7 +29,7 @@ export const SummaryPlots: React.FC<{
   state: EvaluationComparisonState;
   setSelectedMetrics: (newModel: Record<string, boolean>) => void;
 }> = ({state, setSelectedMetrics}) => {
-  const {radarData, allMetricNames} = useNormalizedPlotDataFromMetrics(state);
+  const {radarData, allMetricNames} = usePlotDataFromMetrics(state);
   const {selectedMetrics} = state;
 
   // Initialize selectedMetrics if null
@@ -237,11 +237,10 @@ const useFilteredData = (
     return data;
   }, [radarData, selectedMetrics]);
 
-function getMetricValuesFromRadarData(radarData: RadarPlotData): {
+function getMetricValuesMap(radarData: RadarPlotData): {
   [metric: string]: number[];
 } {
   const metricValues: {[metric: string]: number[]} = {};
-  // Gather all values for each metric
   Object.values(radarData).forEach(callData => {
     Object.entries(callData.metrics).forEach(([metric, value]) => {
       if (!metricValues[metric]) {
@@ -253,37 +252,54 @@ function getMetricValuesFromRadarData(radarData: RadarPlotData): {
   return metricValues;
 }
 
-function getMetricMinsFromRadarData(radarData: RadarPlotData): {
-  [metric: string]: number;
+function normalizeMetricValues(values: number[]): {
+  normalizedValues: number[];
+  normalizer: number;
 } {
-  const metricValues = getMetricValuesFromRadarData(radarData);
-  const metricMins: {[metric: string]: number} = {};
-  Object.entries(metricValues).forEach(([metric, values]) => {
-    metricMins[metric] = Math.min(...values);
-  });
-  return metricMins;
+  const min = Math.min(...values);
+  const max = Math.max(...values);
+
+  if (min === max) {
+    return {
+      normalizedValues: values.map(() => 0.5),
+      normalizer: 1,
+    };
+  }
+
+  // Handle negative values by shifting
+  const shiftedValues = min < 0 ? values.map(v => v - min) : values;
+  const maxValue = min < 0 ? max - min : max;
+
+  const maxPower = Math.ceil(Math.log2(maxValue));
+  const normalizer = Math.pow(2, maxPower);
+
+  return {
+    normalizedValues: shiftedValues.map(v => v / normalizer),
+    normalizer,
+  };
 }
 
-function normalizeDataForRadarPlot(radarData: RadarPlotData): RadarPlotData {
-  const metricMins = getMetricMinsFromRadarData(radarData);
+function normalizeDataForRadarPlot(
+  radarDataOriginal: RadarPlotData
+): RadarPlotData {
+  const radarData = Object.fromEntries(
+    Object.entries(radarDataOriginal).map(([callId, callData]) => [
+      callId,
+      {...callData, metrics: {...callData.metrics}},
+    ])
+  );
 
-  const normalizedData: RadarPlotData = {};
-  Object.entries(radarData).forEach(([callId, callData]) => {
-    normalizedData[callId] = {
-      name: callData.name,
-      color: callData.color,
-      metrics: {},
-    };
+  const metricValues = getMetricValuesMap(radarData);
 
-    Object.entries(callData.metrics).forEach(([metric, value]) => {
-      const min = metricMins[metric];
-      // Only shift values if there are negative values
-      const normalizedValue = min < 0 ? value - min : value;
-      normalizedData[callId].metrics[metric] = normalizedValue;
+  // Normalize each metric independently
+  Object.entries(metricValues).forEach(([metric, values]) => {
+    const {normalizedValues} = normalizeMetricValues(values);
+    Object.values(radarData).forEach((callData, index) => {
+      callData.metrics[metric] = normalizedValues[index];
     });
   });
 
-  return normalizedData;
+  return radarData;
 }
 
 const useBarPlotData = (filteredData: RadarPlotData) =>
@@ -317,7 +333,9 @@ const useBarPlotData = (filteredData: RadarPlotData) =>
         type: 'bar',
         y: metricBin.values,
         x: metricBin.callIds,
-        text: metricBin.values.map(value => value.toFixed(3)),
+        text: metricBin.values.map(value =>
+          Number.isInteger(value) ? value.toString() : value.toFixed(3)
+        ),
         textposition: 'outside',
         textfont: {size: 14, color: 'black'},
         name: metric,
@@ -408,16 +426,7 @@ const usePaginatedPlots = (
   return {plotsToShow, totalPlots, startIndex, endIndex, totalPages};
 };
 
-function normalizeValues(values: Array<number | undefined>): number[] {
-  // find the max value
-  // find the power of 2 that is greater than the max value
-  // divide all values by that power of 2
-  const maxVal = Math.max(...(values.filter(v => v !== undefined) as number[]));
-  const maxPower = Math.ceil(Math.log2(maxVal));
-  return values.map(val => (val ? val / 2 ** maxPower : 0));
-}
-
-const useNormalizedPlotDataFromMetrics = (
+const usePlotDataFromMetrics = (
   state: EvaluationComparisonState
 ): {radarData: RadarPlotData; allMetricNames: Set<string>} => {
   const compositeMetrics = useMemo(() => {
@@ -428,7 +437,7 @@ const useNormalizedPlotDataFromMetrics = (
   }, [state]);
 
   return useMemo(() => {
-    const normalizedMetrics = Object.values(compositeMetrics)
+    const metrics = Object.values(compositeMetrics)
       .map(scoreGroup => Object.values(scoreGroup.metrics))
       .flat()
       .map(metric => {
@@ -449,11 +458,8 @@ const useNormalizedPlotDataFromMetrics = (
             return val;
           }
         });
-        const normalizedValues = normalizeValues(values);
         const evalScores: {[evalCallId: string]: number | undefined} =
-          Object.fromEntries(
-            callIds.map((key, i) => [key, normalizedValues[i]])
-          );
+          Object.fromEntries(callIds.map((key, i) => [key, values[i]]));
 
         const metricLabel = flattenedDimensionPath(
           Object.values(metric.scorerRefs)[0].metric
@@ -472,7 +478,7 @@ const useNormalizedPlotDataFromMetrics = (
             name: evalCall.name,
             color: evalCall.color,
             metrics: Object.fromEntries(
-              normalizedMetrics.map(metric => {
+              metrics.map(metric => {
                 return [
                   metric.metricLabel,
                   metric.evalScores[evalCall.callId] ?? 0,
@@ -483,7 +489,7 @@ const useNormalizedPlotDataFromMetrics = (
         ];
       })
     );
-    const allMetricNames = new Set(normalizedMetrics.map(m => m.metricLabel));
+    const allMetricNames = new Set(metrics.map(m => m.metricLabel));
     return {radarData, allMetricNames};
   }, [callIds, compositeMetrics, state.data.evaluationCalls]);
 };

From af81503aa330c0659a8c65b316b1071782a543e6 Mon Sep 17 00:00:00 2001
From: Jeff Raubitschek <jeff@wandb.com>
Date: Wed, 11 Dec 2024 12:34:22 -0800
Subject: [PATCH 30/52] chore(weave): update db to use replicated tables
 (#3148)

---
 .../test_clickhouse_trace_server_migrator.py  | 230 ++++++++++++++++++
 .../clickhouse_trace_server_migrator.py       | 132 ++++++++--
 2 files changed, 336 insertions(+), 26 deletions(-)
 create mode 100644 tests/trace_server/test_clickhouse_trace_server_migrator.py

diff --git a/tests/trace_server/test_clickhouse_trace_server_migrator.py b/tests/trace_server/test_clickhouse_trace_server_migrator.py
new file mode 100644
index 00000000000..3a6a92f2479
--- /dev/null
+++ b/tests/trace_server/test_clickhouse_trace_server_migrator.py
@@ -0,0 +1,230 @@
+import types
+from unittest.mock import Mock, call, patch
+
+import pytest
+
+from weave.trace_server import clickhouse_trace_server_migrator as trace_server_migrator
+from weave.trace_server.clickhouse_trace_server_migrator import MigrationError
+
+
+@pytest.fixture
+def mock_costs():
+    with patch(
+        "weave.trace_server.costs.insert_costs.should_insert_costs", return_value=False
+    ) as mock_should_insert:
+        with patch(
+            "weave.trace_server.costs.insert_costs.get_current_costs", return_value=[]
+        ) as mock_get_costs:
+            yield
+
+
+@pytest.fixture
+def migrator():
+    ch_client = Mock()
+    migrator = trace_server_migrator.ClickHouseTraceServerMigrator(ch_client)
+    migrator._get_migration_status = Mock()
+    migrator._get_migrations = Mock()
+    migrator._determine_migrations_to_apply = Mock()
+    migrator._update_migration_status = Mock()
+    ch_client.command.reset_mock()
+    return migrator
+
+
+def test_apply_migrations_with_target_version(mock_costs, migrator, tmp_path):
+    # Setup
+    migrator._get_migration_status.return_value = {
+        "curr_version": 1,
+        "partially_applied_version": None,
+    }
+    migrator._get_migrations.return_value = {
+        "1": {"up": "1.up.sql", "down": "1.down.sql"},
+        "2": {"up": "2.up.sql", "down": "2.down.sql"},
+    }
+    migrator._determine_migrations_to_apply.return_value = [(2, "2.up.sql")]
+
+    # Create a temporary migration file
+    migration_dir = tmp_path / "migrations"
+    migration_dir.mkdir()
+    migration_file = migration_dir / "2.up.sql"
+    migration_file.write_text(
+        "CREATE TABLE test1 (id Int32);\nCREATE TABLE test2 (id Int32);"
+    )
+
+    # Mock the migration directory path
+    with patch("os.path.dirname") as mock_dirname:
+        mock_dirname.return_value = str(tmp_path)
+
+        # Execute
+        migrator.apply_migrations("test_db", target_version=2)
+
+    # Verify
+    migrator._get_migration_status.assert_called_once_with("test_db")
+    migrator._get_migrations.assert_called_once()
+    migrator._determine_migrations_to_apply.assert_called_once_with(
+        1, migrator._get_migrations.return_value, 2
+    )
+
+    # Verify migration execution
+    assert migrator._update_migration_status.call_count == 2
+    migrator._update_migration_status.assert_has_calls(
+        [call("test_db", 2, is_start=True), call("test_db", 2, is_start=False)]
+    )
+
+    # Verify the actual SQL commands were executed
+    ch_client = migrator.ch_client
+    assert ch_client.command.call_count == 2
+    ch_client.command.assert_has_calls(
+        [call("CREATE TABLE test1 (id Int32)"), call("CREATE TABLE test2 (id Int32)")]
+    )
+
+
+def test_execute_migration_command(mock_costs, migrator):
+    # Setup
+    ch_client = migrator.ch_client
+    ch_client.database = "original_db"
+
+    # Execute
+    migrator._execute_migration_command("test_db", "CREATE TABLE test (id Int32)")
+
+    # Verify
+    assert ch_client.database == "original_db"  # Should restore original database
+    ch_client.command.assert_called_once_with("CREATE TABLE test (id Int32)")
+
+
+def test_migration_replicated(mock_costs, migrator):
+    ch_client = migrator.ch_client
+    orig = "CREATE TABLE test (id String, project_id String) ENGINE = MergeTree ORDER BY (project_id, id);"
+    migrator._execute_migration_command("test_db", orig)
+    ch_client.command.assert_called_once_with(orig)
+
+
+def test_update_migration_status(mock_costs, migrator):
+    # Don't mock _update_migration_status for this test
+    migrator._update_migration_status = types.MethodType(
+        trace_server_migrator.ClickHouseTraceServerMigrator._update_migration_status,
+        migrator,
+    )
+
+    # Test start of migration
+    migrator._update_migration_status("test_db", 2, is_start=True)
+    migrator.ch_client.command.assert_called_with(
+        "ALTER TABLE db_management.migrations UPDATE partially_applied_version = 2 WHERE db_name = 'test_db'"
+    )
+
+    # Test end of migration
+    migrator._update_migration_status("test_db", 2, is_start=False)
+    migrator.ch_client.command.assert_called_with(
+        "ALTER TABLE db_management.migrations UPDATE curr_version = 2, partially_applied_version = NULL WHERE db_name = 'test_db'"
+    )
+
+
+def test_is_safe_identifier(mock_costs, migrator):
+    # Valid identifiers
+    assert migrator._is_safe_identifier("test_db")
+    assert migrator._is_safe_identifier("my_db123")
+    assert migrator._is_safe_identifier("db.table")
+
+    # Invalid identifiers
+    assert not migrator._is_safe_identifier("test-db")
+    assert not migrator._is_safe_identifier("db;")
+    assert not migrator._is_safe_identifier("db'name")
+    assert not migrator._is_safe_identifier("db/*")
+
+
+def test_create_db_sql_validation(mock_costs, migrator):
+    # Test invalid database name
+    with pytest.raises(MigrationError, match="Invalid database name"):
+        migrator._create_db_sql("test;db")
+
+    # Test replicated mode with invalid values
+    migrator.replicated = True
+    migrator.replicated_cluster = "test;cluster"
+    with pytest.raises(MigrationError, match="Invalid cluster name"):
+        migrator._create_db_sql("test_db")
+
+    migrator.replicated_cluster = "test_cluster"
+    migrator.replicated_path = "/clickhouse/bad;path/{db}"
+    with pytest.raises(MigrationError, match="Invalid replicated path"):
+        migrator._create_db_sql("test_db")
+
+
+def test_create_db_sql_non_replicated(mock_costs, migrator):
+    # Test non-replicated mode
+    migrator.replicated = False
+    sql = migrator._create_db_sql("test_db")
+    assert sql.strip() == "CREATE DATABASE IF NOT EXISTS test_db"
+
+
+def test_create_db_sql_replicated(mock_costs, migrator):
+    # Test replicated mode
+    migrator.replicated = True
+    migrator.replicated_path = "/clickhouse/tables/{db}"
+    migrator.replicated_cluster = "test_cluster"
+
+    sql = migrator._create_db_sql("test_db")
+    expected = """
+        CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER test_cluster ENGINE=Replicated('/clickhouse/tables/test_db', '{shard}', '{replica}')
+    """.strip()
+    assert sql.strip() == expected
+
+
+def test_format_replicated_sql_non_replicated(mock_costs, migrator):
+    # Test that SQL is unchanged when replicated=False
+    migrator.replicated = False
+    test_cases = [
+        "CREATE TABLE test (id Int32) ENGINE = MergeTree",
+        "CREATE TABLE test (id Int32) ENGINE = SummingMergeTree",
+        "CREATE TABLE test (id Int32) ENGINE=ReplacingMergeTree",
+    ]
+
+    for sql in test_cases:
+        assert migrator._format_replicated_sql(sql) == sql
+
+
+def test_format_replicated_sql_replicated(mock_costs, migrator):
+    # Test that MergeTree engines are converted to Replicated variants
+    migrator.replicated = True
+
+    test_cases = [
+        (
+            "CREATE TABLE test (id Int32) ENGINE = MergeTree",
+            "CREATE TABLE test (id Int32) ENGINE = ReplicatedMergeTree",
+        ),
+        (
+            "CREATE TABLE test (id Int32) ENGINE = SummingMergeTree",
+            "CREATE TABLE test (id Int32) ENGINE = ReplicatedSummingMergeTree",
+        ),
+        (
+            "CREATE TABLE test (id Int32) ENGINE=ReplacingMergeTree",
+            "CREATE TABLE test (id Int32) ENGINE = ReplicatedReplacingMergeTree",
+        ),
+        # Test with extra whitespace
+        (
+            "CREATE TABLE test (id Int32) ENGINE  =   MergeTree",
+            "CREATE TABLE test (id Int32) ENGINE = ReplicatedMergeTree",
+        ),
+        # Test with parameters
+        (
+            "CREATE TABLE test (id Int32) ENGINE = MergeTree()",
+            "CREATE TABLE test (id Int32) ENGINE = ReplicatedMergeTree()",
+        ),
+    ]
+
+    for input_sql, expected_sql in test_cases:
+        assert migrator._format_replicated_sql(input_sql) == expected_sql
+
+
+def test_format_replicated_sql_non_mergetree(mock_costs, migrator):
+    # Test that non-MergeTree engines are left unchanged
+    migrator.replicated = True
+
+    test_cases = [
+        "CREATE TABLE test (id Int32) ENGINE = Memory",
+        "CREATE TABLE test (id Int32) ENGINE = Log",
+        "CREATE TABLE test (id Int32) ENGINE = TinyLog",
+        # This should not be changed as it's not a complete word match
+        "CREATE TABLE test (id Int32) ENGINE = MyMergeTreeCustom",
+    ]
+
+    for sql in test_cases:
+        assert migrator._format_replicated_sql(sql) == sql
diff --git a/weave/trace_server/clickhouse_trace_server_migrator.py b/weave/trace_server/clickhouse_trace_server_migrator.py
index 30dffe89365..4336630bf50 100644
--- a/weave/trace_server/clickhouse_trace_server_migrator.py
+++ b/weave/trace_server/clickhouse_trace_server_migrator.py
@@ -1,6 +1,7 @@
 # Clickhouse Trace Server Manager
 import logging
 import os
+import re
 from typing import Optional
 
 from clickhouse_connect.driver.client import Client as CHClient
@@ -9,6 +10,11 @@
 
 logger = logging.getLogger(__name__)
 
+# These settings are only used when `replicated` mode is enabled for
+# self managed clickhouse instances.
+DEFAULT_REPLICATED_PATH = "/clickhouse/tables/{db}"
+DEFAULT_REPLICATED_CLUSTER = "weave_cluster"
+
 
 class MigrationError(RuntimeError):
     """Raised when a migration error occurs."""
@@ -16,15 +22,77 @@ class MigrationError(RuntimeError):
 
 class ClickHouseTraceServerMigrator:
     ch_client: CHClient
+    replicated: bool
+    replicated_path: str
+    replicated_cluster: str
 
     def __init__(
         self,
         ch_client: CHClient,
+        replicated: Optional[bool] = None,
+        replicated_path: Optional[str] = None,
+        replicated_cluster: Optional[str] = None,
     ):
         super().__init__()
         self.ch_client = ch_client
+        self.replicated = False if replicated is None else replicated
+        self.replicated_path = (
+            DEFAULT_REPLICATED_PATH if replicated_path is None else replicated_path
+        )
+        self.replicated_cluster = (
+            DEFAULT_REPLICATED_CLUSTER
+            if replicated_cluster is None
+            else replicated_cluster
+        )
         self._initialize_migration_db()
 
+    def _is_safe_identifier(self, value: str) -> bool:
+        """Check if a string is safe to use as an identifier in SQL."""
+        return bool(re.match(r"^[a-zA-Z0-9_\.]+$", value))
+
+    def _format_replicated_sql(self, sql_query: str) -> str:
+        """Format SQL query to use replicated engines if replicated mode is enabled."""
+        if not self.replicated:
+            return sql_query
+
+        # Match "ENGINE = <optional words>MergeTree" followed by word boundary
+        pattern = r"ENGINE\s*=\s*(\w+)?MergeTree\b"
+
+        def replace_engine(match: re.Match[str]) -> str:
+            engine_prefix = match.group(1) or ""
+            return f"ENGINE = Replicated{engine_prefix}MergeTree"
+
+        return re.sub(pattern, replace_engine, sql_query, flags=re.IGNORECASE)
+
+    def _create_db_sql(self, db_name: str) -> str:
+        """Geneate SQL database create string for normal and replicated databases."""
+        if not self._is_safe_identifier(db_name):
+            raise MigrationError(f"Invalid database name: {db_name}")
+
+        replicated_engine = ""
+        replicated_cluster = ""
+        if self.replicated:
+            if not self._is_safe_identifier(self.replicated_cluster):
+                raise MigrationError(f"Invalid cluster name: {self.replicated_cluster}")
+
+            replicated_path = self.replicated_path.replace("{db}", db_name)
+            if not all(
+                self._is_safe_identifier(part)
+                for part in replicated_path.split("/")
+                if part
+            ):
+                raise MigrationError(f"Invalid replicated path: {replicated_path}")
+
+            replicated_cluster = f" ON CLUSTER {self.replicated_cluster}"
+            replicated_engine = (
+                f" ENGINE=Replicated('{replicated_path}', '{{shard}}', '{{replica}}')"
+            )
+
+        create_db_sql = f"""
+            CREATE DATABASE IF NOT EXISTS {db_name}{replicated_cluster}{replicated_engine}
+        """
+        return create_db_sql
+
     def apply_migrations(
         self, target_db: str, target_version: Optional[int] = None
     ) -> None:
@@ -46,20 +114,15 @@ def apply_migrations(
             return
         logger.info(f"Migrations to apply: {migrations_to_apply}")
         if status["curr_version"] == 0:
-            self.ch_client.command(f"CREATE DATABASE IF NOT EXISTS {target_db}")
+            self.ch_client.command(self._create_db_sql(target_db))
         for target_version, migration_file in migrations_to_apply:
             self._apply_migration(target_db, target_version, migration_file)
         if should_insert_costs(status["curr_version"], target_version):
             insert_costs(self.ch_client, target_db)
 
     def _initialize_migration_db(self) -> None:
-        self.ch_client.command(
-            """
-            CREATE DATABASE IF NOT EXISTS db_management
-        """
-        )
-        self.ch_client.command(
-            """
+        self.ch_client.command(self._create_db_sql("db_management"))
+        create_table_sql = """
             CREATE TABLE IF NOT EXISTS db_management.migrations
             (
                 db_name String,
@@ -69,7 +132,7 @@ def _initialize_migration_db(self) -> None:
             ENGINE = MergeTree()
             ORDER BY (db_name)
         """
-        )
+        self.ch_client.command(self._format_replicated_sql(create_table_sql))
 
     def _get_migration_status(self, db_name: str) -> dict:
         column_names = ["db_name", "curr_version", "partially_applied_version"]
@@ -184,31 +247,48 @@ def _determine_migrations_to_apply(
 
         return []
 
+    def _execute_migration_command(self, target_db: str, command: str) -> None:
+        """Execute a single migration command in the context of the target database."""
+        command = command.strip()
+        if len(command) == 0:
+            return
+        curr_db = self.ch_client.database
+        self.ch_client.database = target_db
+        self.ch_client.command(self._format_replicated_sql(command))
+        self.ch_client.database = curr_db
+
+    def _update_migration_status(
+        self, target_db: str, target_version: int, is_start: bool = True
+    ) -> None:
+        """Update the migration status in db_management.migrations table."""
+        if is_start:
+            self.ch_client.command(
+                f"ALTER TABLE db_management.migrations UPDATE partially_applied_version = {target_version} WHERE db_name = '{target_db}'"
+            )
+        else:
+            self.ch_client.command(
+                f"ALTER TABLE db_management.migrations UPDATE curr_version = {target_version}, partially_applied_version = NULL WHERE db_name = '{target_db}'"
+            )
+
     def _apply_migration(
         self, target_db: str, target_version: int, migration_file: str
     ) -> None:
         logger.info(f"Applying migration {migration_file} to `{target_db}`")
         migration_dir = os.path.join(os.path.dirname(__file__), "migrations")
         migration_file_path = os.path.join(migration_dir, migration_file)
+
         with open(migration_file_path) as f:
             migration_sql = f.read()
-        self.ch_client.command(
-            f"""
-            ALTER TABLE db_management.migrations UPDATE partially_applied_version = {target_version} WHERE db_name = '{target_db}'
-        """
-        )
+
+        # Mark migration as partially applied
+        self._update_migration_status(target_db, target_version, is_start=True)
+
+        # Execute each command in the migration
         migration_sub_commands = migration_sql.split(";")
         for command in migration_sub_commands:
-            command = command.strip()
-            if len(command) == 0:
-                continue
-            curr_db = self.ch_client.database
-            self.ch_client.database = target_db
-            self.ch_client.command(command)
-            self.ch_client.database = curr_db
-        self.ch_client.command(
-            f"""
-            ALTER TABLE db_management.migrations UPDATE curr_version = {target_version}, partially_applied_version = NULL WHERE db_name = '{target_db}'
-        """
-        )
+            self._execute_migration_command(target_db, command)
+
+        # Mark migration as fully applied
+        self._update_migration_status(target_db, target_version, is_start=False)
+
         logger.info(f"Migration {migration_file} applied to `{target_db}`")

From 4c6db183be9b356dfc6781670c5b8f5fb6fd7532 Mon Sep 17 00:00:00 2001
From: Josiah Lee <josiahwlee@gmail.com>
Date: Wed, 11 Dec 2024 12:54:59 -0800
Subject: [PATCH 31/52] add choices drawer (#3203)

---
 .../Browse3/pages/ChatView/ChoiceView.tsx     |   8 +-
 .../Browse3/pages/ChatView/ChoicesDrawer.tsx  | 104 ++++++++++++++++++
 .../Browse3/pages/ChatView/ChoicesView.tsx    |  39 ++++---
 .../pages/ChatView/ChoicesViewCarousel.tsx    |  27 +++--
 .../pages/ChatView/ChoicesViewLinear.tsx      |  73 ------------
 .../Home/Browse3/pages/ChatView/types.ts      |   2 -
 6 files changed, 148 insertions(+), 105 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesDrawer.tsx
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewLinear.tsx

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx
index c8143f9549e..d1a2c59d5d0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx
@@ -6,15 +6,21 @@ import {Choice} from './types';
 type ChoiceViewProps = {
   choice: Choice;
   isStructuredOutput?: boolean;
+  isNested?: boolean;
 };
 
-export const ChoiceView = ({choice, isStructuredOutput}: ChoiceViewProps) => {
+export const ChoiceView = ({
+  choice,
+  isStructuredOutput,
+  isNested,
+}: ChoiceViewProps) => {
   const {message} = choice;
   return (
     <MessagePanel
       index={choice.index}
       message={message}
       isStructuredOutput={isStructuredOutput}
+      isNested={isNested}
       isChoice
     />
   );
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesDrawer.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesDrawer.tsx
new file mode 100644
index 00000000000..16d6897c27b
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesDrawer.tsx
@@ -0,0 +1,104 @@
+import {Box, Drawer} from '@mui/material';
+import {MOON_200} from '@wandb/weave/common/css/color.styles';
+import {Tag} from '@wandb/weave/components/Tag';
+import {Tailwind} from '@wandb/weave/components/Tailwind';
+import React from 'react';
+
+import {Button} from '../../../../../Button';
+import {ChoiceView} from './ChoiceView';
+import {Choice} from './types';
+
+type ChoicesDrawerProps = {
+  choices: Choice[];
+  isStructuredOutput?: boolean;
+  isDrawerOpen: boolean;
+  setIsDrawerOpen: React.Dispatch<React.SetStateAction<boolean>>;
+  selectedChoiceIndex: number;
+  setSelectedChoiceIndex: (choiceIndex: number) => void;
+};
+
+export const ChoicesDrawer = ({
+  choices,
+  isStructuredOutput,
+  isDrawerOpen,
+  setIsDrawerOpen,
+  selectedChoiceIndex,
+  setSelectedChoiceIndex,
+}: ChoicesDrawerProps) => {
+  return (
+    <Drawer
+      open={isDrawerOpen}
+      onClose={() => setIsDrawerOpen(false)}
+      title="Choices"
+      anchor="right"
+      sx={{
+        '& .MuiDrawer-paper': {mt: '60px', width: '400px'},
+      }}>
+      <Box
+        sx={{
+          position: 'sticky',
+          top: 0,
+          zIndex: 1,
+          px: 2,
+          height: 44,
+          width: '100%',
+          borderBottom: `1px solid ${MOON_200}`,
+          display: 'flex',
+          flexDirection: 'row',
+          alignItems: 'center',
+          justifyContent: 'space-between',
+        }}>
+        <Box
+          sx={{
+            height: 44,
+            display: 'flex',
+            alignItems: 'center',
+            fontWeight: 600,
+            fontSize: '1.25rem',
+          }}>
+          Responses
+        </Box>
+        <Button
+          size="medium"
+          variant="ghost"
+          icon="close"
+          onClick={() => setIsDrawerOpen(false)}
+          tooltip="Close"
+        />
+      </Box>
+      <Tailwind>
+        <div className="flex flex-col p-12">
+          {choices.map((c, index) => (
+            <div key={c.index}>
+              <div className="flex items-center gap-4 font-semibold">
+                <Tag color="moon" label={`Response ${index + 1}`} />
+                {index === selectedChoiceIndex ? (
+                  <Button
+                    className="text-green-500"
+                    size="small"
+                    variant="ghost"
+                    icon="checkmark">
+                    <span className="text-moon-500">Response selected</span>
+                  </Button>
+                ) : (
+                  <Button
+                    size="small"
+                    variant="ghost"
+                    icon="boolean"
+                    onClick={() => setSelectedChoiceIndex(index)}>
+                    <span className="text-moon-500">Select response</span>
+                  </Button>
+                )}
+              </div>
+              <ChoiceView
+                choice={c}
+                isStructuredOutput={isStructuredOutput}
+                isNested
+              />
+            </div>
+          ))}
+        </div>
+      </Tailwind>
+    </Drawer>
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx
index c22df7c63d7..5ddc7f12202 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx
@@ -1,9 +1,9 @@
 import React, {useState} from 'react';
 
+import {ChoicesDrawer} from './ChoicesDrawer';
 import {ChoicesViewCarousel} from './ChoicesViewCarousel';
-import {ChoicesViewLinear} from './ChoicesViewLinear';
 import {ChoiceView} from './ChoiceView';
-import {Choice, ChoicesMode} from './types';
+import {Choice} from './types';
 
 type ChoicesViewProps = {
   choices: Choice[];
@@ -14,7 +14,12 @@ export const ChoicesView = ({
   choices,
   isStructuredOutput,
 }: ChoicesViewProps) => {
-  const [mode, setMode] = useState<ChoicesMode>('linear');
+  const [isDrawerOpen, setIsDrawerOpen] = useState(false);
+  const [localSelectedChoiceIndex, setLocalSelectedChoiceIndex] = useState(0);
+
+  const handleSetSelectedChoiceIndex = (choiceIndex: number) => {
+    setLocalSelectedChoiceIndex(choiceIndex);
+  };
 
   if (choices.length === 0) {
     return null;
@@ -26,20 +31,20 @@ export const ChoicesView = ({
   }
   return (
     <>
-      {mode === 'linear' && (
-        <ChoicesViewLinear
-          choices={choices}
-          isStructuredOutput={isStructuredOutput}
-          setMode={setMode}
-        />
-      )}
-      {mode === 'carousel' && (
-        <ChoicesViewCarousel
-          choices={choices}
-          isStructuredOutput={isStructuredOutput}
-          setMode={setMode}
-        />
-      )}
+      <ChoicesViewCarousel
+        choices={choices}
+        isStructuredOutput={isStructuredOutput}
+        selectedChoiceIndex={localSelectedChoiceIndex}
+        setSelectedChoiceIndex={handleSetSelectedChoiceIndex}
+        setIsDrawerOpen={setIsDrawerOpen}
+      />
+      <ChoicesDrawer
+        choices={choices}
+        isDrawerOpen={isDrawerOpen}
+        setIsDrawerOpen={setIsDrawerOpen}
+        selectedChoiceIndex={localSelectedChoiceIndex}
+        setSelectedChoiceIndex={handleSetSelectedChoiceIndex}
+      />
     </>
   );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx
index f4a52fc6801..a34932dea17 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx
@@ -1,34 +1,37 @@
-import React, {useState} from 'react';
+import React from 'react';
 
 import {Button} from '../../../../../Button';
 import {ChoiceView} from './ChoiceView';
-import {Choice, ChoicesMode} from './types';
+import {Choice} from './types';
 
 type ChoicesViewCarouselProps = {
   choices: Choice[];
   isStructuredOutput?: boolean;
-  setMode: React.Dispatch<React.SetStateAction<ChoicesMode>>;
+  setIsDrawerOpen: React.Dispatch<React.SetStateAction<boolean>>;
+  selectedChoiceIndex: number;
+  setSelectedChoiceIndex: (choiceIndex: number) => void;
 };
 
 export const ChoicesViewCarousel = ({
   choices,
   isStructuredOutput,
-  setMode,
+  setIsDrawerOpen,
+  selectedChoiceIndex,
+  setSelectedChoiceIndex,
 }: ChoicesViewCarouselProps) => {
-  const [step, setStep] = useState(0);
-
   const onNext = () => {
-    setStep((step + 1) % choices.length);
+    setSelectedChoiceIndex((selectedChoiceIndex + 1) % choices.length);
   };
   const onBack = () => {
-    const newStep = step === 0 ? choices.length - 1 : step - 1;
-    setStep(newStep);
+    const newStep =
+      selectedChoiceIndex === 0 ? choices.length - 1 : selectedChoiceIndex - 1;
+    setSelectedChoiceIndex(newStep);
   };
 
   return (
     <>
       <ChoiceView
-        choice={choices[step]}
+        choice={choices[selectedChoiceIndex]}
         isStructuredOutput={isStructuredOutput}
       />
       <div className="flex items-center">
@@ -37,7 +40,7 @@ export const ChoicesViewCarousel = ({
             size="small"
             variant="quiet"
             icon="expand-uncollapse"
-            onClick={() => setMode('linear')}
+            onClick={() => setIsDrawerOpen(true)}
             tooltip="Switch to linear view"
           />
         </div>
@@ -48,7 +51,7 @@ export const ChoicesViewCarousel = ({
             size="small"
             onClick={onBack}
           />
-          {step + 1} of {choices.length}
+          {selectedChoiceIndex + 1} of {choices.length}
           <Button
             variant="ghost"
             icon="chevron-next"
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewLinear.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewLinear.tsx
deleted file mode 100644
index 92668c6504e..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewLinear.tsx
+++ /dev/null
@@ -1,73 +0,0 @@
-import React from 'react';
-
-import {Button} from '../../../../../Button';
-import {ChoiceView} from './ChoiceView';
-import {Choice, ChoicesMode} from './types';
-
-type ChoicesViewLinearProps = {
-  choices: Choice[];
-  isStructuredOutput?: boolean;
-  setMode: React.Dispatch<React.SetStateAction<ChoicesMode>>;
-};
-
-export const ChoicesViewLinear = ({
-  choices,
-  isStructuredOutput,
-  setMode,
-}: ChoicesViewLinearProps) => {
-  return (
-    <div className="flex flex-col gap-36">
-      {choices.map(c => (
-        <div>
-          <div className="mt-8 flex items-center">
-            <div
-              id={`choice-${c.index}`}
-              className="flex flex-auto items-center gap-4 text-xs">
-              {c.index !== 0 && (
-                <Button
-                  size="small"
-                  variant="quiet"
-                  icon="chevron-back"
-                  onClick={() => {
-                    const prevChoice = document.getElementById(
-                      `choice-${c.index - 1}`
-                    );
-                    prevChoice?.scrollIntoView({behavior: 'smooth'});
-                  }}
-                />
-              )}
-              <span>
-                Choice {c.index + 1} of {choices.length}
-              </span>
-              {c.index !== choices.length - 1 && (
-                <Button
-                  size="small"
-                  variant="quiet"
-                  icon="chevron-next"
-                  onClick={() => {
-                    const nextChoice = document.getElementById(
-                      `choice-${c.index + 1}`
-                    );
-                    nextChoice?.scrollIntoView({behavior: 'smooth'});
-                  }}
-                />
-              )}
-            </div>
-            <Button
-              size="small"
-              variant="quiet"
-              icon="collapse"
-              onClick={() => setMode('carousel')}
-              tooltip="Switch to carousel view"
-            />
-          </div>
-          <ChoiceView
-            key={c.index}
-            choice={c}
-            isStructuredOutput={isStructuredOutput}
-          />
-        </div>
-      ))}
-    </div>
-  );
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/types.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/types.ts
index b5696055712..3bbe65a5baf 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/types.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/types.ts
@@ -113,5 +113,3 @@ export type Chat = {
   request: ChatRequest | null;
   result: ChatCompletion | null;
 };
-
-export type ChoicesMode = 'linear' | 'carousel';

From 0b2d99c6afb9a0c3c7e95e86ae03b7faa43d8acd Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 11 Dec 2024 12:56:01 -0800
Subject: [PATCH 32/52] chore(weave): hide admin only scorers tabs (#3207)

* re-hide

* init

* init
---
 .../Home/Browse3/pages/CallPage/CallPage.tsx               | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx
index 484b038c193..a3315266f65 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx
@@ -1,5 +1,4 @@
 import Box from '@mui/material/Box';
-import {useViewerInfo} from '@wandb/weave/common/hooks/useViewerInfo';
 import {Loading} from '@wandb/weave/components/Loading';
 import {urlPrefixed} from '@wandb/weave/config';
 import {useViewTraceEvent} from '@wandb/weave/integrations/analytics/useViewEvents';
@@ -71,8 +70,10 @@ export const CallPage: FC<{
 };
 
 export const useShowRunnableUI = () => {
-  const viewerInfo = useViewerInfo();
-  return viewerInfo.loading ? false : viewerInfo.userInfo?.admin;
+  return false;
+  // Uncomment to re-enable
+  // const viewerInfo = useViewerInfo();
+  // return viewerInfo.loading ? false : viewerInfo.userInfo?.admin;
 };
 
 const useCallTabs = (call: CallSchema) => {

From 3e43f5f7e5eace87e857889c1dce814983febb42 Mon Sep 17 00:00:00 2001
From: Josiah Lee <josiahwlee@gmail.com>
Date: Wed, 11 Dec 2024 13:14:41 -0800
Subject: [PATCH 33/52] chore(weave): fix choices in weave (#3204)

* add choices drawer

* fix choices in playground
---
 .../Browse3/pages/ChatView/ChoiceView.tsx     |  4 ++-
 .../Browse3/pages/ChatView/ChoicesDrawer.tsx  |  1 +
 .../Browse3/pages/ChatView/ChoicesView.tsx    |  5 ++++
 .../pages/ChatView/ChoicesViewCarousel.tsx    |  1 +
 .../Browse3/pages/ChatView/MessagePanel.tsx   |  8 +++---
 .../PlaygroundMessagePanelButtons.tsx         | 10 +++----
 .../ChatView/PlaygroundMessagePanelEditor.tsx | 10 +++----
 .../PlaygroundChat/PlaygroundChat.tsx         | 10 +++++--
 .../useChatCompletionFunctions.tsx            | 28 ++++++++++++-------
 .../PlaygroundChat/useChatFunctions.tsx       | 13 ++-------
 .../PlaygroundPage/PlaygroundContext.tsx      |  7 +++--
 .../Browse3/pages/PlaygroundPage/types.ts     |  1 +
 .../PlaygroundPage/usePlaygroundState.ts      |  1 +
 13 files changed, 60 insertions(+), 39 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx
index d1a2c59d5d0..9e24e7e0a4a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx
@@ -7,12 +7,14 @@ type ChoiceViewProps = {
   choice: Choice;
   isStructuredOutput?: boolean;
   isNested?: boolean;
+  choiceIndex?: number;
 };
 
 export const ChoiceView = ({
   choice,
   isStructuredOutput,
   isNested,
+  choiceIndex,
 }: ChoiceViewProps) => {
   const {message} = choice;
   return (
@@ -21,7 +23,7 @@ export const ChoiceView = ({
       message={message}
       isStructuredOutput={isStructuredOutput}
       isNested={isNested}
-      isChoice
+      choiceIndex={choiceIndex}
     />
   );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesDrawer.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesDrawer.tsx
index 16d6897c27b..1e7571cea53 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesDrawer.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesDrawer.tsx
@@ -93,6 +93,7 @@ export const ChoicesDrawer = ({
               <ChoiceView
                 choice={c}
                 isStructuredOutput={isStructuredOutput}
+                choiceIndex={index}
                 isNested
               />
             </div>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx
index 5ddc7f12202..138ca10c7e8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx
@@ -1,5 +1,6 @@
 import React, {useState} from 'react';
 
+import {usePlaygroundContext} from '../PlaygroundPage/PlaygroundContext';
 import {ChoicesDrawer} from './ChoicesDrawer';
 import {ChoicesViewCarousel} from './ChoicesViewCarousel';
 import {ChoiceView} from './ChoiceView';
@@ -14,11 +15,15 @@ export const ChoicesView = ({
   choices,
   isStructuredOutput,
 }: ChoicesViewProps) => {
+  const {setSelectedChoiceIndex: setGlobalSelectedChoiceIndex} =
+    usePlaygroundContext();
+
   const [isDrawerOpen, setIsDrawerOpen] = useState(false);
   const [localSelectedChoiceIndex, setLocalSelectedChoiceIndex] = useState(0);
 
   const handleSetSelectedChoiceIndex = (choiceIndex: number) => {
     setLocalSelectedChoiceIndex(choiceIndex);
+    setGlobalSelectedChoiceIndex(choiceIndex);
   };
 
   if (choices.length === 0) {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx
index a34932dea17..18760817665 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx
@@ -33,6 +33,7 @@ export const ChoicesViewCarousel = ({
       <ChoiceView
         choice={choices[selectedChoiceIndex]}
         isStructuredOutput={isStructuredOutput}
+        choiceIndex={selectedChoiceIndex}
       />
       <div className="flex items-center">
         <div className="flex-auto">
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
index cc1911b60d4..1e778727522 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
@@ -15,7 +15,7 @@ type MessagePanelProps = {
   index: number;
   message: Message;
   isStructuredOutput?: boolean;
-  isChoice?: boolean;
+  choiceIndex?: number;
   isNested?: boolean;
   pendingToolResponseId?: string;
 };
@@ -24,7 +24,7 @@ export const MessagePanel = ({
   index,
   message,
   isStructuredOutput,
-  isChoice,
+  choiceIndex,
   isNested,
   // The id of the tool call response that is pending
   // If the tool call response is pending, the editor will be shown automatically
@@ -120,7 +120,7 @@ export const MessagePanel = ({
                 <PlaygroundMessagePanelEditor
                   message={message}
                   index={index}
-                  isChoice={isChoice ?? false}
+                  choiceIndex={choiceIndex}
                   editorHeight={editorHeight}
                   isNested={isNested ?? false}
                   pendingToolResponseId={pendingToolResponseId}
@@ -173,7 +173,7 @@ export const MessagePanel = ({
         <div className="flex w-full items-center justify-start opacity-0 group-hover:opacity-100">
           <PlaygroundMessagePanelButtons
             index={message.original_index ?? index}
-            isChoice={isChoice ?? false}
+            choiceIndex={choiceIndex}
             isTool={isTool}
             hasContent={hasContent}
             contentRef={contentRef}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelButtons.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelButtons.tsx
index 771a9d4075f..923f70762de 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelButtons.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelButtons.tsx
@@ -5,7 +5,7 @@ import {usePlaygroundContext} from '../PlaygroundPage/PlaygroundContext';
 
 type PlaygroundMessagePanelButtonsProps = {
   index: number;
-  isChoice: boolean;
+  choiceIndex?: number;
   isTool: boolean;
   hasContent: boolean;
   contentRef: React.RefObject<HTMLDivElement>;
@@ -17,7 +17,7 @@ export const PlaygroundMessagePanelButtons: React.FC<
   PlaygroundMessagePanelButtonsProps
 > = ({
   index,
-  isChoice,
+  choiceIndex,
   isTool,
   hasContent,
   contentRef,
@@ -32,7 +32,7 @@ export const PlaygroundMessagePanelButtons: React.FC<
         variant="quiet"
         size="small"
         startIcon="randomize-reset-reload"
-        onClick={() => retry?.(index, isChoice)}
+        onClick={() => retry?.(index, choiceIndex)}
         tooltip={
           !hasContent
             ? 'We currently do not support retrying functions'
@@ -64,8 +64,8 @@ export const PlaygroundMessagePanelButtons: React.FC<
         size="small"
         startIcon="delete"
         onClick={() => {
-          if (isChoice) {
-            deleteChoice?.(index);
+          if (choiceIndex !== undefined) {
+            deleteChoice?.(index, choiceIndex);
           } else {
             deleteMessage?.(index, responseIndexes);
           }
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx
index 746b033579a..aa519c9659b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx
@@ -13,7 +13,7 @@ type PlaygroundMessagePanelEditorProps = {
   pendingToolResponseId?: string;
   message: Message;
   index: number;
-  isChoice: boolean;
+  choiceIndex?: number;
   setEditorHeight: (height: number | null) => void;
 };
 
@@ -21,7 +21,7 @@ export const PlaygroundMessagePanelEditor: React.FC<
   PlaygroundMessagePanelEditorProps
 > = ({
   index,
-  isChoice,
+  choiceIndex,
   setEditorHeight,
   editorHeight,
   isNested,
@@ -45,10 +45,10 @@ export const PlaygroundMessagePanelEditor: React.FC<
   }, [initialContent]);
 
   const handleSave = () => {
-    if (isChoice) {
-      editChoice?.(index, {
+    if (choiceIndex !== undefined) {
+      editChoice?.(choiceIndex, {
+        ...message,
         content: editedContent,
-        role: message.role,
       });
     } else {
       editMessage?.(index, {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
index 94cd3c17644..b6b6e7c420d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
@@ -161,8 +161,8 @@ export const PlaygroundChat = ({
                           addMessage: newMessage => addMessage(idx, newMessage),
                           editChoice: (choiceIndex, newChoice) =>
                             editChoice(idx, choiceIndex, newChoice),
-                          retry: (messageIndex: number, isChoice?: boolean) =>
-                            handleRetry(idx, messageIndex, isChoice),
+                          retry: (messageIndex: number, choiceIndex?: number) =>
+                            handleRetry(idx, messageIndex, choiceIndex),
                           sendMessage: (
                             role: PlaygroundMessageRole,
                             content: string,
@@ -170,6 +170,12 @@ export const PlaygroundChat = ({
                           ) => {
                             handleSend(role, idx, content, toolCallId);
                           },
+                          setSelectedChoiceIndex: (choiceIndex: number) =>
+                            setPlaygroundStateField(
+                              idx,
+                              'selectedChoiceIndex',
+                              choiceIndex
+                            ),
                         }}>
                         <CallChat call={state.traceCall as TraceCallSchema} />
                       </PlaygroundContext.Provider>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatCompletionFunctions.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatCompletionFunctions.tsx
index 10c76fc82d1..c73e5d42919 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatCompletionFunctions.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatCompletionFunctions.tsx
@@ -70,7 +70,7 @@ export const useChatCompletionFunctions = (
         if (callIndex !== undefined && callIndex !== index) {
           return state;
         }
-        const updatedState = appendChoicesToMessages(state);
+        const updatedState = appendChoiceToMessages(state);
         if (updatedState.traceCall?.inputs?.messages) {
           updatedState.traceCall.inputs.messages.push(newMessage);
         }
@@ -99,14 +99,14 @@ export const useChatCompletionFunctions = (
   const handleRetry = async (
     callIndex: number,
     messageIndex: number,
-    isChoice?: boolean
+    choiceIndex?: number
   ) => {
     try {
       setIsLoading(true);
       const updatedStates = playgroundStates.map((state, index) => {
         if (index === callIndex) {
-          if (isChoice) {
-            return appendChoicesToMessages(state);
+          if (choiceIndex !== undefined) {
+            return appendChoiceToMessages(state, choiceIndex);
           }
           const updatedState = JSON.parse(JSON.stringify(state));
           if (updatedState.traceCall?.inputs?.messages) {
@@ -203,17 +203,25 @@ const handleUpdateCallWithResponse = (
   };
 };
 
-const appendChoicesToMessages = (state: PlaygroundState): PlaygroundState => {
+const appendChoiceToMessages = (
+  state: PlaygroundState,
+  choiceIndex?: number
+): PlaygroundState => {
   const updatedState = JSON.parse(JSON.stringify(state));
   if (
     updatedState.traceCall?.inputs?.messages &&
     updatedState.traceCall.output?.choices
   ) {
-    updatedState.traceCall.output.choices.forEach((choice: any) => {
-      if (choice.message) {
-        updatedState.traceCall.inputs.messages.push(choice.message);
-      }
-    });
+    if (choiceIndex !== undefined) {
+      updatedState.traceCall.inputs.messages.push(
+        updatedState.traceCall.output.choices[choiceIndex].message
+      );
+    } else {
+      updatedState.traceCall.inputs.messages.push(
+        updatedState.traceCall.output.choices[updatedState.selectedChoiceIndex]
+          .message
+      );
+    }
     updatedState.traceCall.output.choices = undefined;
   }
   return updatedState;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx
index e84e2f75d4b..804670a1dc3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx
@@ -114,16 +114,9 @@ export const useChatFunctions = (
         newTraceCall?.output &&
         Array.isArray((newTraceCall.output as TraceCallOutput).choices)
       ) {
-        // Delete the old choice
-        (newTraceCall.output as TraceCallOutput).choices!.splice(
-          choiceIndex,
-          1
-        );
-
-        // Add the new choice as a message
-        newTraceCall.inputs = newTraceCall.inputs ?? {};
-        newTraceCall.inputs.messages = newTraceCall.inputs.messages ?? [];
-        newTraceCall.inputs.messages.push(newChoice);
+        // Replace the choice
+        (newTraceCall.output as TraceCallOutput).choices![choiceIndex].message =
+          newChoice;
       }
       return newTraceCall;
     });
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundContext.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundContext.tsx
index a8176292d1a..31369602560 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundContext.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundContext.tsx
@@ -10,14 +10,16 @@ export type PlaygroundContextType = {
   deleteMessage: (messageIndex: number, responseIndexes?: number[]) => void;
 
   editChoice: (choiceIndex: number, newChoice: Message) => void;
-  deleteChoice: (choiceIndex: number) => void;
+  deleteChoice: (messageIndex: number, choiceIndex: number) => void;
 
-  retry: (messageIndex: number, isChoice?: boolean) => void;
+  retry: (messageIndex: number, choiceIndex?: number) => void;
   sendMessage: (
     role: PlaygroundMessageRole,
     content: string,
     toolCallId?: string
   ) => void;
+
+  setSelectedChoiceIndex: (choiceIndex: number) => void;
 };
 
 const DEFAULT_CONTEXT: PlaygroundContextType = {
@@ -31,6 +33,7 @@ const DEFAULT_CONTEXT: PlaygroundContextType = {
 
   retry: () => {},
   sendMessage: () => {},
+  setSelectedChoiceIndex: () => {},
 };
 
 // Create context that can be undefined
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/types.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/types.ts
index fa73e87bf45..29495fcf535 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/types.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/types.ts
@@ -23,6 +23,7 @@ export type PlaygroundState = {
   //   nTimes: number;
   maxTokensLimit: number;
   model: LLMMaxTokensKey;
+  selectedChoiceIndex: number;
 };
 
 export type PlaygroundStateKey = keyof PlaygroundState;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts
index 8c556edaef2..b0c06ddc677 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts
@@ -37,6 +37,7 @@ const DEFAULT_PLAYGROUND_STATE = {
   //   nTimes: 1,
   maxTokensLimit: 16384,
   model: DEFAULT_MODEL,
+  selectedChoiceIndex: 0,
 };
 
 export const usePlaygroundState = () => {

From 1a9485f64e4ef7e1909d53cc31239bede8d17e95 Mon Sep 17 00:00:00 2001
From: Josiah Lee <josiahwlee@gmail.com>
Date: Wed, 11 Dec 2024 13:19:30 -0800
Subject: [PATCH 34/52] unhide completion iterations (#3137)

---
 .../PlaygroundSettings/PlaygroundSettings.tsx | 41 ++++++++++++-------
 .../Browse3/pages/PlaygroundPage/types.ts     |  2 +-
 .../PlaygroundPage/usePlaygroundState.ts      | 10 ++---
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundSettings/PlaygroundSettings.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundSettings/PlaygroundSettings.tsx
index 5a2e8fae32c..e0971d35bfb 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundSettings/PlaygroundSettings.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundSettings/PlaygroundSettings.tsx
@@ -59,6 +59,12 @@ export const PlaygroundSettings: React.FC<PlaygroundSettingsProps> = ({
                 gap: '4px',
                 mt: 2,
               }}>
+              <ResponseFormatEditor
+                responseFormat={playgroundState.responseFormat}
+                setResponseFormat={value =>
+                  setPlaygroundStateField(idx, 'responseFormat', value)
+                }
+              />
               <FunctionEditor
                 playgroundState={playgroundState}
                 functions={playgroundState.functions}
@@ -71,22 +77,24 @@ export const PlaygroundSettings: React.FC<PlaygroundSettingsProps> = ({
                 }
               />
 
-              <ResponseFormatEditor
-                responseFormat={playgroundState.responseFormat}
-                setResponseFormat={value =>
-                  setPlaygroundStateField(idx, 'responseFormat', value)
+              <StopSequenceEditor
+                stopSequences={playgroundState.stopSequences}
+                setStopSequences={value =>
+                  setPlaygroundStateField(idx, 'stopSequences', value)
                 }
               />
 
+              {/* TODO: N times to run is not supported for all models */}
+              {/* TODO: rerun if this is not supported in the backend */}
               <PlaygroundSlider
-                min={0}
-                max={2}
-                step={0.01}
+                min={1}
+                max={100}
+                step={1}
                 setValue={value =>
-                  setPlaygroundStateField(idx, 'temperature', value)
+                  setPlaygroundStateField(idx, 'nTimes', value)
                 }
-                label="Temperature"
-                value={playgroundState.temperature}
+                label="Completion iterations"
+                value={playgroundState.nTimes}
               />
 
               <PlaygroundSlider
@@ -100,11 +108,15 @@ export const PlaygroundSettings: React.FC<PlaygroundSettingsProps> = ({
                 value={playgroundState.maxTokens}
               />
 
-              <StopSequenceEditor
-                stopSequences={playgroundState.stopSequences}
-                setStopSequences={value =>
-                  setPlaygroundStateField(idx, 'stopSequences', value)
+              <PlaygroundSlider
+                min={0}
+                max={2}
+                step={0.01}
+                setValue={value =>
+                  setPlaygroundStateField(idx, 'temperature', value)
                 }
+                label="Temperature"
+                value={playgroundState.temperature}
               />
 
               <PlaygroundSlider
@@ -137,6 +149,7 @@ export const PlaygroundSettings: React.FC<PlaygroundSettingsProps> = ({
                 label="Presence penalty"
                 value={playgroundState.presencePenalty}
               />
+
               <Box
                 sx={{
                   width: '100%',
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/types.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/types.ts
index 29495fcf535..dc0e0a370e0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/types.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/types.ts
@@ -20,7 +20,7 @@ export type PlaygroundState = {
   topP: number;
   frequencyPenalty: number;
   presencePenalty: number;
-  //   nTimes: number;
+  nTimes: number;
   maxTokensLimit: number;
   model: LLMMaxTokensKey;
   selectedChoiceIndex: number;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts
index b0c06ddc677..8439abc4ddf 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts
@@ -34,7 +34,7 @@ const DEFAULT_PLAYGROUND_STATE = {
   topP: 1,
   frequencyPenalty: 0,
   presencePenalty: 0,
-  //   nTimes: 1,
+  nTimes: 1,
   maxTokensLimit: 16384,
   model: DEFAULT_MODEL,
   selectedChoiceIndex: 0,
@@ -91,9 +91,9 @@ export const usePlaygroundState = () => {
             }
           }
         }
-        // if (inputs.n) {
-        //   newState.nTimes = parseInt(inputs.n, 10);
-        // }
+        if (inputs.n) {
+          newState.nTimes = parseInt(inputs.n, 10);
+        }
         if (inputs.temperature) {
           newState.temperature = parseFloat(inputs.temperature);
         }
@@ -148,7 +148,7 @@ export const getInputFromPlaygroundState = (state: PlaygroundState) => {
     top_p: state.topP,
     frequency_penalty: state.frequencyPenalty,
     presence_penalty: state.presencePenalty,
-    // n: state.nTimes,
+    n: state.nTimes,
     response_format: {
       type: state.responseFormat,
     },

From 74430483998c6f0ace4e248669bc58a81e54cb2d Mon Sep 17 00:00:00 2001
From: Weave Build Bot <weave@wandb.com>
Date: Wed, 11 Dec 2024 21:28:01 +0000
Subject: [PATCH 35/52] =?UTF-8?q?Release=20version:=200.51.24-dev0=20?=
 =?UTF-8?q?=E2=86=92=200.51.24?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pyproject.toml   | 2 +-
 weave/version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index d392d527b60..0f8eec3a30c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -226,7 +226,7 @@ module = "weave_query.*"
 ignore_errors = true
 
 [tool.bumpversion]
-current_version = "0.51.24-dev0"
+current_version = "0.51.24"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
diff --git a/weave/version.py b/weave/version.py
index 5212f6aee7d..9ad7975da7d 100644
--- a/weave/version.py
+++ b/weave/version.py
@@ -44,4 +44,4 @@
 
 """
 
-VERSION = "0.51.24-dev0"
+VERSION = "0.51.24"

From 43e2b7bd9a83ea8e1017f999994e08a933bc77a0 Mon Sep 17 00:00:00 2001
From: Weave Build Bot <weave@wandb.com>
Date: Wed, 11 Dec 2024 21:28:01 +0000
Subject: [PATCH 36/52] =?UTF-8?q?Release=20version:=200.51.24=20=E2=86=92?=
 =?UTF-8?q?=200.51.25-dev0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pyproject.toml   | 2 +-
 weave/version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 0f8eec3a30c..f34757b315e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -226,7 +226,7 @@ module = "weave_query.*"
 ignore_errors = true
 
 [tool.bumpversion]
-current_version = "0.51.24"
+current_version = "0.51.25-dev0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
diff --git a/weave/version.py b/weave/version.py
index 9ad7975da7d..70f670abc21 100644
--- a/weave/version.py
+++ b/weave/version.py
@@ -44,4 +44,4 @@
 
 """
 
-VERSION = "0.51.24"
+VERSION = "0.51.25-dev0"

From 2c93fa653fb2b3c21e7a2b7d3fd40962e17eb02b Mon Sep 17 00:00:00 2001
From: Josiah Lee <josiahwlee@gmail.com>
Date: Wed, 11 Dec 2024 13:53:24 -0800
Subject: [PATCH 37/52] style(weave): restyle carousel view (#3205)

* add choices drawer

* fix choices in playground

* restyle carousel viewer

* prettier
---
 weave-js/src/assets/icons/icon-visible.svg    |  4 ++
 weave-js/src/components/Icon/Icon.tsx         |  5 ++
 weave-js/src/components/Icon/index.ts         |  1 +
 weave-js/src/components/Icon/types.ts         |  1 +
 .../Browse3/pages/ChatView/ChoiceView.tsx     |  3 +
 .../pages/ChatView/ChoicesViewCarousel.tsx    | 57 ++++++++++---------
 .../Browse3/pages/ChatView/MessagePanel.tsx   |  3 +
 .../PlaygroundChat/PlaygroundChat.tsx         |  1 +
 .../PlaygroundChat/useChatFunctions.tsx       |  3 +
 9 files changed, 50 insertions(+), 28 deletions(-)
 create mode 100644 weave-js/src/assets/icons/icon-visible.svg

diff --git a/weave-js/src/assets/icons/icon-visible.svg b/weave-js/src/assets/icons/icon-visible.svg
new file mode 100644
index 00000000000..823c36bb17f
--- /dev/null
+++ b/weave-js/src/assets/icons/icon-visible.svg
@@ -0,0 +1,4 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M17 12C17 14.7614 14.7614 17 12 17C9.23858 17 7 14.7614 7 12C7 9.23858 9.23858 7 12 7C14.7614 7 17 9.23858 17 12ZM15.5 12C15.5 13.933 13.933 15.5 12 15.5C10.067 15.5 8.5 13.933 8.5 12C8.5 10.067 10.067 8.5 12 8.5C13.933 8.5 15.5 10.067 15.5 12Z" fill="currentColor"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M12 3.75C8.28497 3.75 5.5985 5.21584 3.84467 6.96967C2.97227 7.84207 2.33072 8.78514 1.90468 9.65426C1.48605 10.5083 1.25 11.3438 1.25 12C1.25 12.6562 1.48605 13.4917 1.90468 14.3457C2.33072 15.2149 2.97227 16.1579 3.84467 17.0303C5.5985 18.7842 8.28497 20.25 12 20.25C15.715 20.25 18.4015 18.7842 20.1553 17.0303C21.0277 16.1579 21.6693 15.2149 22.0953 14.3457C22.5139 13.4917 22.75 12.6562 22.75 12C22.75 11.3438 22.5139 10.5083 22.0953 9.65426C21.6693 8.78514 21.0277 7.84207 20.1553 6.96967C18.4015 5.21584 15.715 3.75 12 3.75ZM2.75 12C2.75 11.6562 2.88895 11.0542 3.25156 10.3145C3.60678 9.58986 4.15273 8.78293 4.90533 8.03033C6.4015 6.53416 8.71503 5.25 12 5.25C15.285 5.25 17.5985 6.53416 19.0947 8.03033C19.8473 8.78293 20.3932 9.58986 20.7484 10.3145C21.1111 11.0542 21.25 11.6562 21.25 12C21.25 12.3438 21.1111 12.9458 20.7484 13.6855C20.3932 14.4101 19.8473 15.2171 19.0947 15.9697C17.5985 17.4658 15.285 18.75 12 18.75C8.71503 18.75 6.4015 17.4658 4.90533 15.9697C4.15273 15.2171 3.60678 14.4101 3.25156 13.6855C2.88895 12.9458 2.75 12.3438 2.75 12Z" fill="currentColor"/>
+</svg>
diff --git a/weave-js/src/components/Icon/Icon.tsx b/weave-js/src/components/Icon/Icon.tsx
index 9c020b1bab9..32b9ca65921 100644
--- a/weave-js/src/components/Icon/Icon.tsx
+++ b/weave-js/src/components/Icon/Icon.tsx
@@ -256,6 +256,7 @@ import {ReactComponent as ImportVersionsLayers} from '../../assets/icons/icon-ve
 import {ReactComponent as ImportVertexGCP} from '../../assets/icons/icon-vertex-gcp.svg';
 import {ReactComponent as ImportVideoPlay} from '../../assets/icons/icon-video-play.svg';
 import {ReactComponent as ImportViewGlasses} from '../../assets/icons/icon-view-glasses.svg';
+import {ReactComponent as ImportVisible} from '../../assets/icons/icon-visible.svg';
 import {ReactComponent as ImportWandb} from '../../assets/icons/icon-wandb.svg';
 import {ReactComponent as ImportWarning} from '../../assets/icons/icon-warning.svg';
 import {ReactComponent as ImportWarningAlt} from '../../assets/icons/icon-warning-alt.svg';
@@ -1048,6 +1049,9 @@ export const IconVideoPlay = (props: SVGIconProps) => (
 export const IconViewGlasses = (props: SVGIconProps) => (
   <ImportViewGlasses {...updateIconProps(props)} />
 );
+export const IconVisible = (props: SVGIconProps) => (
+  <ImportVisible {...updateIconProps(props)} />
+);
 export const IconWandb = (props: SVGIconProps) => (
   <ImportWandb {...updateIconProps(props)} />
 );
@@ -1336,6 +1340,7 @@ const ICON_NAME_TO_ICON: Record<IconName, ElementType> = {
   'vertex-gcp': IconVertexGCP,
   'video-play': IconVideoPlay,
   'view-glasses': IconViewGlasses,
+  visible: IconVisible,
   wandb: IconWandb,
   warning: IconWarning,
   'warning-alt': IconWarningAlt,
diff --git a/weave-js/src/components/Icon/index.ts b/weave-js/src/components/Icon/index.ts
index 85ea5332649..08bf7854ad2 100644
--- a/weave-js/src/components/Icon/index.ts
+++ b/weave-js/src/components/Icon/index.ts
@@ -256,6 +256,7 @@ export {
   IconVertexGCP,
   IconVideoPlay,
   IconViewGlasses,
+  IconVisible,
   IconWandb,
   IconWarning,
   IconWarningAlt,
diff --git a/weave-js/src/components/Icon/types.ts b/weave-js/src/components/Icon/types.ts
index 87a1207bc85..7ca30049257 100644
--- a/weave-js/src/components/Icon/types.ts
+++ b/weave-js/src/components/Icon/types.ts
@@ -255,6 +255,7 @@ export const IconNames = {
   VertexGCP: 'vertex-gcp',
   VideoPlay: 'video-play',
   ViewGlasses: 'view-glasses',
+  Visible: 'visible',
   Wandb: 'wandb',
   Warning: 'warning',
   WarningAlt: 'warning-alt',
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx
index 9e24e7e0a4a..e511d6fbf5c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoiceView.tsx
@@ -8,6 +8,7 @@ type ChoiceViewProps = {
   isStructuredOutput?: boolean;
   isNested?: boolean;
   choiceIndex?: number;
+  messageHeader?: React.ReactNode;
 };
 
 export const ChoiceView = ({
@@ -15,6 +16,7 @@ export const ChoiceView = ({
   isStructuredOutput,
   isNested,
   choiceIndex,
+  messageHeader,
 }: ChoiceViewProps) => {
   const {message} = choice;
   return (
@@ -24,6 +26,7 @@ export const ChoiceView = ({
       isStructuredOutput={isStructuredOutput}
       isNested={isNested}
       choiceIndex={choiceIndex}
+      messageHeader={messageHeader}
     />
   );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx
index 18760817665..b7dc6eb427d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesViewCarousel.tsx
@@ -29,38 +29,39 @@ export const ChoicesViewCarousel = ({
   };
 
   return (
-    <>
-      <ChoiceView
-        choice={choices[selectedChoiceIndex]}
-        isStructuredOutput={isStructuredOutput}
-        choiceIndex={selectedChoiceIndex}
-      />
-      <div className="flex items-center">
-        <div className="flex-auto">
+    <ChoiceView
+      choice={choices[selectedChoiceIndex]}
+      isStructuredOutput={isStructuredOutput}
+      choiceIndex={selectedChoiceIndex}
+      messageHeader={
+        <div className="mb-8 flex items-center gap-4">
+          <div className="flex items-center gap-12">
+            <Button
+              variant="ghost"
+              icon="chevron-back"
+              size="small"
+              onClick={onBack}
+            />
+            <span className="text-sm text-moon-500">
+              {selectedChoiceIndex + 1} of {choices.length}
+            </span>
+            <Button
+              variant="ghost"
+              icon="chevron-next"
+              size="small"
+              onClick={onNext}
+            />
+          </div>
           <Button
             size="small"
             variant="quiet"
-            icon="expand-uncollapse"
+            icon="visible"
             onClick={() => setIsDrawerOpen(true)}
-            tooltip="Switch to linear view"
-          />
+            tooltip="Switch to linear view">
+            Review
+          </Button>
         </div>
-        <div className="flex items-center gap-12">
-          <Button
-            variant="ghost"
-            icon="chevron-back"
-            size="small"
-            onClick={onBack}
-          />
-          {selectedChoiceIndex + 1} of {choices.length}
-          <Button
-            variant="ghost"
-            icon="chevron-next"
-            size="small"
-            onClick={onNext}
-          />
-        </div>
-      </div>
-    </>
+      }
+    />
   );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
index 1e778727522..f570b2f6295 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
@@ -18,6 +18,7 @@ type MessagePanelProps = {
   choiceIndex?: number;
   isNested?: boolean;
   pendingToolResponseId?: string;
+  messageHeader?: React.ReactNode;
 };
 
 export const MessagePanel = ({
@@ -30,6 +31,7 @@ export const MessagePanel = ({
   // If the tool call response is pending, the editor will be shown automatically
   // and on save the tool call response will be updated and sent to the LLM
   pendingToolResponseId,
+  messageHeader,
 }: MessagePanelProps) => {
   const [isShowingMore, setIsShowingMore] = useState(false);
   const [isOverflowing, setIsOverflowing] = useState(false);
@@ -116,6 +118,7 @@ export const MessagePanel = ({
                 'max-h-[400px]': !isShowingMore,
                 'max-h-full': isShowingMore,
               })}>
+              {messageHeader}
               {isPlayground && editorHeight ? (
                 <PlaygroundMessagePanelEditor
                   message={message}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
index b6b6e7c420d..38f69c4482e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
@@ -35,6 +35,7 @@ export const PlaygroundChat = ({
   setSettingsTab,
   settingsTab,
 }: PlaygroundChatProps) => {
+  console.log('playgroundStates', playgroundStates);
   const [chatText, setChatText] = useState('');
   const [isLoading, setIsLoading] = useState(false);
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx
index 804670a1dc3..0ce3ad02b51 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx
@@ -43,6 +43,8 @@ export const useChatFunctions = (
     messageIndex: number,
     newMessage: Message
   ) => {
+    console.log('editMessage', callIndex, messageIndex, newMessage);
+
     setPlaygroundStateField(callIndex, 'traceCall', prevTraceCall => {
       const newTraceCall = clearTraceCall(
         cloneDeep(prevTraceCall as OptionalTraceCallSchema)
@@ -106,6 +108,7 @@ export const useChatFunctions = (
     choiceIndex: number,
     newChoice: Message
   ) => {
+    console.log('editChoice', callIndex, choiceIndex, newChoice);
     setPlaygroundStateField(callIndex, 'traceCall', prevTraceCall => {
       const newTraceCall = clearTraceCall(
         cloneDeep(prevTraceCall as OptionalTraceCallSchema)

From a0f12639451979efc01ef3b31767444542688ea4 Mon Sep 17 00:00:00 2001
From: Andrew Truong <itsandrewtruong@gmail.com>
Date: Wed, 11 Dec 2024 16:56:46 -0500
Subject: [PATCH 38/52] chore(weave): Add generic iterator for trace server API
 objects (#3177)

---
 weave/trace/weave_client.py | 207 ++++++++++++++++++++++--------------
 1 file changed, 129 insertions(+), 78 deletions(-)

diff --git a/weave/trace/weave_client.py b/weave/trace/weave_client.py
index 0eca3fcbedb..1d5d54b9b23 100644
--- a/weave/trace/weave_client.py
+++ b/weave/trace/weave_client.py
@@ -10,7 +10,7 @@
 from collections.abc import Iterator, Sequence
 from concurrent.futures import Future
 from functools import lru_cache
-from typing import Any, Callable, cast
+from typing import Any, Callable, Generic, Protocol, TypeVar, cast, overload
 
 import pydantic
 from requests import HTTPError
@@ -90,6 +90,128 @@
 
 logger = logging.getLogger(__name__)
 
+T = TypeVar("T")
+R = TypeVar("R", covariant=True)
+
+
+class FetchFunc(Protocol[T]):
+    def __call__(self, offset: int, limit: int) -> list[T]: ...
+
+
+TransformFunc = Callable[[T], R]
+
+
+class PaginatedIterator(Generic[T, R]):
+    """An iterator that fetches pages of items from a server and optionally transforms them
+    into a more user-friendly type."""
+
+    def __init__(
+        self,
+        fetch_func: FetchFunc[T],
+        page_size: int = 1000,
+        transform_func: TransformFunc[T, R] | None = None,
+    ) -> None:
+        self.fetch_func = fetch_func
+        self.page_size = page_size
+        self.transform_func = transform_func
+
+        if page_size <= 0:
+            raise ValueError("page_size must be greater than 0")
+
+    @lru_cache
+    def _fetch_page(self, index: int) -> list[T]:
+        return self.fetch_func(index * self.page_size, self.page_size)
+
+    @overload
+    def _get_one(self: PaginatedIterator[T, T], index: int) -> T: ...
+    @overload
+    def _get_one(self: PaginatedIterator[T, R], index: int) -> R: ...
+    def _get_one(self, index: int) -> T | R:
+        if index < 0:
+            raise IndexError("Negative indexing not supported")
+
+        page_index = index // self.page_size
+        page_offset = index % self.page_size
+
+        page = self._fetch_page(page_index)
+        if page_offset >= len(page):
+            raise IndexError(f"Index {index} out of range")
+
+        res = page[page_offset]
+        if transform := self.transform_func:
+            return transform(res)
+        return res
+
+    @overload
+    def _get_slice(self: PaginatedIterator[T, T], key: slice) -> Iterator[T]: ...
+    @overload
+    def _get_slice(self: PaginatedIterator[T, R], key: slice) -> Iterator[R]: ...
+    def _get_slice(self, key: slice) -> Iterator[T] | Iterator[R]:
+        if (start := key.start or 0) < 0:
+            raise ValueError("Negative start not supported")
+        if (stop := key.stop) is not None and stop < 0:
+            raise ValueError("Negative stop not supported")
+        if (step := key.step or 1) < 0:
+            raise ValueError("Negative step not supported")
+
+        i = start
+        while stop is None or i < stop:
+            try:
+                yield self._get_one(i)
+            except IndexError:
+                break
+            i += step
+
+    @overload
+    def __getitem__(self: PaginatedIterator[T, T], key: int) -> T: ...
+    @overload
+    def __getitem__(self: PaginatedIterator[T, R], key: int) -> R: ...
+    @overload
+    def __getitem__(self: PaginatedIterator[T, T], key: slice) -> list[T]: ...
+    @overload
+    def __getitem__(self: PaginatedIterator[T, R], key: slice) -> list[R]: ...
+    def __getitem__(self, key: slice | int) -> T | R | list[T] | list[R]:
+        if isinstance(key, slice):
+            return list(self._get_slice(key))
+        return self._get_one(key)
+
+    @overload
+    def __iter__(self: PaginatedIterator[T, T]) -> Iterator[T]: ...
+    @overload
+    def __iter__(self: PaginatedIterator[T, R]) -> Iterator[R]: ...
+    def __iter__(self) -> Iterator[T] | Iterator[R]:
+        return self._get_slice(slice(0, None, 1))
+
+
+# TODO: should be Call, not WeaveObject
+CallsIter = PaginatedIterator[CallSchema, WeaveObject]
+
+
+def _make_calls_iterator(
+    server: TraceServerInterface,
+    project_id: str,
+    filter: CallsFilter,
+    include_costs: bool = False,
+) -> CallsIter:
+    def fetch_func(offset: int, limit: int) -> list[CallSchema]:
+        response = server.calls_query(
+            CallsQueryReq(
+                project_id=project_id,
+                filter=filter,
+                offset=offset,
+                limit=limit,
+                include_costs=include_costs,
+            )
+        )
+        return response.calls
+
+    # TODO: Should be Call, not WeaveObject
+    def transform_func(call: CallSchema) -> WeaveObject:
+        entity, project = project_id.split("/")
+        return make_client_call(entity, project, call, server)
+
+    return PaginatedIterator(fetch_func, transform_func=transform_func)
+
 
 class OpNameError(ValueError):
     """Raised when an op name is invalid."""
@@ -284,7 +406,7 @@ def children(self) -> CallsIter:
             )
 
         client = weave_client_context.require_weave_client()
-        return CallsIter(
+        return _make_calls_iterator(
             client.server,
             self.project_id,
             CallsFilter(parent_ids=[self.id]),
@@ -362,80 +484,6 @@ def _apply_scorer(self, scorer_op: Op) -> None:
         )
 
 
-class CallsIter:
-    server: TraceServerInterface
-    filter: CallsFilter
-    include_costs: bool
-
-    def __init__(
-        self,
-        server: TraceServerInterface,
-        project_id: str,
-        filter: CallsFilter,
-        include_costs: bool = False,
-    ) -> None:
-        self.server = server
-        self.project_id = project_id
-        self.filter = filter
-        self._page_size = 1000
-        self.include_costs = include_costs
-
-    # seems like this caching should be on the server, but it's here for now...
-    @lru_cache
-    def _fetch_page(self, index: int) -> list[CallSchema]:
-        # caching in here means that any other CallsIter objects would also
-        # benefit from the cache
-        response = self.server.calls_query(
-            CallsQueryReq(
-                project_id=self.project_id,
-                filter=self.filter,
-                offset=index * self._page_size,
-                limit=self._page_size,
-                include_costs=self.include_costs,
-            )
-        )
-        return response.calls
-
-    def _get_one(self, index: int) -> WeaveObject:
-        if index < 0:
-            raise IndexError("Negative indexing not supported")
-
-        page_index = index // self._page_size
-        page_offset = index % self._page_size
-
-        calls = self._fetch_page(page_index)
-        if page_offset >= len(calls):
-            raise IndexError(f"Index {index} out of range")
-
-        call = calls[page_offset]
-        entity, project = self.project_id.split("/")
-        return make_client_call(entity, project, call, self.server)
-
-    def _get_slice(self, key: slice) -> Iterator[WeaveObject]:
-        if (start := key.start or 0) < 0:
-            raise ValueError("Negative start not supported")
-        if (stop := key.stop) is not None and stop < 0:
-            raise ValueError("Negative stop not supported")
-        if (step := key.step or 1) < 0:
-            raise ValueError("Negative step not supported")
-
-        i = start
-        while stop is None or i < stop:
-            try:
-                yield self._get_one(i)
-            except IndexError:
-                break
-            i += step
-
-    def __getitem__(self, key: slice | int) -> WeaveObject | list[WeaveObject]:
-        if isinstance(key, slice):
-            return list(self._get_slice(key))
-        return self._get_one(key)
-
-    def __iter__(self) -> Iterator[WeaveObject]:
-        return self._get_slice(slice(0, None, 1))
-
-
 def make_client_call(
     entity: str, project: str, server_call: CallSchema, server: TraceServerInterface
 ) -> WeaveObject:
@@ -642,8 +690,11 @@ def get_calls(
         if filter is None:
             filter = CallsFilter()
 
-        return CallsIter(
-            self.server, self._project_id(), filter, include_costs or False
+        return _make_calls_iterator(
+            self.server,
+            self._project_id(),
+            filter,
+            include_costs,
         )
 
     @deprecated(new_name="get_calls")

From 0525621c503ee6082a2d809f17bef64b04ea71ef Mon Sep 17 00:00:00 2001
From: Andrew Truong <itsandrewtruong@gmail.com>
Date: Wed, 11 Dec 2024 17:05:51 -0500
Subject: [PATCH 39/52] feat(weave): Support op configuration for autopatched
 functions (starting with OpenAI) (#3197)

---
 tests/conftest.py                             |  12 +-
 .../test_configuration_with_dicts.yaml        | 102 +++++++++++++
 ...est_disabled_integration_doesnt_patch.yaml | 102 +++++++++++++
 .../test_enabled_integration_patches.yaml     | 102 +++++++++++++
 .../test_passthrough_op_kwargs.yaml           | 102 +++++++++++++
 tests/integrations/openai/test_autopatch.py   | 116 +++++++++++++++
 weave/integrations/openai/openai_sdk.py       | 136 +++++++++++-------
 weave/scorers/llm_utils.py                    |   4 -
 weave/trace/api.py                            |   9 +-
 weave/trace/autopatch.py                      |  59 +++++++-
 weave/trace/patcher.py                        |   8 ++
 weave/trace/weave_init.py                     |   6 +-
 12 files changed, 689 insertions(+), 69 deletions(-)
 create mode 100644 tests/integrations/openai/cassettes/test_autopatch/test_configuration_with_dicts.yaml
 create mode 100644 tests/integrations/openai/cassettes/test_autopatch/test_disabled_integration_doesnt_patch.yaml
 create mode 100644 tests/integrations/openai/cassettes/test_autopatch/test_enabled_integration_patches.yaml
 create mode 100644 tests/integrations/openai/cassettes/test_autopatch/test_passthrough_op_kwargs.yaml
 create mode 100644 tests/integrations/openai/test_autopatch.py

diff --git a/tests/conftest.py b/tests/conftest.py
index b28187a3833..85e9b53c36b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -477,7 +477,9 @@ def __getattribute__(self, name):
     return ServerRecorder(server)
 
 
-def create_client(request) -> weave_init.InitializedClient:
+def create_client(
+    request, autopatch_settings: typing.Optional[autopatch.AutopatchSettings] = None
+) -> weave_init.InitializedClient:
     inited_client = None
     weave_server_flag = request.config.getoption("--weave-server")
     server: tsi.TraceServerInterface
@@ -513,7 +515,7 @@ def create_client(request) -> weave_init.InitializedClient:
             entity, project, make_server_recorder(server)
         )
         inited_client = weave_init.InitializedClient(client)
-        autopatch.autopatch()
+        autopatch.autopatch(autopatch_settings)
 
     return inited_client
 
@@ -527,6 +529,7 @@ def client(request):
         yield inited_client.client
     finally:
         inited_client.reset()
+        autopatch.reset_autopatch()
 
 
 @pytest.fixture()
@@ -534,12 +537,13 @@ def client_creator(request):
     """This fixture is useful for delaying the creation of the client (ex. when you want to set settings first)"""
 
     @contextlib.contextmanager
-    def client():
-        inited_client = create_client(request)
+    def client(autopatch_settings: typing.Optional[autopatch.AutopatchSettings] = None):
+        inited_client = create_client(request, autopatch_settings)
         try:
             yield inited_client.client
         finally:
             inited_client.reset()
+            autopatch.reset_autopatch()
 
     yield client
 
diff --git a/tests/integrations/openai/cassettes/test_autopatch/test_configuration_with_dicts.yaml b/tests/integrations/openai/cassettes/test_autopatch/test_configuration_with_dicts.yaml
new file mode 100644
index 00000000000..7245829a0b3
--- /dev/null
+++ b/tests/integrations/openai/cassettes/test_autopatch/test_configuration_with_dicts.yaml
@@ -0,0 +1,102 @@
+interactions:
+- request:
+    body: '{"messages":[{"role":"user","content":"tell me a joke"}],"model":"gpt-4o"}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '74'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.57.2
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.57.2
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.13.0rc2
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//jFJNa9wwEL37V0x1yWVd7P3KspcSSKE5thvooSlGK40tJbJGSOOSNOx/
+        L/Z+2KEp9KLDe/Me743mNQMQVostCGUkqza4/Eav1f1O/v4aVvvbL/Pdt7tVHUp1s/vsnhsx6xW0
+        f0TFZ9VHRW1wyJb8kVYRJWPvWl4vFpvNYl2UA9GSRtfLmsD5kvJ5MV/mxSYv1iehIaswiS38yAAA
+        Xoe3j+g1PostFLMz0mJKskGxvQwBiEiuR4RMySaWnsVsJBV5Rj+k/m5eQJO/YkhP6JDJJ6htYxhQ
+        KgPEBuOnB//g7w2eJ438hcAGoek4fZgaR6y7JPtevnPuhB8uSR01IdI+nfgLXltvk6kiykS+T5WY
+        ghjYQwbwc9hI96akCJHawBXTE/resCyPdmL8ggm5PJFMLN2Iz1ezd9wqjSytS5ONCiWVQT0qx/XL
+        TluaENmk899h3vM+9ra++R/7kVAKA6OuQkRt1dvC41jE/kD/NXbZ8RBYpJfE2Fa19Q3GEO3xRupQ
+        qWslC9xLJUV2yP4AAAD//wMA4O+DUSwDAAA=
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8f01fe3aabd037cf-YYZ
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 11 Dec 2024 02:20:01 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=xqe_jHZdTV5LijJQYQ3GMY5MjtVrCyxbFO4glgLvgD0-1733883601-1.0.1.1-p.DDUca_cHppJu2hXzzA0CXU1mtalxHUNfBWVgPIQj.UkU603pbNscCvSIi4_Zjlz9Zuc3.hjlvoyZxcDBJTsw;
+        path=/; expires=Wed, 11-Dec-24 02:50:01 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=WEjxXqkGswaEDhllTROGX_go9tgaWNJcUJ3cCd50xDI-1733883601764-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-organization:
+      - wandb
+      openai-processing-ms:
+      - '607'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '30000000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '29999979'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_8592a74b531c806f65c63c7471101cb6
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/integrations/openai/cassettes/test_autopatch/test_disabled_integration_doesnt_patch.yaml b/tests/integrations/openai/cassettes/test_autopatch/test_disabled_integration_doesnt_patch.yaml
new file mode 100644
index 00000000000..1895cdcd5f2
--- /dev/null
+++ b/tests/integrations/openai/cassettes/test_autopatch/test_disabled_integration_doesnt_patch.yaml
@@ -0,0 +1,102 @@
+interactions:
+- request:
+    body: '{"messages":[{"role":"user","content":"tell me a joke"}],"model":"gpt-4o"}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '74'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.57.2
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.57.2
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.13.0rc2
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//jFLJbtswEL3rK6a85GIV8lYvl6KX9NQFrYEckkKgyZHImOII5KiJEfjf
+        C8qLHDQFeuHhbXgzw5cMQFgt1iCUkaya1uWf9Hyuv5H29ebu89Pt80Z9//H0RS2nX/e3LEbJQdtH
+        VHx2vVfUtA7Zkj/SKqBkTKnjxXS6XCwWk3FPNKTRJVvdcj6jfFJMZnmxzIsPJ6MhqzCKNdxnAAAv
+        /Zsqeo3PYg3F6Iw0GKOsUawvIgARyCVEyBhtZOmPdU+kIs/o+9Y/u4AjMBjwJoIEZ2vDuUEZGDU8
+        0g6hogB76tYP/sHfmT1o8jcMcYcOmXyEKlkApTJAbDB8TMKNwbPSyN8IbBDqjuO76xoBqy7KtAXf
+        OXfCD5e5HNVtoG088Re8st5GUwaUkXyaITK1omcPGcCvfn/dq5WINlDTcsm0Q58Cx+NjnBgONpCT
+        2YlkYukGfDofvZFWamRpXbzav1BSGdSDcziW7LSlKyK7mvnvMm9lH+e2vv6f+IFQCltGXbYBtVWv
+        Bx5kAdN3/pfssuO+sIj7yNiUlfU1hjbY44+q2nKl54XSq1WxFdkh+wMAAP//AwAWTTnuWgMAAA==
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8f016eadbff439d2-YYZ
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 11 Dec 2024 00:42:01 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=8FO1yMjc3pMQWRpWrkIe5mcs39GLeqQPmgHQq0YTT8s-1733877721-1.0.1.1-i4G06DBN08aH1F1H73U_TB9OLK3jLsV1jXydB1cQ4Hqx7I.r8xDn.7hFRZe2hy3D_nABTG1nDcdDoXL_wYiqug;
+        path=/; expires=Wed, 11-Dec-24 01:12:01 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=jxwySgtriPkUP8L2os1nb_gRq_SSUo3yWFUyJmHPmGY-1733877721989-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-organization:
+      - wandb
+      openai-processing-ms:
+      - '652'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '30000000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '29999979'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_1c86d4fda2ad715edfd41bcd2f4bdd89
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/integrations/openai/cassettes/test_autopatch/test_enabled_integration_patches.yaml b/tests/integrations/openai/cassettes/test_autopatch/test_enabled_integration_patches.yaml
new file mode 100644
index 00000000000..f0cdca54158
--- /dev/null
+++ b/tests/integrations/openai/cassettes/test_autopatch/test_enabled_integration_patches.yaml
@@ -0,0 +1,102 @@
+interactions:
+- request:
+    body: '{"messages":[{"role":"user","content":"tell me a joke"}],"model":"gpt-4o"}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '74'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.57.2
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.57.2
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.13.0rc2
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//jFLBjtMwEL3nKwZfuDQoTXc3VS8IBOIACCQOHHZR5NrTxDTxWJ4J2rDq
+        v6Ok2SYrFomLD+/Ne3pvxg8JgHJW7UCZWotpQ5O+sdfXaL9+/PDp+L6gG3ffyudv735v+y82eLUa
+        FLT/iUYeVa8MtaFBcTTRJqIWHFzXxWazLYoiz0eiJYvNIKuCpFeU5ll+lWbbNLuZhDU5g6x2cJsA
+        ADyM7xDRW7xXO8hWj0iLzLpCtbsMAahIzYAozexYtBe1mklDXtCPqb/XPVjyLwXYOPTiWBgkdiyg
+        hVp+fefv/Fs0umMEqbGHVh8RugD4C2MvtfPVi6V3xEPHeqjmu6aZ8NMlbENViLTnib/gB+cd12VE
+        zeSHYCwU1MieEoAf41K6Jz1ViNQGKYWO6AfD9fpsp+YrLMh8IoVENzOeb1bPuJUWRbuGF0tVRpsa
+        7aycL6A762hBJIvOf4d5zvvc2/nqf+xnwhgMgrYMEa0zTwvPYxGHP/qvscuOx8CKexZsy4PzFcYQ
+        3fmbHEJpCqMz3GujVXJK/gAAAP//AwAyhdwOLwMAAA==
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8f016eb36bb3a240-YYZ
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 11 Dec 2024 00:42:02 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=Q_ATX8JU4jFqXJPdwlneOua9wmNmAaASyAfcbPyPqng-1733877722-1.0.1.1-eTMEvBW7oqQa2i3l.Or2I3LF_cCESxfseq.S9DBr8dAJWsVoFfPxKtr5vMaO6yj4hRW8XOSOHcgIcwwqbHrLbg;
+        path=/; expires=Wed, 11-Dec-24 01:12:02 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=2ak.tRpn6uEHbM8GrWy_ALtrN34jVSNIJI1mFG2etvM-1733877722703-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-organization:
+      - wandb
+      openai-processing-ms:
+      - '476'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '30000000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '29999979'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_52e061e1cc55cdd8847a7ba9342f1a14
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/integrations/openai/cassettes/test_autopatch/test_passthrough_op_kwargs.yaml b/tests/integrations/openai/cassettes/test_autopatch/test_passthrough_op_kwargs.yaml
new file mode 100644
index 00000000000..646c57c6123
--- /dev/null
+++ b/tests/integrations/openai/cassettes/test_autopatch/test_passthrough_op_kwargs.yaml
@@ -0,0 +1,102 @@
+interactions:
+- request:
+    body: '{"messages":[{"role":"user","content":"tell me a joke"}],"model":"gpt-4o"}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '74'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.57.2
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.57.2
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.13.0rc2
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//jFLLbtswELzrK7a89GIVsmLXsS9Fr0UvBQIERVMINLkS2VBcglwVcQL/
+        e0H5IQVNgV54mNkZzCz3pQAQVosdCGUkqz648rNer7G2z3fL8ITq+X6lvn7x/SHhN/d9LxZZQftf
+        qPii+qCoDw7Zkj/RKqJkzK7Lzc3N7WazqeuR6Emjy7IucLmisq7qVVndltXHs9CQVZjEDn4UAAAv
+        45sjeo1PYgfV4oL0mJLsUOyuQwAiksuIkCnZxNKzWEykIs/ox9T35gCa/HuG9IgOmXyC1naGAaUy
+        QGwwfnrwD/7O4GXSyN8IbBC6gdO7uXHEdkgy9/KDc2f8eE3qqAuR9unMX/HWeptME1Em8jlVYgpi
+        ZI8FwM9xI8OrkiJE6gM3TI/os+FyebIT0xfMyNWZZGLpJrxeL95wazSytC7NNiqUVAb1pJzWLwdt
+        aUYUs85/h3nL+9Tb+u5/7CdCKQyMugkRtVWvC09jEfOB/mvsuuMxsEiHxNg3rfUdxhDt6Uba0Gz1
+        ulJ6u632ojgWfwAAAP//AwCOwDMjLAMAAA==
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8f016eb76b71ac9a-YYZ
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Wed, 11 Dec 2024 00:42:03 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=r.xSSsYQNFPvMiizFSvjQiecNA6Q1wQa0VR1YElfXi4-1733877723-1.0.1.1-GVW0i7wrpHCQSY5eXu7sIQgxYWl6jfeSordQ7JFxV3lO6UfFhwxRT92bBP4DfnrSYpBpRw3k4aONAURyvKctiQ;
+        path=/; expires=Wed, 11-Dec-24 01:12:03 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=CQJVOdASzL9ency5_q6SDaInTsvpjA240cIxf.AUwXM-1733877723385-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-organization:
+      - wandb
+      openai-processing-ms:
+      - '523'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '30000000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '29999979'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_c9c57cfa6f37a99aaf0abac013237ed6
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/integrations/openai/test_autopatch.py b/tests/integrations/openai/test_autopatch.py
new file mode 100644
index 00000000000..2c2f5201d3f
--- /dev/null
+++ b/tests/integrations/openai/test_autopatch.py
@@ -0,0 +1,116 @@
+# This is included here for convenience.  Instead of creating a dummy API, we can test
+# autopatching against the actual OpenAI API.
+
+from typing import Any
+
+import pytest
+from openai import OpenAI
+
+from weave.integrations.openai import openai_sdk
+from weave.trace.autopatch import AutopatchSettings, IntegrationSettings, OpSettings
+
+
+@pytest.mark.skip_clickhouse_client  # TODO:VCR recording does not seem to allow us to make requests to the clickhouse db in non-recording mode
+@pytest.mark.vcr(
+    filter_headers=["authorization"], allowed_hosts=["api.wandb.ai", "localhost"]
+)
+def test_disabled_integration_doesnt_patch(client_creator):
+    autopatch_settings = AutopatchSettings(
+        openai=IntegrationSettings(enabled=False),
+    )
+
+    with client_creator(autopatch_settings=autopatch_settings) as client:
+        oaiclient = OpenAI()
+        oaiclient.chat.completions.create(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "tell me a joke"}],
+        )
+
+        calls = list(client.get_calls())
+        assert len(calls) == 0
+
+
+@pytest.mark.skip_clickhouse_client  # TODO:VCR recording does not seem to allow us to make requests to the clickhouse db in non-recording mode
+@pytest.mark.vcr(
+    filter_headers=["authorization"], allowed_hosts=["api.wandb.ai", "localhost"]
+)
+def test_enabled_integration_patches(client_creator):
+    autopatch_settings = AutopatchSettings(
+        openai=IntegrationSettings(enabled=True),
+    )
+
+    with client_creator(autopatch_settings=autopatch_settings) as client:
+        oaiclient = OpenAI()
+        oaiclient.chat.completions.create(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "tell me a joke"}],
+        )
+
+        calls = list(client.get_calls())
+        assert len(calls) == 1
+
+
+@pytest.mark.skip_clickhouse_client  # TODO:VCR recording does not seem to allow us to make requests to the clickhouse db in non-recording mode
+@pytest.mark.vcr(
+    filter_headers=["authorization"], allowed_hosts=["api.wandb.ai", "localhost"]
+)
+def test_passthrough_op_kwargs(client_creator):
+    def redact_inputs(inputs: dict[str, Any]) -> dict[str, Any]:
+        return dict.fromkeys(inputs, "REDACTED")
+
+    autopatch_settings = AutopatchSettings(
+        openai=IntegrationSettings(
+            op_settings=OpSettings(
+                postprocess_inputs=redact_inputs,
+            )
+        )
+    )
+
+    # Explicitly reset the patcher here to pretend like we're starting fresh.  We need
+    # to do this because `_openai_patcher` is a global variable that is shared across
+    # tests.  If we don't reset it, it will retain the state from the previous test,
+    # which can cause this test to fail.
+    openai_sdk._openai_patcher = None
+
+    with client_creator(autopatch_settings=autopatch_settings) as client:
+        oaiclient = OpenAI()
+        oaiclient.chat.completions.create(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "tell me a joke"}],
+        )
+
+        calls = list(client.get_calls())
+        assert len(calls) == 1
+
+        call = calls[0]
+        assert all(v == "REDACTED" for v in call.inputs.values())
+
+
+@pytest.mark.skip_clickhouse_client  # TODO:VCR recording does not seem to allow us to make requests to the clickhouse db in non-recording mode
+@pytest.mark.vcr(
+    filter_headers=["authorization"], allowed_hosts=["api.wandb.ai", "localhost"]
+)
+def test_configuration_with_dicts(client_creator):
+    def redact_inputs(inputs: dict[str, Any]) -> dict[str, Any]:
+        return dict.fromkeys(inputs, "REDACTED")
+
+    autopatch_settings = {
+        "openai": {
+            "op_settings": {"postprocess_inputs": redact_inputs},
+        }
+    }
+
+    openai_sdk._openai_patcher = None
+
+    with client_creator(autopatch_settings=autopatch_settings) as client:
+        oaiclient = OpenAI()
+        oaiclient.chat.completions.create(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "tell me a joke"}],
+        )
+
+        calls = list(client.get_calls())
+        assert len(calls) == 1
+
+        call = calls[0]
+        assert all(v == "REDACTED" for v in call.inputs.values())
diff --git a/weave/integrations/openai/openai_sdk.py b/weave/integrations/openai/openai_sdk.py
index 7814700d4d3..a1e3a9b5831 100644
--- a/weave/integrations/openai/openai_sdk.py
+++ b/weave/integrations/openai/openai_sdk.py
@@ -1,15 +1,20 @@
+from __future__ import annotations
+
 import importlib
 from functools import wraps
-from typing import TYPE_CHECKING, Any, Callable, Optional
+from typing import TYPE_CHECKING, Any, Callable
 
 import weave
+from weave.trace.autopatch import IntegrationSettings, OpSettings
 from weave.trace.op import Op, ProcessedInputs
 from weave.trace.op_extensions.accumulator import add_accumulator
-from weave.trace.patcher import MultiPatcher, SymbolPatcher
+from weave.trace.patcher import MultiPatcher, NoOpPatcher, SymbolPatcher
 
 if TYPE_CHECKING:
     from openai.types.chat import ChatCompletionChunk
 
+_openai_patcher: MultiPatcher | None = None
+
 
 def maybe_unwrap_api_response(value: Any) -> Any:
     """If the caller requests a raw response, we unwrap the APIResponse object.
@@ -43,9 +48,7 @@ def maybe_unwrap_api_response(value: Any) -> Any:
     return value
 
 
-def openai_on_finish_post_processor(
-    value: Optional["ChatCompletionChunk"],
-) -> Optional[dict]:
+def openai_on_finish_post_processor(value: ChatCompletionChunk | None) -> dict | None:
     from openai.types.chat import ChatCompletion, ChatCompletionChunk
     from openai.types.chat.chat_completion_chunk import (
         ChoiceDeltaFunctionCall,
@@ -60,8 +63,8 @@ def openai_on_finish_post_processor(
     value = maybe_unwrap_api_response(value)
 
     def _get_function_call(
-        function_call: Optional[ChoiceDeltaFunctionCall],
-    ) -> Optional[FunctionCall]:
+        function_call: ChoiceDeltaFunctionCall | None,
+    ) -> FunctionCall | None:
         if function_call is None:
             return function_call
         if isinstance(function_call, ChoiceDeltaFunctionCall):
@@ -73,8 +76,8 @@ def _get_function_call(
             return None
 
     def _get_tool_calls(
-        tool_calls: Optional[list[ChoiceDeltaToolCall]],
-    ) -> Optional[list[ChatCompletionMessageToolCall]]:
+        tool_calls: list[ChoiceDeltaToolCall] | None,
+    ) -> list[ChatCompletionMessageToolCall] | None:
         if tool_calls is None:
             return tool_calls
 
@@ -128,10 +131,10 @@ def _get_tool_calls(
 
 
 def openai_accumulator(
-    acc: Optional["ChatCompletionChunk"],
-    value: "ChatCompletionChunk",
+    acc: ChatCompletionChunk | None,
+    value: ChatCompletionChunk,
     skip_last: bool = False,
-) -> "ChatCompletionChunk":
+) -> ChatCompletionChunk:
     from openai.types.chat import ChatCompletionChunk
     from openai.types.chat.chat_completion_chunk import (
         ChoiceDeltaFunctionCall,
@@ -285,7 +288,7 @@ def should_use_accumulator(inputs: dict) -> bool:
 
 def openai_on_input_handler(
     func: Op, args: tuple, kwargs: dict
-) -> Optional[ProcessedInputs]:
+) -> ProcessedInputs | None:
     if len(args) == 2 and isinstance(args[1], weave.EasyPrompt):
         original_args = args
         original_kwargs = kwargs
@@ -305,20 +308,16 @@ def openai_on_input_handler(
     return None
 
 
-def create_wrapper_sync(
-    name: str,
-) -> Callable[[Callable], Callable]:
+def create_wrapper_sync(settings: OpSettings) -> Callable[[Callable], Callable]:
     def wrapper(fn: Callable) -> Callable:
         "We need to do this so we can check if `stream` is used"
 
         def _add_stream_options(fn: Callable) -> Callable:
             @wraps(fn)
             def _wrapper(*args: Any, **kwargs: Any) -> Any:
-                if bool(kwargs.get("stream")) and kwargs.get("stream_options") is None:
+                if kwargs.get("stream") and kwargs.get("stream_options") is None:
                     kwargs["stream_options"] = {"include_usage": True}
-                return fn(
-                    *args, **kwargs
-                )  # This is where the final execution of fn is happening.
+                return fn(*args, **kwargs)
 
             return _wrapper
 
@@ -327,8 +326,8 @@ def _openai_stream_options_is_set(inputs: dict) -> bool:
                 return True
             return False
 
-        op = weave.op()(_add_stream_options(fn))
-        op.name = name  # type: ignore
+        op_kwargs = settings.model_dump()
+        op = weave.op(_add_stream_options(fn), **op_kwargs)
         op._set_on_input_handler(openai_on_input_handler)
         return add_accumulator(
             op,  # type: ignore
@@ -345,16 +344,14 @@ def _openai_stream_options_is_set(inputs: dict) -> bool:
 # Surprisingly, the async `client.chat.completions.create` does not pass
 # `inspect.iscoroutinefunction`, so we can't dispatch on it and must write
 # it manually here...
-def create_wrapper_async(
-    name: str,
-) -> Callable[[Callable], Callable]:
+def create_wrapper_async(settings: OpSettings) -> Callable[[Callable], Callable]:
     def wrapper(fn: Callable) -> Callable:
         "We need to do this so we can check if `stream` is used"
 
         def _add_stream_options(fn: Callable) -> Callable:
             @wraps(fn)
             async def _wrapper(*args: Any, **kwargs: Any) -> Any:
-                if bool(kwargs.get("stream")) and kwargs.get("stream_options") is None:
+                if kwargs.get("stream") and kwargs.get("stream_options") is None:
                     kwargs["stream_options"] = {"include_usage": True}
                 return await fn(*args, **kwargs)
 
@@ -365,8 +362,8 @@ def _openai_stream_options_is_set(inputs: dict) -> bool:
                 return True
             return False
 
-        op = weave.op()(_add_stream_options(fn))
-        op.name = name  # type: ignore
+        op_kwargs = settings.model_dump()
+        op = weave.op(_add_stream_options(fn), **op_kwargs)
         op._set_on_input_handler(openai_on_input_handler)
         return add_accumulator(
             op,  # type: ignore
@@ -380,28 +377,61 @@ def _openai_stream_options_is_set(inputs: dict) -> bool:
     return wrapper
 
 
-symbol_patchers = [
-    # Patch the Completions.create method
-    SymbolPatcher(
-        lambda: importlib.import_module("openai.resources.chat.completions"),
-        "Completions.create",
-        create_wrapper_sync(name="openai.chat.completions.create"),
-    ),
-    SymbolPatcher(
-        lambda: importlib.import_module("openai.resources.chat.completions"),
-        "AsyncCompletions.create",
-        create_wrapper_async(name="openai.chat.completions.create"),
-    ),
-    SymbolPatcher(
-        lambda: importlib.import_module("openai.resources.beta.chat.completions"),
-        "Completions.parse",
-        create_wrapper_sync(name="openai.beta.chat.completions.parse"),
-    ),
-    SymbolPatcher(
-        lambda: importlib.import_module("openai.resources.beta.chat.completions"),
-        "AsyncCompletions.parse",
-        create_wrapper_async(name="openai.beta.chat.completions.parse"),
-    ),
-]
-
-openai_patcher = MultiPatcher(symbol_patchers)  # type: ignore
+def get_openai_patcher(
+    settings: IntegrationSettings | None = None,
+) -> MultiPatcher | NoOpPatcher:
+    if settings is None:
+        settings = IntegrationSettings()
+
+    if not settings.enabled:
+        return NoOpPatcher()
+
+    global _openai_patcher
+    if _openai_patcher is not None:
+        return _openai_patcher
+
+    base = settings.op_settings
+
+    completions_create_settings = base.model_copy(
+        update={"name": base.name or "openai.chat.completions.create"}
+    )
+    async_completions_create_settings = base.model_copy(
+        update={"name": base.name or "openai.chat.completions.create"}
+    )
+    completions_parse_settings = base.model_copy(
+        update={"name": base.name or "openai.beta.chat.completions.parse"}
+    )
+    async_completions_parse_settings = base.model_copy(
+        update={"name": base.name or "openai.beta.chat.completions.parse"}
+    )
+
+    _openai_patcher = MultiPatcher(
+        [
+            SymbolPatcher(
+                lambda: importlib.import_module("openai.resources.chat.completions"),
+                "Completions.create",
+                create_wrapper_sync(settings=completions_create_settings),
+            ),
+            SymbolPatcher(
+                lambda: importlib.import_module("openai.resources.chat.completions"),
+                "AsyncCompletions.create",
+                create_wrapper_async(settings=async_completions_create_settings),
+            ),
+            SymbolPatcher(
+                lambda: importlib.import_module(
+                    "openai.resources.beta.chat.completions"
+                ),
+                "Completions.parse",
+                create_wrapper_sync(settings=completions_parse_settings),
+            ),
+            SymbolPatcher(
+                lambda: importlib.import_module(
+                    "openai.resources.beta.chat.completions"
+                ),
+                "AsyncCompletions.parse",
+                create_wrapper_async(settings=async_completions_parse_settings),
+            ),
+        ]
+    )
+
+    return _openai_patcher
diff --git a/weave/scorers/llm_utils.py b/weave/scorers/llm_utils.py
index 68ae2ccb366..eef6f018b0f 100644
--- a/weave/scorers/llm_utils.py
+++ b/weave/scorers/llm_utils.py
@@ -2,10 +2,6 @@
 
 from typing import TYPE_CHECKING, Any, Union
 
-from weave.trace.autopatch import autopatch
-
-autopatch()  # ensure both weave patching and instructor patching are applied
-
 OPENAI_DEFAULT_MODEL = "gpt-4o"
 OPENAI_DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"
 OPENAI_DEFAULT_MODERATION_MODEL = "text-moderation-latest"
diff --git a/weave/trace/api.py b/weave/trace/api.py
index ee8131b0875..294308cbb67 100644
--- a/weave/trace/api.py
+++ b/weave/trace/api.py
@@ -13,6 +13,7 @@
 # There is probably a better place for this, but including here for now to get the fix in.
 from weave import type_handlers  # noqa: F401
 from weave.trace import urls, util, weave_client, weave_init
+from weave.trace.autopatch import AutopatchSettings
 from weave.trace.constants import TRACE_OBJECT_EMOJI
 from weave.trace.context import call_context
 from weave.trace.context import weave_client_context as weave_client_context
@@ -32,6 +33,7 @@ def init(
     project_name: str,
     *,
     settings: UserSettings | dict[str, Any] | None = None,
+    autopatch_settings: AutopatchSettings | None = None,
 ) -> weave_client.WeaveClient:
     """Initialize weave tracking, logging to a wandb project.
 
@@ -52,7 +54,12 @@ def init(
     if should_disable_weave():
         return weave_init.init_weave_disabled().client
 
-    return weave_init.init_weave(project_name).client
+    initialized_client = weave_init.init_weave(
+        project_name,
+        autopatch_settings=autopatch_settings,
+    )
+
+    return initialized_client.client
 
 
 @contextlib.contextmanager
diff --git a/weave/trace/autopatch.py b/weave/trace/autopatch.py
index 3a5dca14556..0619194a224 100644
--- a/weave/trace/autopatch.py
+++ b/weave/trace/autopatch.py
@@ -4,8 +4,54 @@
 check if libraries are installed and imported and patch in the case that they are.
 """
 
+from typing import Any, Callable, Optional, Union
 
-def autopatch() -> None:
+from pydantic import BaseModel, Field, validate_call
+
+from weave.trace.weave_client import Call
+
+
+class OpSettings(BaseModel):
+    """Op settings for a specific integration.
+    These currently subset the `op` decorator args to provide a consistent interface
+    when working with auto-patched functions.  See the `op` decorator for more details."""
+
+    name: Optional[str] = None
+    call_display_name: Optional[Union[str, Callable[[Call], str]]] = None
+    postprocess_inputs: Optional[Callable[[dict[str, Any]], dict[str, Any]]] = None
+    postprocess_output: Optional[Callable[[Any], Any]] = None
+
+
+class IntegrationSettings(BaseModel):
+    """Configuration for a specific integration."""
+
+    enabled: bool = True
+    op_settings: OpSettings = Field(default_factory=OpSettings)
+
+
+class AutopatchSettings(BaseModel):
+    """Settings for auto-patching integrations."""
+
+    # These will be uncommented as we add support for more integrations.  Note that
+
+    # anthropic: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # cerebras: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # cohere: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # dspy: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # google_ai_studio: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # groq: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # instructor: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # langchain: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # litellm: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # llamaindex: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # mistral: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # notdiamond: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    openai: IntegrationSettings = Field(default_factory=IntegrationSettings)
+    # vertexai: IntegrationSettings = Field(default_factory=IntegrationSettings)
+
+
+@validate_call
+def autopatch(settings: Optional[AutopatchSettings] = None) -> None:
     from weave.integrations.anthropic.anthropic_sdk import anthropic_patcher
     from weave.integrations.cerebras.cerebras_sdk import cerebras_patcher
     from weave.integrations.cohere.cohere_sdk import cohere_patcher
@@ -20,10 +66,13 @@ def autopatch() -> None:
     from weave.integrations.llamaindex.llamaindex import llamaindex_patcher
     from weave.integrations.mistral import mistral_patcher
     from weave.integrations.notdiamond.tracing import notdiamond_patcher
-    from weave.integrations.openai.openai_sdk import openai_patcher
+    from weave.integrations.openai.openai_sdk import get_openai_patcher
     from weave.integrations.vertexai.vertexai_sdk import vertexai_patcher
 
-    openai_patcher.attempt_patch()
+    if settings is None:
+        settings = AutopatchSettings()
+
+    get_openai_patcher(settings.openai).attempt_patch()
     mistral_patcher.attempt_patch()
     litellm_patcher.attempt_patch()
     llamaindex_patcher.attempt_patch()
@@ -54,10 +103,10 @@ def reset_autopatch() -> None:
     from weave.integrations.llamaindex.llamaindex import llamaindex_patcher
     from weave.integrations.mistral import mistral_patcher
     from weave.integrations.notdiamond.tracing import notdiamond_patcher
-    from weave.integrations.openai.openai_sdk import openai_patcher
+    from weave.integrations.openai.openai_sdk import get_openai_patcher
     from weave.integrations.vertexai.vertexai_sdk import vertexai_patcher
 
-    openai_patcher.undo_patch()
+    get_openai_patcher().undo_patch()
     mistral_patcher.undo_patch()
     litellm_patcher.undo_patch()
     llamaindex_patcher.undo_patch()
diff --git a/weave/trace/patcher.py b/weave/trace/patcher.py
index 1567c4e2bb9..c1d0d653ffa 100644
--- a/weave/trace/patcher.py
+++ b/weave/trace/patcher.py
@@ -17,6 +17,14 @@ def undo_patch(self) -> bool:
         raise NotImplementedError()
 
 
+class NoOpPatcher(Patcher):
+    def attempt_patch(self) -> bool:
+        return True
+
+    def undo_patch(self) -> bool:
+        return True
+
+
 class MultiPatcher(Patcher):
     def __init__(self, patchers: Sequence[Patcher]) -> None:
         self.patchers = patchers
diff --git a/weave/trace/weave_init.py b/weave/trace/weave_init.py
index 563dcbdaed4..f51d42d5018 100644
--- a/weave/trace/weave_init.py
+++ b/weave/trace/weave_init.py
@@ -63,7 +63,9 @@ def get_entity_project_from_project_name(project_name: str) -> tuple[str, str]:
 
 
 def init_weave(
-    project_name: str, ensure_project_exists: bool = True
+    project_name: str,
+    ensure_project_exists: bool = True,
+    autopatch_settings: autopatch.AutopatchSettings | None = None,
 ) -> InitializedClient:
     global _current_inited_client
     if _current_inited_client is not None:
@@ -120,7 +122,7 @@ def init_weave(
     # autopatching is only supported for the wandb client, because OpenAI calls are not
     # logged in local mode currently. When that's fixed, this autopatch call can be
     # moved to InitializedClient.__init__
-    autopatch.autopatch()
+    autopatch.autopatch(autopatch_settings)
 
     username = get_username()
     try:

From 16e47c3a8d804db1f7c8c80fe53b5b58082a6757 Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Wed, 11 Dec 2024 16:19:24 -0600
Subject: [PATCH 40/52] chore(ui): update UUID dependency to v11 (latest)
 (#3208)

---
 weave-js/package.json |  3 +--
 weave-js/yarn.lock    | 15 +++++----------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/weave-js/package.json b/weave-js/package.json
index cb57125143a..1f551021ed4 100644
--- a/weave-js/package.json
+++ b/weave-js/package.json
@@ -192,7 +192,6 @@
     "@types/react-virtualized-auto-sizer": "^1.0.0",
     "@types/safe-json-stringify": "^1.1.2",
     "@types/styled-components": "^5.1.26",
-    "@types/uuid": "^9.0.1",
     "@types/wavesurfer.js": "^2.0.0",
     "@types/zen-observable": "^0.8.3",
     "@typescript-eslint/eslint-plugin": "5.35.1",
@@ -237,7 +236,7 @@
     "tslint-config-prettier": "^1.18.0",
     "tslint-plugin-prettier": "^2.3.0",
     "typescript": "4.7.4",
-    "uuid": "^9.0.0",
+    "uuid": "^11.0.3",
     "vite": "5.2.9",
     "vitest": "^1.6.0"
   },
diff --git a/weave-js/yarn.lock b/weave-js/yarn.lock
index c7f9379e32a..6a5ec14e872 100644
--- a/weave-js/yarn.lock
+++ b/weave-js/yarn.lock
@@ -4776,11 +4776,6 @@
   resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.7.tgz#5b06ad6894b236a1d2bd6b2f07850ca5c59cf4d6"
   integrity sha512-cputDpIbFgLUaGQn6Vqg3/YsJwxUwHLO13v3i5ouxT4lat0khip9AEWxtERujXV9wxIB1EyF97BSJFt6vpdI8g==
 
-"@types/uuid@^9.0.1":
-  version "9.0.2"
-  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.2.tgz#ede1d1b1e451548d44919dc226253e32a6952c4b"
-  integrity sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==
-
 "@types/wavesurfer.js@^2.0.0":
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/@types/wavesurfer.js/-/wavesurfer.js-2.0.2.tgz#b98a4d57ca24ee2028ae6dd5c2208b568bb73842"
@@ -15032,6 +15027,11 @@ util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1:
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==
 
+uuid@^11.0.3:
+  version "11.0.3"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-11.0.3.tgz#248451cac9d1a4a4128033e765d137e2b2c49a3d"
+  integrity sha512-d0z310fCWv5dJwnX1Y/MncBAqGMKEzlBb1AOf7z9K8ALnd0utBX/msg/fA0+sbyN1ihbMsLhrBlnl1ak7Wa0rg==
+
 uuid@^2.0.2:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
@@ -15042,11 +15042,6 @@ uuid@^3.0.0, uuid@^3.4.0:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
 
-uuid@^9.0.0:
-  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"
-  integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==
-
 uvu@^0.5.0:
   version "0.5.6"
   resolved "https://registry.yarnpkg.com/uvu/-/uvu-0.5.6.tgz#2754ca20bcb0bb59b64e9985e84d2e81058502df"

From 444c04dcdeb965b531d7508f5e82af56cfcb00f2 Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Wed, 11 Dec 2024 18:50:14 -0600
Subject: [PATCH 41/52] chore(ui): remove some unused code (#3157)

---
 .../PagePanelComponents/Home/Browse3.tsx      | 138 +-----------------
 1 file changed, 1 insertion(+), 137 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
index 3517f4d3b9c..761fd536930 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
@@ -1,15 +1,5 @@
 import {ApolloProvider} from '@apollo/client';
-import {Home} from '@mui/icons-material';
-import {
-  AppBar,
-  Box,
-  Breadcrumbs,
-  Drawer,
-  IconButton,
-  Link as MaterialLink,
-  Toolbar,
-  Typography,
-} from '@mui/material';
+import {Box, Drawer} from '@mui/material';
 import {
   GridColumnVisibilityModel,
   GridFilterModel,
@@ -21,9 +11,7 @@ import {LicenseInfo} from '@mui/x-license';
 import {makeGorillaApolloClient} from '@wandb/weave/apollo';
 import {EVALUATE_OP_NAME_POST_PYDANTIC} from '@wandb/weave/components/PagePanelComponents/Home/Browse3/pages/common/heuristics';
 import {opVersionKeyToRefUri} from '@wandb/weave/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/utilities';
-import _ from 'lodash';
 import React, {
-  ComponentProps,
   FC,
   useCallback,
   useEffect,
@@ -33,7 +21,6 @@ import React, {
 } from 'react';
 import useMousetrap from 'react-hook-mousetrap';
 import {
-  Link as RouterLink,
   Redirect,
   Route,
   Switch,
@@ -199,7 +186,6 @@ export const Browse3: FC<{
               `/${URL_BROWSE3}`,
             ]}>
             <Browse3Mounted
-              hideHeader={props.hideHeader}
               headerOffset={props.headerOffset}
               navigateAwayFromProject={props.navigateAwayFromProject}
             />
@@ -211,7 +197,6 @@ export const Browse3: FC<{
 };
 
 const Browse3Mounted: FC<{
-  hideHeader?: boolean;
   headerOffset?: number;
   navigateAwayFromProject?: () => void;
 }> = props => {
@@ -225,37 +210,6 @@ const Browse3Mounted: FC<{
         overflow: 'auto',
         flexDirection: 'column',
       }}>
-      {!props.hideHeader && (
-        <AppBar
-          sx={{
-            zIndex: theme => theme.zIndex.drawer + 1,
-            height: '60px',
-            flex: '0 0 auto',
-            position: 'static',
-          }}>
-          <Toolbar
-            sx={{
-              backgroundColor: '#1976d2',
-              minHeight: '30px',
-            }}>
-            <IconButton
-              component={RouterLink}
-              to={`/`}
-              sx={{
-                color: theme =>
-                  theme.palette.getContrastText(theme.palette.primary.main),
-                '&:hover': {
-                  color: theme =>
-                    theme.palette.getContrastText(theme.palette.primary.dark),
-                },
-                marginRight: theme => theme.spacing(2),
-              }}>
-              <Home />
-            </IconButton>
-            <Browse3Breadcrumbs />
-          </Toolbar>
-        </AppBar>
-      )}
       <Switch>
         <Route path={baseRouter.projectUrl(':entity', ':project')} exact>
           <ProjectRedirect />
@@ -1050,20 +1004,6 @@ const ComparePageBinding = () => {
   return <ComparePage entity={params.entity} project={params.project} />;
 };
 
-const AppBarLink = (props: ComponentProps<typeof RouterLink>) => (
-  <MaterialLink
-    sx={{
-      color: theme => theme.palette.getContrastText(theme.palette.primary.main),
-      '&:hover': {
-        color: theme =>
-          theme.palette.getContrastText(theme.palette.primary.dark),
-      },
-    }}
-    {...props}
-    component={RouterLink}
-  />
-);
-
 const PlaygroundPageBinding = () => {
   const params = useParamsDecoded<Browse3TabItemParams>();
   return (
@@ -1074,79 +1014,3 @@ const PlaygroundPageBinding = () => {
     />
   );
 };
-
-const Browse3Breadcrumbs: FC = props => {
-  const params = useParamsDecoded<Browse3Params>();
-  const query = useURLSearchParamsDict();
-  const filePathParts = query.path?.split('/') ?? [];
-  const refFields = query.extra?.split('/') ?? [];
-
-  return (
-    <Breadcrumbs>
-      {params.entity && (
-        <AppBarLink to={`/${URL_BROWSE3}/${params.entity}`}>
-          {params.entity}
-        </AppBarLink>
-      )}
-      {params.project && (
-        <AppBarLink to={`/${URL_BROWSE3}/${params.entity}/${params.project}`}>
-          {params.project}
-        </AppBarLink>
-      )}
-      {params.tab && (
-        <AppBarLink
-          to={`/${URL_BROWSE3}/${params.entity}/${params.project}/${params.tab}`}>
-          {params.tab}
-        </AppBarLink>
-      )}
-      {params.itemName && (
-        <AppBarLink
-          to={`/${URL_BROWSE3}/${params.entity}/${params.project}/${params.tab}/${params.itemName}`}>
-          {params.itemName}
-        </AppBarLink>
-      )}
-      {params.version && (
-        <AppBarLink
-          to={`/${URL_BROWSE3}/${params.entity}/${params.project}/${params.tab}/${params.itemName}/versions/${params.version}`}>
-          {params.version}
-        </AppBarLink>
-      )}
-      {filePathParts.map((part, idx) => (
-        <AppBarLink
-          key={idx}
-          to={`/${URL_BROWSE3}/${params.entity}/${params.project}/${
-            params.tab
-          }/${params.itemName}/versions/${
-            params.version
-          }?path=${encodeURIComponent(
-            filePathParts.slice(0, idx + 1).join('/')
-          )}`}>
-          {part}
-        </AppBarLink>
-      ))}
-      {_.range(0, refFields.length, 2).map(idx => (
-        <React.Fragment key={idx}>
-          <Typography
-            sx={{
-              color: theme =>
-                theme.palette.getContrastText(theme.palette.primary.main),
-            }}>
-            {refFields[idx]}
-          </Typography>
-          <AppBarLink
-            to={`/${URL_BROWSE3}/${params.entity}/${params.project}/${
-              params.tab
-            }/${params.itemName}/versions/${
-              params.version
-            }?path=${encodeURIComponent(
-              filePathParts.join('/')
-            )}&extra=${encodeURIComponent(
-              refFields.slice(0, idx + 2).join('/')
-            )}`}>
-            {refFields[idx + 1]}
-          </AppBarLink>
-        </React.Fragment>
-      ))}
-    </Breadcrumbs>
-  );
-};

From 96d1d0d0f48cd571ae7b7737de2fd58d663588d9 Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Wed, 11 Dec 2024 17:55:26 -0800
Subject: [PATCH 42/52] chore(ui): Create scorer drawer style + small
 annotation drawer style tweaks (#3186)

---
 .../StructuredFeedback/HumanAnnotation.tsx    |   4 +-
 .../ScorersPage/AnnotationScorerForm.tsx      |   9 +-
 .../pages/ScorersPage/FormComponents.tsx      |   2 +-
 .../pages/ScorersPage/NewScorerDrawer.tsx     |  25 +++-
 .../pages/ScorersPage/ZodSchemaForm.tsx       | 133 +++++++++++-------
 5 files changed, 111 insertions(+), 62 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/HumanAnnotation.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/HumanAnnotation.tsx
index 7facffe9556..2821c02affa 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/HumanAnnotation.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/HumanAnnotation.tsx
@@ -415,7 +415,7 @@ export const TextFeedbackColumn = ({
         placeholder=""
       />
       {maxLength && (
-        <div className="mb-1 text-xs text-moon-500">
+        <div className="mb-1 mt-4 text-xs text-moon-500">
           {`Maximum characters: ${maxLength}`}
         </div>
       )}
@@ -603,7 +603,7 @@ export const NumericalTextField: React.FC<NumericalTextFieldProps> = ({
         errorState={error}
       />
       {(min != null || max != null) && (
-        <div className="mb-1 text-xs text-moon-500">
+        <div className="mb-1 mt-4 text-xs text-moon-500">
           {isInteger ? 'Integer required. ' : ''}
           {min != null && `Min: ${min}`}
           {min != null && max != null && ', '}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/AnnotationScorerForm.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/AnnotationScorerForm.tsx
index 9acbdfe6c2f..a478437facb 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/AnnotationScorerForm.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/AnnotationScorerForm.tsx
@@ -1,5 +1,5 @@
 import {Box} from '@material-ui/core';
-import React, {FC, useCallback, useState} from 'react';
+import React, {FC, useCallback, useEffect, useState} from 'react';
 import {z} from 'zod';
 
 import {createBaseObjectInstance} from '../wfReactInterface/baseObjectClassQuery';
@@ -28,7 +28,7 @@ const AnnotationScorerFormSchema = z.object({
     }),
     z.object({
       type: z.literal('String'),
-      'Max length': z.number().optional(),
+      'Maximum length': z.number().optional(),
     }),
     z.object({
       type: z.literal('Select'),
@@ -45,6 +45,9 @@ export const AnnotationScorerForm: FC<
   ScorerFormProps<z.infer<typeof AnnotationScorerFormSchema>>
 > = ({data, onDataChange}) => {
   const [config, setConfig] = useState(data ?? DEFAULT_STATE);
+  useEffect(() => {
+    setConfig(data ?? DEFAULT_STATE);
+  }, [data]);
   const [isValid, setIsValid] = useState(false);
 
   const handleConfigChange = useCallback(
@@ -113,7 +116,7 @@ function convertTypeExtrasToJsonSchema(
   const typeSchema = obj.Type;
   const typeExtras: Record<string, any> = {};
   if (typeSchema.type === 'String') {
-    typeExtras.maxLength = typeSchema['Max length'];
+    typeExtras.maxLength = typeSchema['Maximum length'];
   } else if (typeSchema.type === 'Integer' || typeSchema.type === 'Number') {
     typeExtras.minimum = typeSchema.Minimum;
     typeExtras.maximum = typeSchema.Maximum;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/FormComponents.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/FormComponents.tsx
index 2716bfbfa81..250c896cfea 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/FormComponents.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/FormComponents.tsx
@@ -3,7 +3,7 @@ import {Select} from '@wandb/weave/components/Form/Select';
 import {TextField} from '@wandb/weave/components/Form/TextField';
 import React from 'react';
 
-export const GAP_BETWEEN_ITEMS_PX = 10;
+export const GAP_BETWEEN_ITEMS_PX = 16;
 export const GAP_BETWEEN_LABEL_AND_FIELD_PX = 10;
 
 type AutocompleteWithLabelType<Option = any> = (
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/NewScorerDrawer.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/NewScorerDrawer.tsx
index 80f4cedd3f6..96621b142f0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/NewScorerDrawer.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/NewScorerDrawer.tsx
@@ -138,11 +138,16 @@ export const NewScorerDrawer: FC<NewScorerDrawerProps> = ({
     );
   }, [showRunnableUI]);
 
+  const handleClose = () => {
+    setFormData(null);
+    onClose();
+  };
+
   return (
     <SaveableDrawer
       open={open}
       title="Create scorer"
-      onClose={onClose}
+      onClose={handleClose}
       onSave={onSave}
       saveDisabled={!isFormValid}>
       <AutocompleteWithLabel
@@ -208,12 +213,22 @@ export const SaveableDrawer: FC<SaveableDrawerProps> = ({
           sx={{
             flex: '0 0 auto',
             borderBottom: '1px solid #e0e0e0',
-            p: '10px',
+            px: '24px',
+            py: '20px',
             display: 'flex',
             fontWeight: 600,
+            fontSize: '24px',
+            lineHeight: '40px',
           }}>
           <Box sx={{flexGrow: 1}}>{title}</Box>
-          <Button size="small" variant="quiet" icon="close" onClick={onClose} />
+          <Box>
+            <Button
+              size="large"
+              variant="quiet"
+              icon="close"
+              onClick={onClose}
+            />
+          </Box>
         </Box>
 
         <Box
@@ -230,11 +245,13 @@ export const SaveableDrawer: FC<SaveableDrawerProps> = ({
             display: 'flex',
             flex: '0 0 auto',
             borderTop: '1px solid #e0e0e0',
-            p: '10px',
+            px: '24px',
+            py: '20px',
           }}>
           <Button
             onClick={onSave}
             color="primary"
+            size="large"
             disabled={saveDisabled}
             className="w-full">
             Create scorer
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/ZodSchemaForm.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/ZodSchemaForm.tsx
index 473a2ebcba7..94106a7a3b2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/ZodSchemaForm.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/ZodSchemaForm.tsx
@@ -163,8 +163,13 @@ const NestedForm: React.FC<{
   hideLabel,
   autoFocus,
 }) => {
-  const currentPath = [...path, keyName];
-  const currentValue = getNestedValue(config, currentPath);
+  const currentPath = useMemo(() => [...path, keyName], [path, keyName]);
+  const [currentValue, setCurrentValue] = useState(
+    getNestedValue(config, currentPath)
+  );
+  useEffect(() => {
+    setCurrentValue(getNestedValue(config, currentPath));
+  }, [config, currentPath]);
 
   const unwrappedSchema = unwrapSchema(fieldSchema);
 
@@ -289,9 +294,11 @@ const NestedForm: React.FC<{
   } else if (isZodType(fieldSchema, s => s instanceof z.ZodBoolean)) {
     fieldType = 'checkbox';
   }
+  const isOptional = fieldSchema instanceof z.ZodOptional;
 
   return (
     <TextFieldWithLabel
+      isOptional={isOptional}
       label={!hideLabel ? keyName : undefined}
       type={fieldType}
       value={currentValue ?? ''}
@@ -345,58 +352,72 @@ const ArrayField: React.FC<{
           <DescriptionTooltip description={fieldDescription} />
         )}
       </Box>
-      {arrayValue.map((item, index) => (
-        <Box
-          key={index}
-          display="flex"
-          flexDirection="column"
-          alignItems="flex-start"
-          style={{
-            width: '100%',
-            gap: 4,
-            alignItems: 'center',
-            height: '35px',
-            marginBottom: '4px',
-          }}>
-          <Box flexGrow={1} width="100%" display="flex" alignItems="center">
-            <Box flexGrow={1}>
-              <NestedForm
-                keyName={`${index}`}
-                fieldSchema={elementSchema}
-                config={{[`${index}`]: item}}
-                setConfig={newItemConfig => {
-                  const newArray = [...arrayValue];
-                  newArray[index] = newItemConfig[`${index}`];
-                  updateConfig(targetPath, newArray, config, setConfig);
-                }}
-                path={[]}
-                hideLabel
-                autoFocus={index === arrayValue.length - 1}
-              />
-            </Box>
-            <Box mb={2} ml={1}>
-              <Button
-                size="small"
-                variant="ghost"
-                icon="delete"
-                tooltip="Remove this entry"
-                disabled={arrayValue.length <= minItems}
-                onClick={() =>
-                  removeArrayItem(targetPath, index, config, setConfig)
-                }
-              />
+      <Box border="1px solid #e0e0e0" borderRadius="4px" p={2}>
+        {arrayValue.map((item, index) => (
+          <Box
+            key={index}
+            display="flex"
+            flexDirection="column"
+            alignItems="flex-start"
+            style={{
+              width: '100%',
+              gap: 4,
+              alignItems: 'center',
+              height: '35px',
+              marginBottom: '16px',
+              marginTop: '8px',
+              marginLeft: '8px',
+            }}>
+            <Box flexGrow={1} width="100%" display="flex" alignItems="center">
+              <Box flexGrow={1}>
+                <NestedForm
+                  keyName={`${index}`}
+                  fieldSchema={elementSchema}
+                  config={{[`${index}`]: item}}
+                  setConfig={newItemConfig => {
+                    const newArray = [...arrayValue];
+                    newArray[index] = newItemConfig[`${index}`];
+                    updateConfig(targetPath, newArray, config, setConfig);
+                  }}
+                  path={[]}
+                  hideLabel
+                  autoFocus={index === arrayValue.length - 1}
+                />
+              </Box>
+              <Box mb={4} ml={4} mr={4}>
+                <Button
+                  size="small"
+                  variant="ghost"
+                  icon="delete"
+                  tooltip="Remove this entry"
+                  disabled={arrayValue.length <= minItems}
+                  onClick={() =>
+                    removeArrayItem(targetPath, index, config, setConfig)
+                  }
+                />
+              </Box>
             </Box>
           </Box>
+        ))}
+        <Box mt={2} style={{width: '100%'}}>
+          <Button
+            variant="secondary"
+            icon="add-new"
+            className="w-full"
+            style={{
+              padding: '4px',
+              width: '100%',
+              marginLeft: '8px',
+              marginRight: '8px',
+              marginBottom: '8px',
+            }}
+            onClick={() =>
+              addArrayItem(targetPath, elementSchema, config, setConfig)
+            }>
+            Add item
+          </Button>
         </Box>
-      ))}
-      <Button
-        variant="secondary"
-        style={{padding: '4px', width: '100%'}}
-        onClick={() =>
-          addArrayItem(targetPath, elementSchema, config, setConfig)
-        }>
-        Add item
-      </Button>
+      </Box>
     </FormControl>
   );
 };
@@ -758,6 +779,7 @@ const LiteralField: React.FC<{
   setConfig,
 }) => {
   const literalValue = unwrappedSchema.value;
+  const isOptional = fieldSchema instanceof z.ZodOptional;
 
   useEffect(() => {
     if (value !== literalValue) {
@@ -765,7 +787,14 @@ const LiteralField: React.FC<{
     }
   }, [value, literalValue, targetPath, config, setConfig]);
 
-  return <TextFieldWithLabel label={keyName} disabled value={literalValue} />;
+  return (
+    <TextFieldWithLabel
+      isOptional={isOptional}
+      label={keyName}
+      disabled
+      value={literalValue}
+    />
+  );
 };
 
 const BooleanField: React.FC<{

From 3acb3d38b8022ba9cd8773df36535a5c5f361e1b Mon Sep 17 00:00:00 2001
From: Griffin Tarpenning <griffin.tarpenning@wandb.com>
Date: Wed, 11 Dec 2024 18:10:49 -0800
Subject: [PATCH 43/52] chore(ui): fix save state in human annotations when
 null or invalid (#3185)

---
 .../StructuredFeedback/HumanAnnotation.tsx    | 50 +++++++++----------
 1 file changed, 23 insertions(+), 27 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/HumanAnnotation.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/HumanAnnotation.tsx
index 2821c02affa..21dedf10ea0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/HumanAnnotation.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/HumanAnnotation.tsx
@@ -53,17 +53,11 @@ export const HumanAnnotationCell: React.FC<HumanAnnotationProps> = props => {
   const feedbackSpecRef = props.hfSpec.ref;
 
   useEffect(() => {
-    if (!props.readOnly) {
-      // We don't need to listen for feedback changes if the cell is editable
-      // it is being controlled by local state
-      return;
-    }
     return getTsClient().registerOnFeedbackListener(
       props.callRef,
       query.refetch
     );
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [props.callRef]);
+  }, [props.callRef, query.refetch, getTsClient]);
 
   useEffect(() => {
     if (foundFeedbackCallRef && props.callRef !== foundFeedbackCallRef) {
@@ -183,13 +177,22 @@ const FeedbackComponentSelector: React.FC<{
   }) => {
     const wrappedOnAddFeedback = useCallback(
       async (value: any) => {
+        if (value == null || value === foundValue || value === '') {
+          // Remove from unsaved changes if value is invalid
+          setUnsavedFeedbackChanges(curr => {
+            const rest = {...curr};
+            delete rest[feedbackSpecRef];
+            return rest;
+          });
+          return true;
+        }
         setUnsavedFeedbackChanges(curr => ({
           ...curr,
           [feedbackSpecRef]: () => onAddFeedback(value),
         }));
         return true;
       },
-      [onAddFeedback, setUnsavedFeedbackChanges, feedbackSpecRef]
+      [onAddFeedback, setUnsavedFeedbackChanges, feedbackSpecRef, foundValue]
     );
 
     switch (type) {
@@ -346,21 +349,10 @@ export const NumericalFeedbackColumn = ({
   focused?: boolean;
   isInteger?: boolean;
 }) => {
-  const debouncedFn = useMemo(
-    () =>
-      _.debounce((val: number | null) => onAddFeedback?.(val), DEBOUNCE_VAL),
-    [onAddFeedback]
-  );
-  useEffect(() => {
-    return () => {
-      debouncedFn.cancel();
-    };
-  }, [debouncedFn]);
-
   return (
     <NumericalTextField
       value={defaultValue}
-      onChange={debouncedFn}
+      onChange={value => onAddFeedback?.(value)}
       min={min}
       max={max}
       isInteger={isInteger}
@@ -446,7 +438,7 @@ export const EnumFeedbackColumn = ({
     }));
     return opts;
   }, [options]);
-  const [value, setValue] = useState<Option | undefined>(undefined);
+  const [value, setValue] = useState<Option | null>(null);
 
   useEffect(() => {
     const found = dropdownOptions.find(option => option.value === defaultValue);
@@ -457,17 +449,19 @@ export const EnumFeedbackColumn = ({
 
   const onValueChange = useCallback(
     (newValue: Option | null) => {
-      if (newValue == null || newValue.value === value?.value) {
-        return;
-      }
       setValue(newValue);
-      onAddFeedback?.(newValue.value);
+      onAddFeedback?.(newValue?.value ?? '');
     },
-    [value?.value, onAddFeedback]
+    [onAddFeedback]
   );
 
   return (
-    <Select options={dropdownOptions} value={value} onChange={onValueChange} />
+    <Select
+      autoFocus={focused}
+      options={dropdownOptions}
+      value={value}
+      onChange={onValueChange}
+    />
   );
 };
 
@@ -573,6 +567,7 @@ export const NumericalTextField: React.FC<NumericalTextFieldProps> = ({
       // If val is null but v isn't empty, there's a format error
       if (val === null) {
         setError(true);
+        onChange(null);
         return;
       }
 
@@ -582,6 +577,7 @@ export const NumericalTextField: React.FC<NumericalTextFieldProps> = ({
         (max != null && parsedVal > max)
       ) {
         setError(true);
+        onChange(null);
         return;
       }
 

From 9bda890bbe28742180c809a2a7a6f59c85e0a36d Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 12 Dec 2024 08:09:06 -0700
Subject: [PATCH 44/52] Remove api_key hardcodes in code examples (#3214)

---
 docs/docs/guides/evaluation/scorers.md        | 2 +-
 docs/docs/guides/integrations/local_models.md | 1 -
 docs/docs/guides/integrations/notdiamond.md   | 1 -
 docs/docs/quickstart.md                       | 2 +-
 docs/docs/tutorial-tracing_2.md               | 2 +-
 5 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/docs/docs/guides/evaluation/scorers.md b/docs/docs/guides/evaluation/scorers.md
index e313dcb5582..233d162b15c 100644
--- a/docs/docs/guides/evaluation/scorers.md
+++ b/docs/docs/guides/evaluation/scorers.md
@@ -455,7 +455,7 @@ In Weave, Scorers are used to evaluate AI outputs and return evaluation metrics.
     from weave.scorers import OpenAIModerationScorer
     from openai import OpenAI
 
-    oai_client = OpenAI(api_key=...) # initialize your LLM client here
+    oai_client = OpenAI() # initialize your LLM client here
 
     scorer = OpenAIModerationScorer(
         client=oai_client,
diff --git a/docs/docs/guides/integrations/local_models.md b/docs/docs/guides/integrations/local_models.md
index 090d22a3f76..2597ad19dd1 100644
--- a/docs/docs/guides/integrations/local_models.md
+++ b/docs/docs/guides/integrations/local_models.md
@@ -14,7 +14,6 @@ First and most important, is the `base_url` change during the `openai.OpenAI()`
 
 ```python
 client = openai.OpenAI(
-    api_key='fake',
     base_url="http://localhost:1234",
 )
 ```
diff --git a/docs/docs/guides/integrations/notdiamond.md b/docs/docs/guides/integrations/notdiamond.md
index 3106ef11f1b..e98a23d1334 100644
--- a/docs/docs/guides/integrations/notdiamond.md
+++ b/docs/docs/guides/integrations/notdiamond.md
@@ -68,7 +68,6 @@ preference_id = train_router(
     response_column="actual",
     language="en",
     maximize=True,
-    api_key=api_key,
 )
 ```
 
diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md
index 59ac16ef236..65bc813af03 100644
--- a/docs/docs/quickstart.md
+++ b/docs/docs/quickstart.md
@@ -50,7 +50,7 @@ _In this example, we're using openai so you will need to add an OpenAI [API key]
     import weave
     from openai import OpenAI
 
-    client = OpenAI(api_key="...")
+    client = OpenAI()
 
     # Weave will track the inputs, outputs and code of this function
     # highlight-next-line
diff --git a/docs/docs/tutorial-tracing_2.md b/docs/docs/tutorial-tracing_2.md
index 719ee99e012..d6e392d9c56 100644
--- a/docs/docs/tutorial-tracing_2.md
+++ b/docs/docs/tutorial-tracing_2.md
@@ -24,7 +24,7 @@ Building on our [basic tracing example](/quickstart), we will now add additional
     import json
     from openai import OpenAI
 
-    client = OpenAI(api_key="...")
+    client = OpenAI()
 
     # highlight-next-line
     @weave.op()

From 968a99dffd39f33b0c2b3fb93f368b607f34d104 Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Thu, 12 Dec 2024 10:02:16 -0600
Subject: [PATCH 45/52] feat(ui): Add error details button (#3215)

---
 weave-js/src/components/ErrorBoundary.tsx |  25 ++++--
 weave-js/src/components/ErrorPanel.tsx    | 105 +++++++++++++++++++++-
 weave-js/src/errors.ts                    |   4 +-
 3 files changed, 123 insertions(+), 11 deletions(-)

diff --git a/weave-js/src/components/ErrorBoundary.tsx b/weave-js/src/components/ErrorBoundary.tsx
index 48b4fcf1a1e..053af08aa97 100644
--- a/weave-js/src/components/ErrorBoundary.tsx
+++ b/weave-js/src/components/ErrorBoundary.tsx
@@ -1,6 +1,7 @@
 import {datadogRum} from '@datadog/browser-rum';
 import * as Sentry from '@sentry/react';
 import React, {Component, ErrorInfo, ReactNode} from 'react';
+import {v7 as uuidv7} from 'uuid';
 
 import {weaveErrorToDDPayload} from '../errors';
 import {ErrorPanel} from './ErrorPanel';
@@ -10,24 +11,31 @@ type Props = {
 };
 
 type State = {
-  hasError: boolean;
+  uuid: string | undefined;
+  timestamp: Date | undefined;
+  error: Error | undefined;
 };
 
 export class ErrorBoundary extends Component<Props, State> {
-  public static getDerivedStateFromError(_: Error): State {
-    return {hasError: true};
+  public static getDerivedStateFromError(error: Error): State {
+    return {uuid: uuidv7(), timestamp: new Date(), error};
   }
   public state: State = {
-    hasError: false,
+    uuid: undefined,
+    timestamp: undefined,
+    error: undefined,
   };
 
   public componentDidCatch(error: Error, errorInfo: ErrorInfo) {
+    const {uuid} = this.state;
     datadogRum.addAction(
       'weave_panel_error_boundary',
-      weaveErrorToDDPayload(error)
+      weaveErrorToDDPayload(error, undefined, uuid)
     );
-
     Sentry.captureException(error, {
+      extra: {
+        uuid,
+      },
       tags: {
         weaveErrorBoundary: 'true',
       },
@@ -35,8 +43,9 @@ export class ErrorBoundary extends Component<Props, State> {
   }
 
   public render() {
-    if (this.state.hasError) {
-      return <ErrorPanel />;
+    const {uuid, timestamp, error} = this.state;
+    if (error != null) {
+      return <ErrorPanel uuid={uuid} timestamp={timestamp} error={error} />;
     }
 
     return this.props.children;
diff --git a/weave-js/src/components/ErrorPanel.tsx b/weave-js/src/components/ErrorPanel.tsx
index cbcfd244ac9..86945922150 100644
--- a/weave-js/src/components/ErrorPanel.tsx
+++ b/weave-js/src/components/ErrorPanel.tsx
@@ -1,7 +1,19 @@
-import React, {forwardRef, useEffect, useRef, useState} from 'react';
+import copyToClipboard from 'copy-to-clipboard';
+import _ from 'lodash';
+import React, {
+  forwardRef,
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+} from 'react';
 import styled from 'styled-components';
 
+import {toast} from '../common/components/elements/Toast';
 import {hexToRGB, MOON_300, MOON_600} from '../common/css/globals.styles';
+import {useViewerInfo} from '../common/hooks/useViewerInfo';
+import {getCookieBool, getFirebaseCookie} from '../common/util/cookie';
+import {Button} from './Button';
 import {Icon} from './Icon';
 import {Tooltip} from './Tooltip';
 
@@ -14,6 +26,11 @@ type ErrorPanelProps = {
   title?: string;
   subtitle?: string;
   subtitle2?: string;
+
+  // These props are for error details object
+  uuid?: string;
+  timestamp?: Date;
+  error?: Error;
 };
 
 export const Centered = styled.div`
@@ -85,11 +102,87 @@ export const ErrorPanelSmall = ({
   );
 };
 
+const getDateObject = (timestamp?: Date): Record<string, any> | null => {
+  if (!timestamp) {
+    return null;
+  }
+  return {
+    // e.g. "2024-12-12T06:10:19.475Z",
+    iso: timestamp.toISOString(),
+    // e.g. "Thursday, December 12, 2024 at 6:10:19 AM Coordinated Universal Time"
+    long: timestamp.toLocaleString('en-US', {
+      weekday: 'long',
+      year: 'numeric', // Full year
+      month: 'long', // Full month name
+      day: 'numeric', // Day of the month
+      hour: 'numeric', // Hour (12-hour or 24-hour depending on locale)
+      minute: 'numeric',
+      second: 'numeric',
+      timeZone: 'UTC', // Ensures it's in UTC
+      timeZoneName: 'long', // Full time zone name
+    }),
+    user: timestamp.toLocaleString('en-US', {
+      dateStyle: 'full',
+      timeStyle: 'full',
+    }),
+  };
+};
+
+const getErrorObject = (error?: Error): Record<string, any> | null => {
+  if (!error) {
+    return null;
+  }
+
+  // Error object properties are not enumerable so we have to copy them manually
+  const stack = (error.stack ?? '').split('\n');
+  return {
+    message: error.message,
+    stack,
+  };
+};
+
 export const ErrorPanelLarge = forwardRef<HTMLDivElement, ErrorPanelProps>(
-  ({title, subtitle, subtitle2}, ref) => {
+  ({title, subtitle, subtitle2, uuid, timestamp, error}, ref) => {
     const titleStr = title ?? DEFAULT_TITLE;
     const subtitleStr = subtitle ?? DEFAULT_SUBTITLE;
     const subtitle2Str = subtitle2 ?? DEFAULT_SUBTITLE2;
+
+    const {userInfo} = useViewerInfo();
+
+    const onClick = useCallback(() => {
+      const betaVersion = getFirebaseCookie('betaVersion');
+      const isUsingAdminPrivileges = getCookieBool('use_admin_privileges');
+      const {location, navigator, screen} = window;
+      const {userAgent, language} = navigator;
+      const details = {
+        uuid,
+        url: location.href,
+        error: getErrorObject(error),
+        timestamp_err: getDateObject(timestamp),
+        timestamp_copied: getDateObject(new Date()),
+        user: _.pick(userInfo, ['id', 'username']), // Skipping teams and admin
+        cookies: {
+          ...(betaVersion && {betaVersion}),
+          ...(isUsingAdminPrivileges && {use_admin_privileges: true}),
+        },
+        browser: {
+          userAgent,
+          language,
+          screenSize: {
+            width: screen.width,
+            height: screen.height,
+          },
+          viewportSize: {
+            width: window.innerWidth,
+            height: window.innerHeight,
+          },
+        },
+      };
+      const detailsText = JSON.stringify(details, null, 2);
+      copyToClipboard(detailsText);
+      toast('Copied to clipboard');
+    }, [uuid, timestamp, error, userInfo]);
+
     return (
       <Large ref={ref}>
         <Circle $size={40} $hoverHighlight={false}>
@@ -98,6 +191,14 @@ export const ErrorPanelLarge = forwardRef<HTMLDivElement, ErrorPanelProps>(
         <Title>{titleStr}</Title>
         <Subtitle>{subtitleStr}</Subtitle>
         <Subtitle>{subtitle2Str}</Subtitle>
+        <Button
+          style={{marginTop: 16}}
+          size="small"
+          variant="secondary"
+          icon="copy"
+          onClick={onClick}>
+          Copy error details
+        </Button>
       </Large>
     );
   }
diff --git a/weave-js/src/errors.ts b/weave-js/src/errors.ts
index daa3e0c97a0..f2582baade8 100644
--- a/weave-js/src/errors.ts
+++ b/weave-js/src/errors.ts
@@ -37,7 +37,8 @@ type DDErrorPayload = {
 
 export const weaveErrorToDDPayload = (
   error: Error,
-  weave?: WeaveApp
+  weave?: WeaveApp,
+  uuid?: string
 ): DDErrorPayload => {
   try {
     return {
@@ -49,6 +50,7 @@ export const weaveErrorToDDPayload = (
       windowLocationURL: trimString(window.location.href),
       weaveContext: weave?.client.debugMeta(),
       isServerError: error instanceof UseNodeValueServerExecutionError,
+      ...(uuid != null && {uuid}),
     };
   } catch (e) {
     // If we fail to serialize the error, just return an empty object.

From a61a50ebf689fbe4c2d390b457212cf2fe2c393a Mon Sep 17 00:00:00 2001
From: Martin Mark <martin.mark@wandb.com>
Date: Thu, 12 Dec 2024 12:58:06 -0500
Subject: [PATCH 46/52] Update CustomGridTreeDataGroupingCell.tsx (#3219)

---
 .../Browse3/pages/CallPage/CustomGridTreeDataGroupingCell.tsx   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CustomGridTreeDataGroupingCell.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CustomGridTreeDataGroupingCell.tsx
index 4db1b6334ad..05a71c7d4ab 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CustomGridTreeDataGroupingCell.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CustomGridTreeDataGroupingCell.tsx
@@ -173,7 +173,7 @@ export const CustomGridTreeDataGroupingCell: FC<
             <Box
               sx={{
                 width: '100%',
-                height: '100%',
+                height: '34px',
                 borderBottom: BORDER_STYLE,
               }}></Box>
             <Box sx={{width: '100%', height: '100%'}}></Box>

From 99d07b4e657122f81c1c0b0b4edb74e1437eada4 Mon Sep 17 00:00:00 2001
From: Martin Mark <martin.mark@wandb.com>
Date: Thu, 12 Dec 2024 12:58:14 -0500
Subject: [PATCH 47/52] Updated Playground editor component (#3217)

---
 .../Browse3/pages/ChatView/MessagePanel.tsx   |  7 +++++
 .../ChatView/PlaygroundMessagePanelEditor.tsx |  5 ++-
 .../pages/PlaygroundPage/StyledTextarea.tsx   | 31 ++++++++++++++++---
 3 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
index f570b2f6295..8cec95707fa 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessagePanel.tsx
@@ -47,6 +47,13 @@ export const MessagePanel = ({
     }
   }, [message.content, contentRef?.current?.scrollHeight]);
 
+  // Set isShowingMore to true when editor is opened
+  useEffect(() => {
+    if (editorHeight !== null) {
+      setIsShowingMore(true);
+    }
+  }, [editorHeight]);
+
   const isUser = message.role === 'user';
   const isSystemPrompt = message.role === 'system';
   const isTool = message.role === 'tool';
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx
index aa519c9659b..d643f103481 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanelEditor.tsx
@@ -68,13 +68,12 @@ export const PlaygroundMessagePanelEditor: React.FC<
     <div
       className={classNames(
         'w-full pt-[6px]',
-        isNested ? 'px-[4px]' : 'px-[8px]'
+        isNested ? 'px-[4px]' : 'px-[16px]'
       )}>
       <StyledTextArea
         value={editedContent}
         onChange={e => setEditedContent(e.target.value)}
-        autoGrow
-        maxHeight={160}
+        startHeight={320}
       />
       {/* 6px vs. 8px to make up for extra padding from textarea field */}
       <div className="z-100 mt-[6px] flex justify-end gap-[8px]">
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/StyledTextarea.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/StyledTextarea.tsx
index 14a5b121da5..97d6aa7d39b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/StyledTextarea.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/StyledTextarea.tsx
@@ -8,11 +8,12 @@ import React, {forwardRef} from 'react';
 type TextAreaProps = React.TextareaHTMLAttributes<HTMLTextAreaElement> & {
   autoGrow?: boolean;
   maxHeight?: string | number;
+  startHeight?: string | number;
   reset?: boolean;
 };
 
 export const StyledTextArea = forwardRef<HTMLTextAreaElement, TextAreaProps>(
-  ({className, autoGrow, maxHeight, reset, ...props}, ref) => {
+  ({className, autoGrow, maxHeight, startHeight, reset, ...props}, ref) => {
     const textareaRef = React.useRef<HTMLTextAreaElement>(null);
 
     React.useEffect(() => {
@@ -26,11 +27,22 @@ export const StyledTextArea = forwardRef<HTMLTextAreaElement, TextAreaProps>(
           return;
         }
 
-        // Disable resize when autoGrow is true
-        textareaElement.style.resize = 'none';
+        // Only disable resize when autoGrow is true
+        textareaElement.style.resize = autoGrow ? 'none' : 'vertical';
+
+        // Set initial height if provided
+        if (startHeight && textareaElement.value === '') {
+          textareaElement.style.height =
+            typeof startHeight === 'number' ? `${startHeight}px` : startHeight;
+          return;
+        }
 
         if (reset || textareaElement.value === '') {
-          textareaElement.style.height = 'auto';
+          textareaElement.style.height = startHeight
+            ? typeof startHeight === 'number'
+              ? `${startHeight}px`
+              : startHeight
+            : 'auto';
           return;
         }
 
@@ -63,7 +75,7 @@ export const StyledTextArea = forwardRef<HTMLTextAreaElement, TextAreaProps>(
 
       return () =>
         textareaRefElement.removeEventListener('input', adjustHeight);
-    }, [autoGrow, maxHeight, reset]);
+    }, [autoGrow, maxHeight, reset, startHeight]);
 
     return (
       <Tailwind style={{display: 'contents'}}>
@@ -86,6 +98,7 @@ export const StyledTextArea = forwardRef<HTMLTextAreaElement, TextAreaProps>(
             'focus:outline-none',
             'relative bottom-0 top-0 items-center rounded-sm',
             'outline outline-1 outline-moon-250',
+            !autoGrow && 'resize-y',
             props.disabled
               ? 'opacity-50'
               : 'hover:outline hover:outline-2 hover:outline-teal-500/40 focus:outline-2',
@@ -94,6 +107,14 @@ export const StyledTextArea = forwardRef<HTMLTextAreaElement, TextAreaProps>(
             'placeholder-moon-500 dark:placeholder-moon-600',
             className
           )}
+          style={{
+            height: startHeight
+              ? typeof startHeight === 'number'
+                ? `${startHeight}px`
+                : startHeight
+              : undefined,
+            ...props.style,
+          }}
           {...props}
         />
       </Tailwind>

From 32fc0da84fad70530da0156348c80ac77969d36b Mon Sep 17 00:00:00 2001
From: Andrew Truong <itsandrewtruong@gmail.com>
Date: Thu, 12 Dec 2024 14:53:57 -0500
Subject: [PATCH 48/52] chore(weave): Better error message for scorer signature
 validation (#3115)

---
 tests/trace/test_evaluations.py | 12 +++++++++---
 weave/scorers/base_scorer.py    |  9 ++++++++-
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/tests/trace/test_evaluations.py b/tests/trace/test_evaluations.py
index e5c38ef0140..ab74d4c0c0b 100644
--- a/tests/trace/test_evaluations.py
+++ b/tests/trace/test_evaluations.py
@@ -1021,13 +1021,19 @@ def my_second_scorer(text, output, model_output):
 
     ds = [{"text": "hello"}]
 
-    with pytest.raises(ValueError, match="Both 'output' and 'model_output'"):
+    with pytest.raises(
+        ValueError, match="cannot include both `output` and `model_output`"
+    ):
         scorer = MyScorer()
 
-    with pytest.raises(ValueError, match="Both 'output' and 'model_output'"):
+    with pytest.raises(
+        ValueError, match="cannot include both `output` and `model_output`"
+    ):
         evaluation = weave.Evaluation(dataset=ds, scorers=[MyScorer()])
 
-    with pytest.raises(ValueError, match="Both 'output' and 'model_output'"):
+    with pytest.raises(
+        ValueError, match="cannot include both `output` and `model_output`"
+    ):
         evaluation = weave.Evaluation(dataset=ds, scorers=[my_second_scorer])
 
 
diff --git a/weave/scorers/base_scorer.py b/weave/scorers/base_scorer.py
index 5a19adcd04f..4ac27f1a76b 100644
--- a/weave/scorers/base_scorer.py
+++ b/weave/scorers/base_scorer.py
@@ -1,4 +1,5 @@
 import inspect
+import textwrap
 from collections.abc import Sequence
 from numbers import Number
 from typing import Any, Callable, Optional, Union
@@ -45,7 +46,13 @@ def _validate_scorer_signature(scorer: Union[Callable, Op, Scorer]) -> bool:
         params = inspect.signature(scorer).parameters
     if "output" in params and "model_output" in params:
         raise ValueError(
-            "Both 'output' and 'model_output' cannot be in the scorer signature; prefer just using `output`."
+            textwrap.dedent(
+                """
+                The scorer signature cannot include both `output` and `model_output` at the same time.
+
+                To resolve, rename one of the arguments to avoid conflict. Prefer using `output` as the model's output.
+                """
+            )
         )
     return True
 

From cdf6574c98451fe06fa85eb5b0036852d304487a Mon Sep 17 00:00:00 2001
From: Josiah Lee <josiahwlee@gmail.com>
Date: Thu, 12 Dec 2024 13:33:39 -0800
Subject: [PATCH 49/52] fix(weave): fix anthropic chats in playground (#3213)

* fix anthropic chats in playground

* deep clone

* lint again

* idk why this isnt auto sorting
---
 .../Browse3/pages/ChatView/ChoicesView.tsx    |  6 ++-
 .../PlaygroundChat/PlaygroundChat.tsx         |  1 -
 .../PlaygroundChat/useChatFunctions.tsx       |  3 --
 .../pages/PlaygroundPage/PlaygroundPage.tsx   | 11 ++++-
 .../PlaygroundPage/usePlaygroundState.ts      | 40 ++++++++++++++++++-
 5 files changed, 53 insertions(+), 8 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx
index 138ca10c7e8..135d297539d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChoicesView.tsx
@@ -31,7 +31,11 @@ export const ChoicesView = ({
   }
   if (choices.length === 1) {
     return (
-      <ChoiceView choice={choices[0]} isStructuredOutput={isStructuredOutput} />
+      <ChoiceView
+        choice={choices[0]}
+        isStructuredOutput={isStructuredOutput}
+        choiceIndex={0}
+      />
     );
   }
   return (
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
index 38f69c4482e..b6b6e7c420d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx
@@ -35,7 +35,6 @@ export const PlaygroundChat = ({
   setSettingsTab,
   settingsTab,
 }: PlaygroundChatProps) => {
-  console.log('playgroundStates', playgroundStates);
   const [chatText, setChatText] = useState('');
   const [isLoading, setIsLoading] = useState(false);
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx
index 0ce3ad02b51..804670a1dc3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/useChatFunctions.tsx
@@ -43,8 +43,6 @@ export const useChatFunctions = (
     messageIndex: number,
     newMessage: Message
   ) => {
-    console.log('editMessage', callIndex, messageIndex, newMessage);
-
     setPlaygroundStateField(callIndex, 'traceCall', prevTraceCall => {
       const newTraceCall = clearTraceCall(
         cloneDeep(prevTraceCall as OptionalTraceCallSchema)
@@ -108,7 +106,6 @@ export const useChatFunctions = (
     choiceIndex: number,
     newChoice: Message
   ) => {
-    console.log('editChoice', callIndex, choiceIndex, newChoice);
     setPlaygroundStateField(callIndex, 'traceCall', prevTraceCall => {
       const newTraceCall = clearTraceCall(
         cloneDeep(prevTraceCall as OptionalTraceCallSchema)
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundPage.tsx
index c6232631e4e..76d1c6d9e31 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundPage.tsx
@@ -7,7 +7,11 @@ import {SimplePageLayoutWithHeader} from '../common/SimplePageLayout';
 import {useWFHooks} from '../wfReactInterface/context';
 import {PlaygroundChat} from './PlaygroundChat/PlaygroundChat';
 import {PlaygroundSettings} from './PlaygroundSettings/PlaygroundSettings';
-import {DEFAULT_SYSTEM_MESSAGE, usePlaygroundState} from './usePlaygroundState';
+import {
+  DEFAULT_SYSTEM_MESSAGE,
+  parseTraceCall,
+  usePlaygroundState,
+} from './usePlaygroundState';
 
 export type PlaygroundPageProps = {
   entity: string;
@@ -89,7 +93,10 @@ export const PlaygroundPageInner = (props: PlaygroundPageProps) => {
       for (const [idx, state] of newStates.entries()) {
         for (const c of calls || []) {
           if (state.traceCall.id === c.callId) {
-            newStates[idx] = {...state, traceCall: c.traceCall || {}};
+            newStates[idx] = {
+              ...state,
+              traceCall: parseTraceCall(c.traceCall || {}),
+            };
             break;
           }
         }
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts
index 8439abc4ddf..cbcc7c52fb8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/usePlaygroundState.ts
@@ -1,5 +1,11 @@
+import {cloneDeep} from 'lodash';
 import {SetStateAction, useCallback, useState} from 'react';
 
+import {
+  anthropicContentBlocksToChoices,
+  hasStringProp,
+  isAnthropicCompletionFormat,
+} from '../ChatView/hooks';
 import {LLM_MAX_TOKENS_KEYS, LLMMaxTokensKey} from './llmMaxTokens';
 import {
   OptionalTraceCallSchema,
@@ -77,7 +83,7 @@ export const usePlaygroundState = () => {
       setPlaygroundStates(prevState => {
         const newState = {...prevState[0]};
 
-        newState.traceCall = traceCall;
+        newState.traceCall = parseTraceCall(traceCall);
 
         if (!inputs) {
           return [newState];
@@ -155,3 +161,35 @@ export const getInputFromPlaygroundState = (state: PlaygroundState) => {
     tools: tools.length > 0 ? tools : undefined,
   };
 };
+
+// This is a helper function to parse the trace call output for anthropic
+// so that the playground can display the choices
+export const parseTraceCall = (traceCall: OptionalTraceCallSchema) => {
+  const parsedTraceCall = cloneDeep(traceCall);
+
+  // Handles anthropic outputs
+  // Anthropic has content and stop_reason as top-level fields
+  if (isAnthropicCompletionFormat(parsedTraceCall.output)) {
+    const {content, stop_reason, ...outputs} = parsedTraceCall.output as any;
+    parsedTraceCall.output = {
+      ...outputs,
+      choices: anthropicContentBlocksToChoices(content, stop_reason),
+    };
+  }
+  // Handles anthropic inputs
+  // Anthropic has system message as a top-level request field
+  if (hasStringProp(parsedTraceCall.inputs, 'system')) {
+    const {messages, system, ...inputs} = parsedTraceCall.inputs as any;
+    parsedTraceCall.inputs = {
+      ...inputs,
+      messages: [
+        {
+          role: 'system',
+          content: system,
+        },
+        ...messages,
+      ],
+    };
+  }
+  return parsedTraceCall;
+};

From aebbfb9cfd8d65998cdd64c03de9f862f2e64a3e Mon Sep 17 00:00:00 2001
From: Jamie Rasmussen <112953339+jamie-rasmussen@users.noreply.github.com>
Date: Thu, 12 Dec 2024 17:34:57 -0600
Subject: [PATCH 50/52] chore(ui): uuid should be in dependencies now (#3225)

---
 weave-js/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weave-js/package.json b/weave-js/package.json
index 1f551021ed4..db96be60691 100644
--- a/weave-js/package.json
+++ b/weave-js/package.json
@@ -141,6 +141,7 @@
     "unified": "^10.1.0",
     "unist-util-visit": "3.1.0",
     "universal-perf-hooks": "^1.0.1",
+    "uuid": "^11.0.3",
     "vega": "^5.24.0",
     "vega-lite": "5.6.0",
     "vega-tooltip": "^0.28.0",
@@ -236,7 +237,6 @@
     "tslint-config-prettier": "^1.18.0",
     "tslint-plugin-prettier": "^2.3.0",
     "typescript": "4.7.4",
-    "uuid": "^11.0.3",
     "vite": "5.2.9",
     "vitest": "^1.6.0"
   },

From 4c73aa3925ffc0edae8b785614363e85a8cf7379 Mon Sep 17 00:00:00 2001
From: Marie Barr-Ramsey <126013019+mbarrramsey@users.noreply.github.com>
Date: Thu, 12 Dec 2024 15:48:34 -0800
Subject: [PATCH 51/52] chore(weave): add spiral icon (#3226)

---
 weave-js/src/assets/icons/icon-spiral.svg | 5 +++++
 weave-js/src/components/Icon/Icon.tsx     | 5 +++++
 weave-js/src/components/Icon/index.ts     | 1 +
 weave-js/src/components/Icon/types.ts     | 1 +
 4 files changed, 12 insertions(+)
 create mode 100644 weave-js/src/assets/icons/icon-spiral.svg

diff --git a/weave-js/src/assets/icons/icon-spiral.svg b/weave-js/src/assets/icons/icon-spiral.svg
new file mode 100644
index 00000000000..ce5c147b43a
--- /dev/null
+++ b/weave-js/src/assets/icons/icon-spiral.svg
@@ -0,0 +1,5 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="icon-spiral">
+<path id="Union" fill-rule="evenodd" clip-rule="evenodd" d="M12 3.75C6.89421 3.75 2.75 7.89421 2.75 13C2.75 17.1458 6.10421 20.5 10.25 20.5C14.3958 20.5 17.75 17.1458 17.75 13C17.75 9.82421 15.1758 7.25 12 7.25C8.82421 7.25 6.25 9.82421 6.25 13C6.25 15.2058 8.04421 17 10.25 17C12.4558 17 14.25 15.2058 14.25 13C14.25 11.7542 13.2458 10.75 12 10.75C11.5858 10.75 11.25 10.4142 11.25 10C11.25 9.58579 11.5858 9.25 12 9.25C14.0742 9.25 15.75 10.9258 15.75 13C15.75 16.0342 13.2842 18.5 10.25 18.5C7.21579 18.5 4.75 16.0342 4.75 13C4.75 8.99579 7.99579 5.75 12 5.75C16.0042 5.75 19.25 8.99579 19.25 13C19.25 17.9742 15.2242 22 10.25 22C5.27579 22 1.25 17.9742 1.25 13C1.25 7.06579 6.06579 2.25 12 2.25C17.9342 2.25 22.75 7.06579 22.75 13C22.75 13.4142 22.4142 13.75 22 13.75C21.5858 13.75 21.25 13.4142 21.25 13C21.25 7.89421 17.1058 3.75 12 3.75Z" fill="currentColor"/>
+</g>
+</svg>
diff --git a/weave-js/src/components/Icon/Icon.tsx b/weave-js/src/components/Icon/Icon.tsx
index 32b9ca65921..20e8e23401d 100644
--- a/weave-js/src/components/Icon/Icon.tsx
+++ b/weave-js/src/components/Icon/Icon.tsx
@@ -211,6 +211,7 @@ import {ReactComponent as ImportSmoothing} from '../../assets/icons/icon-smoothi
 import {ReactComponent as ImportSort} from '../../assets/icons/icon-sort.svg';
 import {ReactComponent as ImportSortAscending} from '../../assets/icons/icon-sort-ascending.svg';
 import {ReactComponent as ImportSortDescending} from '../../assets/icons/icon-sort-descending.svg';
+import {ReactComponent as ImportSpiral} from '../../assets/icons/icon-spiral.svg';
 import {ReactComponent as ImportSplit} from '../../assets/icons/icon-split.svg';
 import {ReactComponent as ImportSquare} from '../../assets/icons/icon-square.svg';
 import {ReactComponent as ImportStar} from '../../assets/icons/icon-star.svg';
@@ -914,6 +915,9 @@ export const IconSortAscending = (props: SVGIconProps) => (
 export const IconSortDescending = (props: SVGIconProps) => (
   <ImportSortDescending {...updateIconProps(props)} />
 );
+export const IconSpiral = (props: SVGIconProps) => (
+  <ImportSpiral {...updateIconProps(props)} />
+);
 export const IconSplit = (props: SVGIconProps) => (
   <ImportSplit {...updateIconProps(props)} />
 );
@@ -1295,6 +1299,7 @@ const ICON_NAME_TO_ICON: Record<IconName, ElementType> = {
   sort: IconSort,
   'sort-ascending': IconSortAscending,
   'sort-descending': IconSortDescending,
+  spiral: IconSpiral,
   split: IconSplit,
   square: IconSquare,
   star: IconStar,
diff --git a/weave-js/src/components/Icon/index.ts b/weave-js/src/components/Icon/index.ts
index 08bf7854ad2..39c6eed3170 100644
--- a/weave-js/src/components/Icon/index.ts
+++ b/weave-js/src/components/Icon/index.ts
@@ -211,6 +211,7 @@ export {
   IconSort,
   IconSortAscending,
   IconSortDescending,
+  IconSpiral,
   IconSplit,
   IconSquare,
   IconStar,
diff --git a/weave-js/src/components/Icon/types.ts b/weave-js/src/components/Icon/types.ts
index 7ca30049257..47f5f357adc 100644
--- a/weave-js/src/components/Icon/types.ts
+++ b/weave-js/src/components/Icon/types.ts
@@ -210,6 +210,7 @@ export const IconNames = {
   Sort: 'sort',
   SortAscending: 'sort-ascending',
   SortDescending: 'sort-descending',
+  Spiral: 'spiral',
   Split: 'split',
   Square: 'square',
   Star: 'star',

From abebaa8f9611d42ba66185bea55e1a851abc87d8 Mon Sep 17 00:00:00 2001
From: Connie Lee <connie.lee0317@gmail.com>
Date: Thu, 12 Dec 2024 15:59:42 -0800
Subject: [PATCH 52/52] chore(app): Allow skipping night-aware class for
 TextField (#3228)

---
 weave-js/src/components/Form/TextField.tsx | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/weave-js/src/components/Form/TextField.tsx b/weave-js/src/components/Form/TextField.tsx
index c40f697ac85..8f5dd1171ff 100644
--- a/weave-js/src/components/Form/TextField.tsx
+++ b/weave-js/src/components/Form/TextField.tsx
@@ -37,6 +37,7 @@ type TextFieldProps = {
   dataTest?: string;
   step?: number;
   variant?: 'default' | 'ghost';
+  isContainerNightAware?: boolean;
 };
 
 export const TextField = ({
@@ -59,6 +60,7 @@ export const TextField = ({
   autoComplete,
   dataTest,
   step,
+  isContainerNightAware,
 }: TextFieldProps) => {
   const textFieldSize = size ?? 'medium';
   const leftPaddingForIcon = textFieldSize === 'medium' ? 'pl-34' : 'pl-36';
@@ -83,7 +85,6 @@ export const TextField = ({
     <Tailwind style={{width: '100%'}}>
       <div
         className={classNames(
-          'night-aware',
           'relative rounded-sm',
           textFieldSize === 'medium' ? 'h-32' : 'h-40',
           'bg-white dark:bg-moon-900',
@@ -93,6 +94,9 @@ export const TextField = ({
           variant === 'ghost' &&
             'outline outline-1 outline-transparent dark:outline-transparent',
           {
+            // must not add "night-aware" class if already in a night-aware
+            // container, otherwise they'll cancel each other out
+            'night-aware': !isContainerNightAware,
             'hover:outline-2 [&:hover:not(:focus-within)]:outline-[#83E4EB] dark:[&:hover:not(:focus-within)]:outline-teal-650':
               !errorState && variant === 'default',
             'focus-within:outline-2 focus-within:outline-teal-400 dark:focus-within:outline-teal-600':