From 84e5fad6eb0e270eff86a3bf99ec8f0b604f3c69 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 10:06:08 -0700
Subject: [PATCH 001/169] stubbed

---
 .../components/FancyPage/useProjectSidebar.ts |  7 ++++
 .../PagePanelComponents/Home/Browse3.tsx      | 10 ++++++
 .../LeaderboardPage.tsx                       | 35 +++++++++++++++++++
 3 files changed, 52 insertions(+)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index 6fcc50703f9..088b0dfddbb 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -130,6 +130,13 @@ export const useProjectSidebar = (
             label: 'Weave',
             isShown: isShowAll,
           },
+          {
+            type: 'button' as const,
+            name: 'Leaderboard',
+            slug: 'weave/leaderboard',
+            isShown: showWeaveSidebarItems || isShowAll,
+            iconName: IconNames.LayoutTabs,
+          },
           {
             type: 'button' as const,
             name: 'Traces',
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
index b2c77cdab6c..66151840d25 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
@@ -82,6 +82,7 @@ import {Empty} from './Browse3/pages/common/Empty';
 import {EMPTY_NO_TRACE_SERVER} from './Browse3/pages/common/EmptyContent';
 import {SimplePageLayoutContext} from './Browse3/pages/common/SimplePageLayout';
 import {CompareEvaluationsPage} from './Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage';
+import {LeaderboardPage} from './Browse3/pages/CompareEvaluationsPage/LeaderboardPage';
 import {ObjectPage} from './Browse3/pages/ObjectPage';
 import {ObjectVersionPage} from './Browse3/pages/ObjectVersionPage';
 import {
@@ -151,6 +152,7 @@ const tabOptions = [
   'op-versions',
   'calls',
   'evaluations',
+  'leaderboard',
   'boards',
   'tables',
 ];
@@ -492,6 +494,9 @@ const Browse3ProjectRoot: FC<{
         <Route path={`${projectRoot}/:tab(compare-evaluations)`}>
           <CompareEvaluationsBinding />
         </Route>
+        <Route path={`${projectRoot}/:tab(leaderboard)`}>
+          <LeaderboardPageBinding />
+        </Route>
         {/* BOARDS */}
         <Route
           path={[
@@ -935,6 +940,11 @@ const CompareEvaluationsBinding = () => {
   );
 };
 
+const LeaderboardPageBinding = () => {
+  const {entity, project} = useParamsDecoded<Browse3TabParams>();
+  return <LeaderboardPage entity={entity} project={project} />;
+};
+
 const OpsPageBinding = () => {
   const params = useParamsDecoded<Browse3TabItemParams>();
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
new file mode 100644
index 00000000000..63d24385af2
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
@@ -0,0 +1,35 @@
+
+import React from 'react';
+
+import {SimplePageLayout} from '../common/SimplePageLayout';
+
+type LeaderboardPageProps = {
+  entity: string;
+  project: string;
+};
+
+export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
+  return (
+    <SimplePageLayout
+      title="Leaderboard"
+      hideTabsIfSingle
+      tabs={[
+        {
+          label: 'All',
+          content: (
+            <LeaderboardPageContent
+              entity={props.entity}
+              project={props.project}
+            />
+          ),
+        },
+      ]}
+    />
+  );
+};
+
+export const LeaderboardPageContent: React.FC<
+  LeaderboardPageProps
+> = props => {
+  return <>Leaderboard</>;
+};

From d563e0c9ee1cb49e93d3fc5dc4f28396cf7fb09b Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 10:23:12 -0700
Subject: [PATCH 002/169] First cursor implementation

---
 .../EditableMarkdown.tsx                      | 58 +++++++++++++
 .../LeaderboardGrid.tsx                       | 84 +++++++++++++++++++
 .../LeaderboardHeader.tsx                     | 26 ++++++
 .../LeaderboardPage.tsx                       | 58 +++++++++++--
 .../pages/CompareEvaluationsPage/fakeData.ts  | 41 +++++++++
 5 files changed, 259 insertions(+), 8 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardHeader.tsx
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
new file mode 100644
index 00000000000..895819691cb
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
@@ -0,0 +1,58 @@
+import React, {useState} from 'react';
+import ReactMarkdown from 'react-markdown';
+import styled from 'styled-components';
+
+interface EditableMarkdownProps {
+  value: string;
+  onChange: (value: string) => void;
+  placeholder: string;
+}
+
+export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
+  value,
+  onChange,
+  placeholder,
+}) => {
+  const [isEditing, setIsEditing] = useState(false);
+
+  return (
+    <Container onDoubleClick={() => setIsEditing(true)}>
+      {isEditing ? (
+        <TextArea
+          value={value}
+          onChange={e => onChange(e.target.value)}
+          onBlur={() => setIsEditing(false)}
+          placeholder={placeholder}
+          autoFocus
+        />
+      ) : (
+        <MarkdownContainer>
+          <ReactMarkdown>{value || placeholder}</ReactMarkdown>
+        </MarkdownContainer>
+      )}
+    </Container>
+  );
+};
+
+const Container = styled.div`
+  width: 100%;
+  min-height: 100px;
+`;
+
+const TextArea = styled.textarea`
+  width: 100%;
+  min-height: 100px;
+  padding: 8px;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  font-family: inherit;
+  font-size: inherit;
+  resize: vertical;
+`;
+
+const MarkdownContainer = styled.div`
+  cursor: pointer;
+  &:hover {
+    background-color: #f0f0f0;
+  }
+`;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
new file mode 100644
index 00000000000..bc0ae4815ed
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
@@ -0,0 +1,84 @@
+import React from 'react';
+import styled from 'styled-components';
+
+interface LeaderboardGridProps {
+  data: {
+    models: string[];
+    metrics: string[];
+    scores: {[key: string]: {[key: string]: number}};
+  };
+  onCellClick: (modelName: string, metricName: string, score: number) => void;
+}
+
+export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
+  data,
+  onCellClick,
+}) => (
+  <GridContainer>
+    <GridHeader>
+      <HeaderCell>Model</HeaderCell>
+      {data.metrics.map(metric => (
+        <HeaderCell key={metric}>{metric}</HeaderCell>
+      ))}
+    </GridHeader>
+    {data.models.map(model => (
+      <GridRow key={model}>
+        <ModelCell>{model}</ModelCell>
+        {data.metrics.map(metric => (
+          <ScoreCell
+            key={`${model}-${metric}`}
+            onClick={() =>
+              onCellClick(model, metric, data.scores[model][metric])
+            }
+            score={data.scores[model][metric]}>
+            {data.scores[model][metric].toFixed(2)}%
+          </ScoreCell>
+        ))}
+      </GridRow>
+    ))}
+  </GridContainer>
+);
+
+const GridContainer = styled.div`
+  background-color: white;
+  border-radius: 8px;
+  overflow: hidden;
+  box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+`;
+
+const GridHeader = styled.div`
+  display: grid;
+  grid-template-columns: 200px repeat(auto-fit, minmax(100px, 1fr));
+  background-color: #34495e;
+  color: white;
+  font-weight: bold;
+`;
+
+const HeaderCell = styled.div`
+  padding: 16px;
+  text-align: center;
+`;
+
+const GridRow = styled.div`
+  display: grid;
+  grid-template-columns: 200px repeat(auto-fit, minmax(100px, 1fr));
+  &:nth-child(even) {
+    background-color: #f8f9fa;
+  }
+`;
+
+const ModelCell = styled.div`
+  padding: 16px;
+  font-weight: bold;
+`;
+
+const ScoreCell = styled.div<{score: number}>`
+  padding: 16px;
+  text-align: center;
+  cursor: pointer;
+  transition: background-color 0.2s;
+  background-color: ${props => `hsl(${120 * (props.score / 100)}, 70%, 80%)`};
+  &:hover {
+    background-color: ${props => `hsl(${120 * (props.score / 100)}, 70%, 70%)`};
+  }
+`;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardHeader.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardHeader.tsx
new file mode 100644
index 00000000000..acf5a0af928
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardHeader.tsx
@@ -0,0 +1,26 @@
+import React from 'react';
+import styled from 'styled-components';
+
+interface LeaderboardHeaderProps {
+  title: string;
+}
+
+export const LeaderboardHeader: React.FC<LeaderboardHeaderProps> = ({
+  title,
+}) => (
+  <HeaderContainer>
+    <Title>{title}</Title>
+  </HeaderContainer>
+);
+
+const HeaderContainer = styled.div`
+  background-color: #2c3e50;
+  color: white;
+  padding: 24px;
+  border-radius: 8px;
+`;
+
+const Title = styled.h1`
+  margin: 0;
+  font-size: 28px;
+`;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
index 63d24385af2..9d02819d466 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
@@ -1,7 +1,11 @@
-
-import React from 'react';
+import React, {useState} from 'react';
+import styled from 'styled-components';
 
 import {SimplePageLayout} from '../common/SimplePageLayout';
+import {EditableMarkdown} from './EditableMarkdown';
+import {fakeLeaderboardData} from './fakeData';
+import {LeaderboardGrid} from './LeaderboardGrid';
+import {LeaderboardHeader} from './LeaderboardHeader';
 
 type LeaderboardPageProps = {
   entity: string;
@@ -11,11 +15,11 @@ type LeaderboardPageProps = {
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
   return (
     <SimplePageLayout
-      title="Leaderboard"
+      title={`${props.project} Leaderboard`}
       hideTabsIfSingle
       tabs={[
         {
-          label: 'All',
+          label: 'Results',
           content: (
             <LeaderboardPageContent
               entity={props.entity}
@@ -28,8 +32,46 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
   );
 };
 
-export const LeaderboardPageContent: React.FC<
-  LeaderboardPageProps
-> = props => {
-  return <>Leaderboard</>;
+export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
+  const [description, setDescription] = useState(
+    'Welcome to the Leaderboard! Compare model performance across different metrics.'
+  );
+  const [data] = useState(fakeLeaderboardData);
+
+  const handleCellClick = (
+    modelName: string,
+    metricName: string,
+    score: number
+  ) => {
+    console.log(`Clicked on ${modelName} for ${metricName}: ${score}%`);
+    // TODO: Implement action on cell click
+  };
+
+  return (
+    <LeaderboardContainer>
+      <DescriptionArea>
+        <EditableMarkdown
+          value={description}
+          onChange={setDescription}
+          placeholder="Add a description for this leaderboard..."
+        />
+      </DescriptionArea>
+      <LeaderboardGrid data={data} onCellClick={handleCellClick} />
+    </LeaderboardContainer>
+  );
 };
+
+const LeaderboardContainer = styled.div`
+  display: flex;
+  flex-direction: column;
+  gap: 24px;
+  padding: 24px;
+  background-color: #f5f5f5;
+`;
+
+const DescriptionArea = styled.div`
+  background-color: white;
+  border-radius: 8px;
+  padding: 16px;
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+`;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts
new file mode 100644
index 00000000000..fb80776fbe0
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts
@@ -0,0 +1,41 @@
+export const fakeLeaderboardData = {
+  models: ['Model A', 'Model B', 'Model C', 'Model D', 'Model E'],
+  metrics: ['Accuracy', 'F1 Score', 'Precision', 'Recall', 'AUC-ROC'],
+  scores: {
+    'Model A': {
+      Accuracy: 95.2,
+      'F1 Score': 94.8,
+      Precision: 96.1,
+      Recall: 93.5,
+      'AUC-ROC': 97.3,
+    },
+    'Model B': {
+      Accuracy: 92.7,
+      'F1 Score': 91.9,
+      Precision: 93.2,
+      Recall: 90.6,
+      'AUC-ROC': 94.5,
+    },
+    'Model C': {
+      Accuracy: 88.9,
+      'F1 Score': 87.6,
+      Precision: 89.4,
+      Recall: 85.8,
+      'AUC-ROC': 91.2,
+    },
+    'Model D': {
+      Accuracy: 91.5,
+      'F1 Score': 90.8,
+      Precision: 92.3,
+      Recall: 89.3,
+      'AUC-ROC': 93.7,
+    },
+    'Model E': {
+      Accuracy: 94.1,
+      'F1 Score': 93.5,
+      Precision: 95.2,
+      Recall: 91.8,
+      'AUC-ROC': 96.4,
+    },
+  },
+};

From dcb3dc65d19073ad13c63e68a10efb5529b59ad6 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 10:36:53 -0700
Subject: [PATCH 003/169] Cursor Checkpoint 2

---
 .../LeaderboardGrid.tsx                       | 126 ++++++++----------
 .../LeaderboardPage.tsx                       |  31 ++---
 .../pages/CompareEvaluationsPage/fakeData.ts  |  58 +++-----
 3 files changed, 86 insertions(+), 129 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
index bc0ae4815ed..72b62072b62 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
@@ -1,5 +1,5 @@
 import React from 'react';
-import styled from 'styled-components';
+import { DataGrid, GridColDef, GridValueGetterParams, GridRenderCellParams } from '@mui/x-data-grid';
 
 interface LeaderboardGridProps {
   data: {
@@ -10,75 +10,65 @@ interface LeaderboardGridProps {
   onCellClick: (modelName: string, metricName: string, score: number) => void;
 }
 
+interface RowData {
+  id: number;
+  model: string;
+  [key: string]: number | string;
+}
+
 export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   data,
   onCellClick,
-}) => (
-  <GridContainer>
-    <GridHeader>
-      <HeaderCell>Model</HeaderCell>
-      {data.metrics.map(metric => (
-        <HeaderCell key={metric}>{metric}</HeaderCell>
-      ))}
-    </GridHeader>
-    {data.models.map(model => (
-      <GridRow key={model}>
-        <ModelCell>{model}</ModelCell>
-        {data.metrics.map(metric => (
-          <ScoreCell
-            key={`${model}-${metric}`}
-            onClick={() =>
-              onCellClick(model, metric, data.scores[model][metric])
-            }
-            score={data.scores[model][metric]}>
-            {data.scores[model][metric].toFixed(2)}%
-          </ScoreCell>
-        ))}
-      </GridRow>
-    ))}
-  </GridContainer>
-);
-
-const GridContainer = styled.div`
-  background-color: white;
-  border-radius: 8px;
-  overflow: hidden;
-  box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
-`;
-
-const GridHeader = styled.div`
-  display: grid;
-  grid-template-columns: 200px repeat(auto-fit, minmax(100px, 1fr));
-  background-color: #34495e;
-  color: white;
-  font-weight: bold;
-`;
-
-const HeaderCell = styled.div`
-  padding: 16px;
-  text-align: center;
-`;
-
-const GridRow = styled.div`
-  display: grid;
-  grid-template-columns: 200px repeat(auto-fit, minmax(100px, 1fr));
-  &:nth-child(even) {
-    background-color: #f8f9fa;
-  }
-`;
+}) => {
+  const columns: GridColDef[] = [
+    { field: 'model', headerName: 'Model', width: 200 },
+    ...data.metrics.map(metric => ({
+      field: metric,
+      headerName: metric,
+      width: 130,
+    //   valueGetter: ((params: GridValueGetterParams)) => params.row[metric],
+      renderCell: (params: GridRenderCellParams) => (
+        <div
+          style={{
+            width: '100%',
+            height: '100%',
+            display: 'flex',
+            alignItems: 'center',
+            justifyContent: 'center',
+            backgroundColor: `hsl(${120 * (params.value as number / 100)}, 70%, 90%)`,
+          }}
+          onClick={() =>
+            onCellClick(
+              params.row.model,
+              params.field as string,
+              params.value as number
+            )
+          }
+        >
+          {`${(params.value as number).toFixed(2)}%`}
+        </div>
+      ),
+    })),
+  ];
 
-const ModelCell = styled.div`
-  padding: 16px;
-  font-weight: bold;
-`;
+  const rows: RowData[] = data.models.map((model, index) => ({
+    id: index,
+    model,
+    ...data.scores[model],
+  }));
 
-const ScoreCell = styled.div<{score: number}>`
-  padding: 16px;
-  text-align: center;
-  cursor: pointer;
-  transition: background-color 0.2s;
-  background-color: ${props => `hsl(${120 * (props.score / 100)}, 70%, 80%)`};
-  &:hover {
-    background-color: ${props => `hsl(${120 * (props.score / 100)}, 70%, 70%)`};
-  }
-`;
+  return (
+    <DataGrid
+      rows={rows}
+      columns={columns}
+      initialState={{
+        pagination: {
+          paginationModel: { pageSize: 25, page: 0 },
+        },
+      }}
+      pageSizeOptions={[25, 50, 100]}
+      disableRowSelectionOnClick
+      sx={{ height: '100%', width: '100%' }}
+    />
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
index 9d02819d466..4114d176c77 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
@@ -1,11 +1,10 @@
 import React, {useState} from 'react';
-import styled from 'styled-components';
+import {Box, Divider} from '@mui/material';
 
 import {SimplePageLayout} from '../common/SimplePageLayout';
 import {EditableMarkdown} from './EditableMarkdown';
 import {fakeLeaderboardData} from './fakeData';
 import {LeaderboardGrid} from './LeaderboardGrid';
-import {LeaderboardHeader} from './LeaderboardHeader';
 
 type LeaderboardPageProps = {
   entity: string;
@@ -48,30 +47,18 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   };
 
   return (
-    <LeaderboardContainer>
-      <DescriptionArea>
+    <Box display="flex" flexDirection="column" height="100%">
+      <Box mb={2}>
         <EditableMarkdown
           value={description}
           onChange={setDescription}
           placeholder="Add a description for this leaderboard..."
         />
-      </DescriptionArea>
-      <LeaderboardGrid data={data} onCellClick={handleCellClick} />
-    </LeaderboardContainer>
+      </Box>
+      <Divider />
+      <Box flexGrow={1} mt={2}>
+        <LeaderboardGrid data={data} onCellClick={handleCellClick} />
+      </Box>
+    </Box>
   );
 };
-
-const LeaderboardContainer = styled.div`
-  display: flex;
-  flex-direction: column;
-  gap: 24px;
-  padding: 24px;
-  background-color: #f5f5f5;
-`;
-
-const DescriptionArea = styled.div`
-  background-color: white;
-  border-radius: 8px;
-  padding: 16px;
-  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
-`;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts
index fb80776fbe0..5e83a82f28d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts
@@ -1,41 +1,21 @@
+const generateRandomScore = () => Math.round((Math.random() * 30 + 70) * 100) / 100;
+
+const metrics = ['Accuracy', 'F1 Score', 'Precision', 'Recall', 'AUC-ROC'];
+const models = Array.from({ length: 100 }, (_, i) => `Model ${i + 1}`);
+
+const generateScores = () => {
+  const scores: { [key: string]: number } = {};
+  metrics.forEach(metric => {
+    scores[metric] = generateRandomScore();
+  });
+  return scores;
+};
+
 export const fakeLeaderboardData = {
-  models: ['Model A', 'Model B', 'Model C', 'Model D', 'Model E'],
-  metrics: ['Accuracy', 'F1 Score', 'Precision', 'Recall', 'AUC-ROC'],
-  scores: {
-    'Model A': {
-      Accuracy: 95.2,
-      'F1 Score': 94.8,
-      Precision: 96.1,
-      Recall: 93.5,
-      'AUC-ROC': 97.3,
-    },
-    'Model B': {
-      Accuracy: 92.7,
-      'F1 Score': 91.9,
-      Precision: 93.2,
-      Recall: 90.6,
-      'AUC-ROC': 94.5,
-    },
-    'Model C': {
-      Accuracy: 88.9,
-      'F1 Score': 87.6,
-      Precision: 89.4,
-      Recall: 85.8,
-      'AUC-ROC': 91.2,
-    },
-    'Model D': {
-      Accuracy: 91.5,
-      'F1 Score': 90.8,
-      Precision: 92.3,
-      Recall: 89.3,
-      'AUC-ROC': 93.7,
-    },
-    'Model E': {
-      Accuracy: 94.1,
-      'F1 Score': 93.5,
-      Precision: 95.2,
-      Recall: 91.8,
-      'AUC-ROC': 96.4,
-    },
-  },
+  models,
+  metrics,
+  scores: models.reduce((acc, model) => {
+    acc[model] = generateScores();
+    return acc;
+  }, {} as { [key: string]: { [key: string]: number } }),
 };

From 470c6573dde798264dcfc5547f4b6a520dfd2c4e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 11:16:48 -0700
Subject: [PATCH 004/169] Cursor Checkpoint 3

---
 .../EditableMarkdown.tsx                      | 129 +++++++++++++-----
 .../LeaderboardGrid.tsx                       |  56 +++++---
 .../LeaderboardPage.tsx                       |  78 +++++++----
 3 files changed, 188 insertions(+), 75 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
index 895819691cb..0e59d68d19e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
@@ -1,6 +1,6 @@
-import React, {useState} from 'react';
+import React, { useState, useRef, useLayoutEffect, useCallback } from 'react';
 import ReactMarkdown from 'react-markdown';
-import styled from 'styled-components';
+import { TextField, Typography, Box } from '@mui/material';
 
 interface EditableMarkdownProps {
   value: string;
@@ -14,45 +14,110 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
   placeholder,
 }) => {
   const [isEditing, setIsEditing] = useState(false);
+  const containerRef = useRef<HTMLDivElement>(null);
+  const contentRef = useRef<HTMLDivElement>(null);
+  const textFieldRef = useRef<HTMLTextAreaElement>(null);
+
+  const updateHeight = useCallback(() => {
+    if (containerRef.current) {
+      let newHeight: number;
+      
+      if (isEditing && textFieldRef.current) {
+        // For editing mode, use scrollHeight of the textarea
+        newHeight = textFieldRef.current.scrollHeight;
+      } else if (!isEditing && contentRef.current) {
+        // For viewing mode, get the rendered content height
+        newHeight = contentRef.current.offsetHeight;
+      } else {
+        return; // Exit if we can't determine the height
+      }
+
+      // Add padding (16px top + 16px bottom)
+      newHeight += 32;
+
+      // Set a minimum height to prevent collapsing
+      newHeight = Math.max(newHeight, 56);
+
+      containerRef.current.style.height = `${newHeight}px`;
+    }
+  }, [isEditing]);
+
+  useLayoutEffect(() => {
+    // Use RAF to ensure DOM is fully updated before measuring
+    requestAnimationFrame(() => {
+      updateHeight();
+    });
+
+    const resizeObserver = new ResizeObserver(() => {
+      requestAnimationFrame(() => {
+        updateHeight();
+      });
+    });
+
+    if (containerRef.current) {
+      resizeObserver.observe(containerRef.current);
+    }
+
+    return () => resizeObserver.disconnect();
+  }, [updateHeight]);
+
+  const handleBlur = () => {
+    setIsEditing(false);
+    requestAnimationFrame(() => {
+      updateHeight();
+    });
+  };
+
+  const handleEdit = () => {
+    setIsEditing(true);
+    requestAnimationFrame(() => {
+      updateHeight();
+    });
+  };
 
   return (
-    <Container onDoubleClick={() => setIsEditing(true)}>
+    <Box ref={containerRef} sx={{ padding: '16px', transition: 'height 0.3s' }}>
       {isEditing ? (
-        <TextArea
+        <TextField
+          fullWidth
+          multiline
           value={value}
-          onChange={e => onChange(e.target.value)}
-          onBlur={() => setIsEditing(false)}
+          onChange={(e) => {
+            onChange(e.target.value);
+            requestAnimationFrame(() => {
+              updateHeight();
+            });
+          }}
+          onBlur={handleBlur}
           placeholder={placeholder}
           autoFocus
+          variant="outlined"
+          inputRef={textFieldRef}
+          sx={{ 
+            '& .MuiOutlinedInput-root': {
+              padding: 0,
+              '& textarea': {
+                padding: '0',
+                overflow: 'hidden',
+              },
+            },
+          }}
         />
       ) : (
-        <MarkdownContainer>
+        <Typography
+          component="div"
+          ref={contentRef}
+          sx={{
+            cursor: 'pointer',
+            '&:hover': {
+              backgroundColor: 'action.hover',
+            },
+          }}
+          onClick={handleEdit}
+        >
           <ReactMarkdown>{value || placeholder}</ReactMarkdown>
-        </MarkdownContainer>
+        </Typography>
       )}
-    </Container>
+    </Box>
   );
 };
-
-const Container = styled.div`
-  width: 100%;
-  min-height: 100px;
-`;
-
-const TextArea = styled.textarea`
-  width: 100%;
-  min-height: 100px;
-  padding: 8px;
-  border: 1px solid #ccc;
-  border-radius: 4px;
-  font-family: inherit;
-  font-size: inherit;
-  resize: vertical;
-`;
-
-const MarkdownContainer = styled.div`
-  cursor: pointer;
-  &:hover {
-    background-color: #f0f0f0;
-  }
-`;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
index 72b62072b62..56bdc55168d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
@@ -1,5 +1,6 @@
-import React from 'react';
-import { DataGrid, GridColDef, GridValueGetterParams, GridRenderCellParams } from '@mui/x-data-grid';
+import React, { useMemo } from 'react';
+import { DataGrid, GridColDef, GridRenderCellParams } from '@mui/x-data-grid';
+import { Box } from '@mui/material';
 
 interface LeaderboardGridProps {
   data: {
@@ -20,13 +21,31 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   data,
   onCellClick,
 }) => {
+  const metricRanges = useMemo(() => {
+    const ranges: {[key: string]: {min: number; max: number}} = {};
+    data.metrics.forEach(metric => {
+      const scores = data.models.map(model => data.scores[model][metric]);
+      ranges[metric] = {
+        min: Math.min(...scores),
+        max: Math.max(...scores)
+      };
+    });
+    return ranges;
+  }, [data]);
+
+  const getColorForScore = (metric: string, score: number) => {
+    const { min, max } = metricRanges[metric];
+    const normalizedScore = (score - min) / (max - min);
+    return `hsl(${120 * normalizedScore}, 70%, 85%)`;
+  };
+
   const columns: GridColDef[] = [
-    { field: 'model', headerName: 'Model', width: 200 },
+    { field: 'model', headerName: 'Model', width: 200, flex: 1 },
     ...data.metrics.map(metric => ({
       field: metric,
       headerName: metric,
       width: 130,
-    //   valueGetter: ((params: GridValueGetterParams)) => params.row[metric],
+      flex: 1,
       renderCell: (params: GridRenderCellParams) => (
         <div
           style={{
@@ -35,7 +54,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
             display: 'flex',
             alignItems: 'center',
             justifyContent: 'center',
-            backgroundColor: `hsl(${120 * (params.value as number / 100)}, 70%, 90%)`,
+            backgroundColor: getColorForScore(metric, params.value as number),
           }}
           onClick={() =>
             onCellClick(
@@ -58,17 +77,20 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   }));
 
   return (
-    <DataGrid
-      rows={rows}
-      columns={columns}
-      initialState={{
-        pagination: {
-          paginationModel: { pageSize: 25, page: 0 },
-        },
-      }}
-      pageSizeOptions={[25, 50, 100]}
-      disableRowSelectionOnClick
-      sx={{ height: '100%', width: '100%' }}
-    />
+    <Box sx={{ height: '100%', width: '100%', display: 'flex', flexDirection: 'column' }}>
+      <DataGrid
+        rows={rows}
+        columns={columns}
+        autoPageSize
+        disableRowSelectionOnClick
+        hideFooterSelectedRowCount
+        sx={{
+          flexGrow: 1,
+          '& .MuiDataGrid-main': { overflow: 'unset' },
+          '& .MuiDataGrid-virtualScroller': { overflow: 'unset' },
+          '& .MuiDataGrid-footerContainer': { position: 'sticky', bottom: 0, bgcolor: 'background.paper', zIndex: 1 },
+        }}
+      />
+    </Box>
   );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
index 4114d176c77..4023c1a85d0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
@@ -1,5 +1,5 @@
-import React, {useState} from 'react';
-import {Box, Divider} from '@mui/material';
+import React, {useMemo, useState} from 'react';
+import {Box} from '@mui/material';
 
 import {SimplePageLayout} from '../common/SimplePageLayout';
 import {EditableMarkdown} from './EditableMarkdown';
@@ -12,29 +12,56 @@ type LeaderboardPageProps = {
 };
 
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
-  return (
-    <SimplePageLayout
-      title={`${props.project} Leaderboard`}
-      hideTabsIfSingle
-      tabs={[
-        {
-          label: 'Results',
-          content: (
-            <LeaderboardPageContent
-              entity={props.entity}
-              project={props.project}
-            />
-          ),
-        },
-      ]}
-    />
-  );
+    return <LeaderboardPageContent
+    entity={props.entity}
+    project={props.project}
+  />
+//   return (
+//     <SimplePageLayout
+//       title={`Leaderboard`}
+//       hideTabsIfSingle
+//       tabs={[
+//         {
+//           label: 'Results',
+//           content: (
+//             <LeaderboardPageContent
+//               entity={props.entity}
+//               project={props.project}
+//             />
+//           ),
+//         },
+//       ]}
+//     />
+//   );
 };
 
+const DEFAULT_DESCRIPTION = `# Leaderboard`;
+const EXAMPLE_DESCRIPTION = `
+# Welcome to the Leaderboard!
+
+This leaderboard showcases the performance of various models across different metrics. Here's a quick guide:
+
+## Metrics Explained
+
+- **Accuracy**: Overall correctness of the model
+- **F1 Score**: Balanced measure of precision and recall
+- **Precision**: Ratio of true positives to all positive predictions
+- **Recall**: Ratio of true positives to all actual positives
+- **AUC-ROC**: Area Under the Receiver Operating Characteristic curve
+
+## How to Interpret the Results
+
+1. Higher scores are generally better for all metrics.
+2. Look for models that perform well across *multiple* metrics.
+3. Consider the trade-offs between different metrics based on your specific use case.
+
+> Note: Click on any cell in the grid to get more detailed information about that specific score.
+
+Happy analyzing!
+`;
+
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
-  const [description, setDescription] = useState(
-    'Welcome to the Leaderboard! Compare model performance across different metrics.'
-  );
+  const [description, setDescription] = useState(EXAMPLE_DESCRIPTION);
   const [data] = useState(fakeLeaderboardData);
 
   const handleCellClick = (
@@ -48,15 +75,14 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
 
   return (
     <Box display="flex" flexDirection="column" height="100%">
-      <Box mb={2}>
+      <Box flexGrow={1} flexShrink={0} maxHeight="50%" overflow="auto">
         <EditableMarkdown
           value={description}
           onChange={setDescription}
-          placeholder="Add a description for this leaderboard..."
+          placeholder={DEFAULT_DESCRIPTION}
         />
       </Box>
-      <Divider />
-      <Box flexGrow={1} mt={2}>
+      <Box flexGrow={1} display="flex" flexDirection="column" overflow="hidden" minHeight="50%">
         <LeaderboardGrid data={data} onCellClick={handleCellClick} />
       </Box>
     </Box>

From 422636b3d828032ee8b5982b6f01ae2879800b7d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 11:21:28 -0700
Subject: [PATCH 005/169] Cursor Checkpoint 4

---
 .../Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
index 56bdc55168d..166bb14f232 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
@@ -86,8 +86,8 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         hideFooterSelectedRowCount
         sx={{
           flexGrow: 1,
-          '& .MuiDataGrid-main': { overflow: 'unset' },
-          '& .MuiDataGrid-virtualScroller': { overflow: 'unset' },
+          '& .MuiDataGrid-main': { overflow: 'auto' },
+          '& .MuiDataGrid-virtualScroller': { overflow: 'auto' },
           '& .MuiDataGrid-footerContainer': { position: 'sticky', bottom: 0, bgcolor: 'background.paper', zIndex: 1 },
         }}
       />

From f28ae1cf3fad877339025f35376432aceb23d7e2 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 11:41:03 -0700
Subject: [PATCH 006/169] Cursor Checkpoint 5

---
 .../EditableMarkdown.tsx                      |  15 +-
 .../LeaderboardGrid.tsx                       | 139 +++++++++++-------
 .../LeaderboardPage.tsx                       |  51 ++++---
 .../pages/CompareEvaluationsPage/fakeData.ts  |   9 +-
 4 files changed, 126 insertions(+), 88 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
index 0e59d68d19e..12d3262044b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
@@ -1,6 +1,6 @@
-import React, { useState, useRef, useLayoutEffect, useCallback } from 'react';
+import {Box, TextField, Typography} from '@mui/material';
+import React, {useCallback, useLayoutEffect, useRef, useState} from 'react';
 import ReactMarkdown from 'react-markdown';
-import { TextField, Typography, Box } from '@mui/material';
 
 interface EditableMarkdownProps {
   value: string;
@@ -21,7 +21,7 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
   const updateHeight = useCallback(() => {
     if (containerRef.current) {
       let newHeight: number;
-      
+
       if (isEditing && textFieldRef.current) {
         // For editing mode, use scrollHeight of the textarea
         newHeight = textFieldRef.current.scrollHeight;
@@ -76,13 +76,13 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
   };
 
   return (
-    <Box ref={containerRef} sx={{ padding: '16px', transition: 'height 0.3s' }}>
+    <Box ref={containerRef} sx={{padding: '16px', transition: 'height 0.3s'}}>
       {isEditing ? (
         <TextField
           fullWidth
           multiline
           value={value}
-          onChange={(e) => {
+          onChange={e => {
             onChange(e.target.value);
             requestAnimationFrame(() => {
               updateHeight();
@@ -93,7 +93,7 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
           autoFocus
           variant="outlined"
           inputRef={textFieldRef}
-          sx={{ 
+          sx={{
             '& .MuiOutlinedInput-root': {
               padding: 0,
               '& textarea': {
@@ -113,8 +113,7 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
               backgroundColor: 'action.hover',
             },
           }}
-          onClick={handleEdit}
-        >
+          onClick={handleEdit}>
           <ReactMarkdown>{value || placeholder}</ReactMarkdown>
         </Typography>
       )}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
index 166bb14f232..90b80430644 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
@@ -1,6 +1,9 @@
-import React, { useMemo } from 'react';
-import { DataGrid, GridColDef, GridRenderCellParams } from '@mui/x-data-grid';
-import { Box } from '@mui/material';
+import {Box} from '@mui/material';
+import {GridColDef, GridRenderCellParams} from '@mui/x-data-grid-pro';
+import React, {useCallback, useMemo} from 'react';
+
+import {StyledDataGrid} from '../../StyledDataGrid';
+import {PaginationButtons} from '../CallsPage/CallsTableButtons';
 
 interface LeaderboardGridProps {
   data: {
@@ -27,68 +30,100 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
       const scores = data.models.map(model => data.scores[model][metric]);
       ranges[metric] = {
         min: Math.min(...scores),
-        max: Math.max(...scores)
+        max: Math.max(...scores),
       };
     });
     return ranges;
   }, [data]);
 
-  const getColorForScore = (metric: string, score: number) => {
-    const { min, max } = metricRanges[metric];
-    const normalizedScore = (score - min) / (max - min);
-    return `hsl(${120 * normalizedScore}, 70%, 85%)`;
-  };
+  const getColorForScore = useCallback(
+    (metric: string, score: number) => {
+      const {min, max} = metricRanges[metric];
+      const normalizedScore = (score - min) / (max - min);
+      return `hsl(${120 * normalizedScore}, 70%, 85%)`;
+    },
+    [metricRanges]
+  );
 
-  const columns: GridColDef[] = [
-    { field: 'model', headerName: 'Model', width: 200, flex: 1 },
-    ...data.metrics.map(metric => ({
-      field: metric,
-      headerName: metric,
-      width: 130,
-      flex: 1,
-      renderCell: (params: GridRenderCellParams) => (
-        <div
-          style={{
-            width: '100%',
-            height: '100%',
-            display: 'flex',
-            alignItems: 'center',
-            justifyContent: 'center',
-            backgroundColor: getColorForScore(metric, params.value as number),
-          }}
-          onClick={() =>
-            onCellClick(
-              params.row.model,
-              params.field as string,
-              params.value as number
-            )
-          }
-        >
-          {`${(params.value as number).toFixed(2)}%`}
-        </div>
-      ),
-    })),
-  ];
+  const columns: GridColDef[] = useMemo(
+    () => [
+      {field: 'model', headerName: 'Model', width: 200, flex: 1},
+      ...data.metrics.map(metric => ({
+        field: metric,
+        headerName: metric,
+        width: 130,
+        flex: 1,
+        renderCell: (params: GridRenderCellParams) => (
+          <div
+            style={{
+              width: '100%',
+              height: '100%',
+              display: 'flex',
+              alignItems: 'center',
+              justifyContent: 'center',
+              backgroundColor: getColorForScore(metric, params.value as number),
+            }}
+            onClick={() =>
+              onCellClick(
+                params.row.model,
+                params.field as string,
+                params.value as number
+              )
+            }>
+            {`${(params.value as number).toFixed(2)}%`}
+          </div>
+        ),
+      })),
+    ],
+    [data.metrics, getColorForScore, onCellClick]
+  );
 
-  const rows: RowData[] = data.models.map((model, index) => ({
-    id: index,
-    model,
-    ...data.scores[model],
-  }));
+  const rows: RowData[] = useMemo(
+    () =>
+      data.models.map((model, index) => ({
+        id: index,
+        model,
+        ...data.scores[model],
+      })),
+    [data.models, data.scores]
+  );
 
   return (
-    <Box sx={{ height: '100%', width: '100%', display: 'flex', flexDirection: 'column' }}>
-      <DataGrid
+    <Box
+      sx={{
+        height: '100%',
+        width: '100%',
+        display: 'flex',
+        flexDirection: 'column',
+      }}>
+      <StyledDataGrid
         rows={rows}
         columns={columns}
-        autoPageSize
-        disableRowSelectionOnClick
-        hideFooterSelectedRowCount
+        pagination
+        // paginationModel={{
+        //     page: 0,
+        //     pageSize: 25,
+        // }}
+        // pageSize={pageSize}
+        // onPageSizeChange={(newPageSize) => setPageSize(newPageSize)}
+        // rowsPerPageOptions={[25]}
+        // disableSelectionOnClick
+        pageSizeOptions={[25]}
+        disableColumnMenu
         sx={{
+          borderRadius: 0,
+          // This moves the pagination controls to the left
+          '& .MuiDataGrid-footerContainer': {
+            justifyContent: 'flex-start',
+          },
+
           flexGrow: 1,
-          '& .MuiDataGrid-main': { overflow: 'auto' },
-          '& .MuiDataGrid-virtualScroller': { overflow: 'auto' },
-          '& .MuiDataGrid-footerContainer': { position: 'sticky', bottom: 0, bgcolor: 'background.paper', zIndex: 1 },
+          //   '& .MuiDataGrid-main': { overflow: 'auto' },
+          //   '& .MuiDataGrid-virtualScroller': { overflow: 'auto' },
+          //   '& .MuiDataGrid-footerContainer': { position: 'sticky', bottom: 0, bgcolor: 'background.paper', zIndex: 1 },
+        }}
+        slots={{
+          pagination: PaginationButtons,
         }}
       />
     </Box>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
index 4023c1a85d0..38cfb0f56cd 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
@@ -1,7 +1,6 @@
-import React, {useMemo, useState} from 'react';
 import {Box} from '@mui/material';
+import React, {useState} from 'react';
 
-import {SimplePageLayout} from '../common/SimplePageLayout';
 import {EditableMarkdown} from './EditableMarkdown';
 import {fakeLeaderboardData} from './fakeData';
 import {LeaderboardGrid} from './LeaderboardGrid';
@@ -12,27 +11,26 @@ type LeaderboardPageProps = {
 };
 
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
-    return <LeaderboardPageContent
-    entity={props.entity}
-    project={props.project}
-  />
-//   return (
-//     <SimplePageLayout
-//       title={`Leaderboard`}
-//       hideTabsIfSingle
-//       tabs={[
-//         {
-//           label: 'Results',
-//           content: (
-//             <LeaderboardPageContent
-//               entity={props.entity}
-//               project={props.project}
-//             />
-//           ),
-//         },
-//       ]}
-//     />
-//   );
+  return (
+    <LeaderboardPageContent entity={props.entity} project={props.project} />
+  );
+  //   return (
+  //     <SimplePageLayout
+  //       title={`Leaderboard`}
+  //       hideTabsIfSingle
+  //       tabs={[
+  //         {
+  //           label: 'Results',
+  //           content: (
+  //             <LeaderboardPageContent
+  //               entity={props.entity}
+  //               project={props.project}
+  //             />
+  //           ),
+  //         },
+  //       ]}
+  //     />
+  //   );
 };
 
 const DEFAULT_DESCRIPTION = `# Leaderboard`;
@@ -82,7 +80,12 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
           placeholder={DEFAULT_DESCRIPTION}
         />
       </Box>
-      <Box flexGrow={1} display="flex" flexDirection="column" overflow="hidden" minHeight="50%">
+      <Box
+        flexGrow={1}
+        display="flex"
+        flexDirection="column"
+        overflow="hidden"
+        minHeight="50%">
         <LeaderboardGrid data={data} onCellClick={handleCellClick} />
       </Box>
     </Box>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts
index 5e83a82f28d..172b9b68576 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts
@@ -1,10 +1,11 @@
-const generateRandomScore = () => Math.round((Math.random() * 30 + 70) * 100) / 100;
+const generateRandomScore = () =>
+  Math.round((Math.random() * 30 + 70) * 100) / 100;
 
 const metrics = ['Accuracy', 'F1 Score', 'Precision', 'Recall', 'AUC-ROC'];
-const models = Array.from({ length: 100 }, (_, i) => `Model ${i + 1}`);
+const models = Array.from({length: 200}, (_, i) => `Model ${i + 1}`);
 
 const generateScores = () => {
-  const scores: { [key: string]: number } = {};
+  const scores: {[key: string]: number} = {};
   metrics.forEach(metric => {
     scores[metric] = generateRandomScore();
   });
@@ -17,5 +18,5 @@ export const fakeLeaderboardData = {
   scores: models.reduce((acc, model) => {
     acc[model] = generateScores();
     return acc;
-  }, {} as { [key: string]: { [key: string]: number } }),
+  }, {} as {[key: string]: {[key: string]: number}}),
 };

From 784cc9d970e1305d5cdcae50c3dbd0e52e4212e1 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 11:58:40 -0700
Subject: [PATCH 007/169] Cursor Checkpoint 6

---
 .../components/FancyPage/useProjectSidebar.ts |  2 +-
 .../LeaderboardGrid.tsx                       | 43 ++++++++++++-------
 2 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index 088b0dfddbb..5b15a1a3ff4 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -135,7 +135,7 @@ export const useProjectSidebar = (
             name: 'Leaderboard',
             slug: 'weave/leaderboard',
             isShown: showWeaveSidebarItems || isShowAll,
-            iconName: IconNames.LayoutTabs,
+            iconName: IconNames.BenchmarkSquare,
           },
           {
             type: 'button' as const,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
index 90b80430644..72f1cc58270 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
@@ -1,6 +1,10 @@
 import {Box} from '@mui/material';
-import {GridColDef, GridRenderCellParams} from '@mui/x-data-grid-pro';
-import React, {useCallback, useMemo} from 'react';
+import {
+  GridColDef,
+  GridPaginationModel,
+  GridRenderCellParams,
+} from '@mui/x-data-grid-pro';
+import React, {useCallback, useMemo, useState} from 'react';
 
 import {StyledDataGrid} from '../../StyledDataGrid';
 import {PaginationButtons} from '../CallsPage/CallsTableButtons';
@@ -24,6 +28,11 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   data,
   onCellClick,
 }) => {
+  const [paginationModel, setPaginationModel] = useState<GridPaginationModel>({
+    pageSize: 25,
+    page: 0,
+  });
+
   const metricRanges = useMemo(() => {
     const ranges: {[key: string]: {min: number; max: number}} = {};
     data.metrics.forEach(metric => {
@@ -95,32 +104,36 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         width: '100%',
         display: 'flex',
         flexDirection: 'column',
+        overflow: 'hidden', // Prevent outer container from scrolling
       }}>
       <StyledDataGrid
         rows={rows}
         columns={columns}
         pagination
-        // paginationModel={{
-        //     page: 0,
-        //     pageSize: 25,
-        // }}
-        // pageSize={pageSize}
-        // onPageSizeChange={(newPageSize) => setPageSize(newPageSize)}
-        // rowsPerPageOptions={[25]}
-        // disableSelectionOnClick
+        paginationModel={paginationModel}
+        onPaginationModelChange={setPaginationModel}
         pageSizeOptions={[25]}
         disableColumnMenu
+        disableRowSelectionOnClick
         sx={{
           borderRadius: 0,
-          // This moves the pagination controls to the left
           '& .MuiDataGrid-footerContainer': {
             justifyContent: 'flex-start',
           },
-
+          '& .MuiDataGrid-cell': {
+            cursor: 'pointer',
+          },
           flexGrow: 1,
-          //   '& .MuiDataGrid-main': { overflow: 'auto' },
-          //   '& .MuiDataGrid-virtualScroller': { overflow: 'auto' },
-          //   '& .MuiDataGrid-footerContainer': { position: 'sticky', bottom: 0, bgcolor: 'background.paper', zIndex: 1 },
+          width: 'calc(100% + 1px)', // Add 1px to account for the right border
+          '& .MuiDataGrid-main': {
+            overflow: 'hidden',
+          },
+          '& .MuiDataGrid-virtualScroller': {
+            overflow: 'auto',
+          },
+          '& .MuiDataGrid-columnHeaders': {
+            overflow: 'hidden',
+          },
         }}
         slots={{
           pagination: PaginationButtons,

From dd019e91f464bbac5a2dfb17addd9ae2f284d018 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 12:09:23 -0700
Subject: [PATCH 008/169] Cursor Checkpoint 7

---
 .../components/FancyPage/useProjectSidebar.ts |  1 +
 .../PagePanelComponents/Home/Browse3.tsx      |  2 +-
 .../LeaderboardGrid.tsx                       |  0
 .../LeaderboardHeader.tsx                     |  0
 .../LeaderboardPage.tsx                       | 33 +++++--------------
 5 files changed, 11 insertions(+), 25 deletions(-)
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/{CompareEvaluationsPage => LeaderboardPage}/LeaderboardGrid.tsx (100%)
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/{CompareEvaluationsPage => LeaderboardPage}/LeaderboardHeader.tsx (100%)
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/{CompareEvaluationsPage => LeaderboardPage}/LeaderboardPage.tsx (77%)

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index 5b15a1a3ff4..258a251ceb9 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -31,6 +31,7 @@ export const useProjectSidebar = (
   const isNoSidebarItems = !showModelsSidebarItems && !showWeaveSidebarItems;
   const isBothSidebarItems = showModelsSidebarItems && showWeaveSidebarItems;
   const isShowAll = isNoSidebarItems || isBothSidebarItems;
+
   return useMemo(() => {
     const allItems = isLoading
       ? []
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
index 66151840d25..d80f60b1632 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
@@ -82,7 +82,7 @@ import {Empty} from './Browse3/pages/common/Empty';
 import {EMPTY_NO_TRACE_SERVER} from './Browse3/pages/common/EmptyContent';
 import {SimplePageLayoutContext} from './Browse3/pages/common/SimplePageLayout';
 import {CompareEvaluationsPage} from './Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage';
-import {LeaderboardPage} from './Browse3/pages/CompareEvaluationsPage/LeaderboardPage';
+import {LeaderboardPage} from './Browse3/pages/LeaderboardPage/LeaderboardPage';
 import {ObjectPage} from './Browse3/pages/ObjectPage';
 import {ObjectVersionPage} from './Browse3/pages/ObjectVersionPage';
 import {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
similarity index 100%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardGrid.tsx
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardHeader.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardHeader.tsx
similarity index 100%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardHeader.tsx
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardHeader.tsx
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
similarity index 77%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 38cfb0f56cd..ddbde825148 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,8 +1,8 @@
 import {Box} from '@mui/material';
-import React, {useState} from 'react';
+import React, {useCallback, useState} from 'react';
 
-import {EditableMarkdown} from './EditableMarkdown';
-import {fakeLeaderboardData} from './fakeData';
+import {EditableMarkdown} from '../CompareEvaluationsPage/EditableMarkdown';
+import {fakeLeaderboardData} from '../CompareEvaluationsPage/fakeData';
 import {LeaderboardGrid} from './LeaderboardGrid';
 
 type LeaderboardPageProps = {
@@ -14,28 +14,10 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
   return (
     <LeaderboardPageContent entity={props.entity} project={props.project} />
   );
-  //   return (
-  //     <SimplePageLayout
-  //       title={`Leaderboard`}
-  //       hideTabsIfSingle
-  //       tabs={[
-  //         {
-  //           label: 'Results',
-  //           content: (
-  //             <LeaderboardPageContent
-  //               entity={props.entity}
-  //               project={props.project}
-  //             />
-  //           ),
-  //         },
-  //       ]}
-  //     />
-  //   );
 };
 
 const DEFAULT_DESCRIPTION = `# Leaderboard`;
-const EXAMPLE_DESCRIPTION = `
-# Welcome to the Leaderboard!
+const EXAMPLE_DESCRIPTION = `# Welcome to the Leaderboard!
 
 This leaderboard showcases the performance of various models across different metrics. Here's a quick guide:
 
@@ -55,13 +37,16 @@ This leaderboard showcases the performance of various models across different me
 
 > Note: Click on any cell in the grid to get more detailed information about that specific score.
 
-Happy analyzing!
-`;
+Happy analyzing!`;
 
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   const [description, setDescription] = useState(EXAMPLE_DESCRIPTION);
   const [data] = useState(fakeLeaderboardData);
 
+  // const setDescription = useCallback((newDescription: string) => {
+  //   setDescriptionRaw(newDescription.trim());
+  // }, []);
+
   const handleCellClick = (
     modelName: string,
     metricName: string,

From 4beb237bac8030b50b174e26f7abcbda000fb365 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 12:13:04 -0700
Subject: [PATCH 009/169] Cleanup

---
 .../EditableMarkdown.tsx                                    | 0
 .../Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx  | 6 +++---
 .../{CompareEvaluationsPage => LeaderboardPage}/fakeData.ts | 0
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/{CompareEvaluationsPage => LeaderboardPage}/EditableMarkdown.tsx (100%)
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/{CompareEvaluationsPage => LeaderboardPage}/fakeData.ts (100%)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
similarity index 100%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/EditableMarkdown.tsx
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index ddbde825148..b76df554a03 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,8 +1,8 @@
 import {Box} from '@mui/material';
-import React, {useCallback, useState} from 'react';
+import React, {useState} from 'react';
 
-import {EditableMarkdown} from '../CompareEvaluationsPage/EditableMarkdown';
-import {fakeLeaderboardData} from '../CompareEvaluationsPage/fakeData';
+import {EditableMarkdown} from './EditableMarkdown';
+import {fakeLeaderboardData} from './fakeData';
 import {LeaderboardGrid} from './LeaderboardGrid';
 
 type LeaderboardPageProps = {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/fakeData.ts
similarity index 100%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CompareEvaluationsPage/fakeData.ts
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/fakeData.ts

From 52376501389e226583cb961bd1c06a0898057471 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 13:23:32 -0700
Subject: [PATCH 010/169] Human: Starting to extract real data from the system

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx |  25 +--
 .../pages/LeaderboardPage/LeaderboardPage.tsx |   4 +-
 .../Browse3/pages/LeaderboardPage/hooks.ts    | 156 ++++++++++++++++++
 .../Home/Browse3/pages/ObjectVersionPage.tsx  |   1 +
 .../pages/common/TypeVersionCategoryChip.tsx  |   1 +
 .../pages/wfReactInterface/constants.ts       |   6 +-
 6 files changed, 180 insertions(+), 13 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 72f1cc58270..4ca1107bc74 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -8,13 +8,10 @@ import React, {useCallback, useMemo, useState} from 'react';
 
 import {StyledDataGrid} from '../../StyledDataGrid';
 import {PaginationButtons} from '../CallsPage/CallsTableButtons';
+import {LeaderboardData} from './hooks';
 
 interface LeaderboardGridProps {
-  data: {
-    models: string[];
-    metrics: string[];
-    scores: {[key: string]: {[key: string]: number}};
-  };
+  data: LeaderboardData;
   onCellClick: (modelName: string, metricName: string, score: number) => void;
 }
 
@@ -35,8 +32,10 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
 
   const metricRanges = useMemo(() => {
     const ranges: {[key: string]: {min: number; max: number}} = {};
-    data.metrics.forEach(metric => {
-      const scores = data.models.map(model => data.scores[model][metric]);
+    Object.keys(data.metrics).forEach(metric => {
+      const scores = data.models
+        .map(model => data.scores?.[model]?.[metric])
+        .filter(score => score !== undefined);
       ranges[metric] = {
         min: Math.min(...scores),
         max: Math.max(...scores),
@@ -46,7 +45,10 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   }, [data]);
 
   const getColorForScore = useCallback(
-    (metric: string, score: number) => {
+    (metric: string, score: number | undefined) => {
+      if (score === undefined) {
+        return 'transparent';
+      }
       const {min, max} = metricRanges[metric];
       const normalizedScore = (score - min) / (max - min);
       return `hsl(${120 * normalizedScore}, 70%, 85%)`;
@@ -57,7 +59,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   const columns: GridColDef[] = useMemo(
     () => [
       {field: 'model', headerName: 'Model', width: 200, flex: 1},
-      ...data.metrics.map(metric => ({
+      ...Object.keys(data.metrics).map(metric => ({
         field: metric,
         headerName: metric,
         width: 130,
@@ -70,7 +72,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
               display: 'flex',
               alignItems: 'center',
               justifyContent: 'center',
-              backgroundColor: getColorForScore(metric, params.value as number),
+              backgroundColor: getColorForScore(metric, params.value),
             }}
             onClick={() =>
               onCellClick(
@@ -79,7 +81,8 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
                 params.value as number
               )
             }>
-            {`${(params.value as number).toFixed(2)}%`}
+            {params.value ?? '-'}
+            {/* {`${(params.value as number).toFixed(2)}%`} */}
           </div>
         ),
       })),
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index b76df554a03..06fa9e444eb 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -3,6 +3,7 @@ import React, {useState} from 'react';
 
 import {EditableMarkdown} from './EditableMarkdown';
 import {fakeLeaderboardData} from './fakeData';
+import {useLeaderboardData} from './hooks';
 import {LeaderboardGrid} from './LeaderboardGrid';
 
 type LeaderboardPageProps = {
@@ -40,8 +41,9 @@ This leaderboard showcases the performance of various models across different me
 Happy analyzing!`;
 
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
+  const {entity, project} = props;
   const [description, setDescription] = useState(EXAMPLE_DESCRIPTION);
-  const [data] = useState(fakeLeaderboardData);
+  const data = useLeaderboardData(entity, project);
 
   // const setDescription = useCallback((newDescription: string) => {
   //   setDescriptionRaw(newDescription.trim());
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
new file mode 100644
index 00000000000..21513e8d192
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -0,0 +1,156 @@
+import {useMemo, useState} from 'react';
+
+import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
+import {parseRefMaybe} from '../../../Browse2/SmallRef';
+import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
+import {useWFHooks} from '../wfReactInterface/context';
+import {
+  objectVersionKeyToRefUri,
+  opVersionKeyToRefUri,
+} from '../wfReactInterface/utilities';
+import {fakeLeaderboardData} from './fakeData';
+
+export type LeaderboardData = {
+  metrics: {
+    [metricId: string]: {
+      evaluationName: string;
+      metricPath: string;
+    };
+  };
+  models: string[];
+  scores: {[modelId: string]: {[metricId: string]: number}};
+};
+
+const useEvaluations = (
+  entity: string,
+  project: string
+): {loading: boolean; data: LeaderboardData} => {
+  const {useRootObjectVersions, useCalls} = useWFHooks();
+
+  // Get the last 100 (latest) evaluation versions
+  const evaluationVersions = useRootObjectVersions(
+    entity,
+    project,
+    {
+      baseObjectClasses: ['Evaluation'],
+      latestOnly: true,
+    },
+    100,
+    true
+  );
+
+  // Unfortunately, the eval framework does not build models automatically!!
+  // const modelVersions = useRootObjectVersions(entity, project, {
+  //     baseObjectClasses: ['Model'],
+  //     latestOnly: true,
+  // }, 100, true)
+
+  // Get the runs for these evaluation versions.
+  const evaluationVersionsResult = evaluationVersions?.result;
+  const evaluationRuns = useCalls(
+    entity,
+    project,
+    {
+      opVersionRefs: [
+        opVersionKeyToRefUri({
+          entity,
+          project,
+          opId: EVALUATE_OP_NAME_POST_PYDANTIC,
+          versionHash: '*',
+        }),
+      ],
+      traceRootsOnly: true,
+      inputObjectVersionRefs: (evaluationVersionsResult ?? []).map(version =>
+        objectVersionKeyToRefUri(version)
+      ),
+    },
+    100,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    {skip: !evaluationVersionsResult}
+  );
+
+  // Build the dataset
+
+  const results: {loading: boolean; data: LeaderboardData} = useMemo(() => {
+    const finalData: LeaderboardData = {
+      metrics: {},
+      models: [],
+      scores: {},
+    };
+    if (evaluationVersions.loading || evaluationRuns.loading) {
+      return {
+        loading: true,
+        data: finalData,
+      };
+    } else if (!evaluationRuns.result) {
+      // || !evaluationVersionsResult) {
+      return {
+        loading: false,
+        data: finalData,
+      };
+    }
+
+    const runs = evaluationRuns.result;
+    // const versions = evaluationVersionsResult
+    runs.forEach(r => {
+      const modelName = r.traceCall?.inputs.model;
+      if (!modelName) {
+        return;
+      }
+      const evaluationVersion = r.traceCall?.inputs.self;
+      if (!evaluationVersion) {
+        return;
+      }
+      const evalName = parseRefMaybe(evaluationVersion)?.artifactName;
+      if (!evalName) {
+        return;
+      }
+      const outputSummary = r.traceCall?.output;
+      if (!outputSummary) {
+        return;
+      }
+      if (!finalData.models.includes(modelName)) {
+        finalData.models.push(modelName);
+      }
+      const modelScores = flattenObjectPreservingWeaveTypes(
+        outputSummary ?? {}
+      );
+      Object.entries(modelScores).forEach(([metric, score]) => {
+        const metricName = `${evalName}.${metric}`;
+        if (!finalData.metrics[metricName]) {
+          finalData.metrics[metricName] = {
+            evaluationName: evalName,
+            metricPath: metric,
+          };
+        }
+        if (!finalData.scores[modelName]) {
+          finalData.scores[modelName] = {};
+        }
+        finalData.scores[modelName][metric] = score;
+      });
+    });
+
+    return {
+      loading: false,
+      data: finalData,
+    };
+  }, [
+    evaluationRuns.loading,
+    evaluationRuns.result,
+    evaluationVersions.loading,
+  ]);
+
+  return results;
+};
+
+export const useLeaderboardData = (
+  entity: string,
+  project: string
+): LeaderboardData => {
+  const {loading, data} = useEvaluations(entity, project);
+  return data;
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx
index 28002ee9bde..ad5825bb962 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx
@@ -45,6 +45,7 @@ type ObjectIconProps = {
 const OBJECT_ICONS: Record<KnownBaseObjectClassType, IconName> = {
   Model: 'model',
   Dataset: 'table',
+  Evaluation: 'benchmark-square',
 };
 const ObjectIcon = ({baseObjectClass}: ObjectIconProps) => {
   if (baseObjectClass in OBJECT_ICONS) {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/TypeVersionCategoryChip.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/TypeVersionCategoryChip.tsx
index 89640b56df5..a1c366fd1da 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/TypeVersionCategoryChip.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/TypeVersionCategoryChip.tsx
@@ -6,6 +6,7 @@ import {KnownBaseObjectClassType} from '../wfReactInterface/wfDataModelHooksInte
 const colorMap: Record<KnownBaseObjectClassType, TagColorName> = {
   Model: 'blue',
   Dataset: 'green',
+  Evaluation: 'cactus',
 };
 
 export const TypeVersionCategoryChip: React.FC<{
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/constants.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/constants.ts
index b413b82e001..8b5aca318c9 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/constants.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/constants.ts
@@ -19,4 +19,8 @@ export const OP_CATEGORIES = [
   'evaluate',
   'tune',
 ] as const;
-export const KNOWN_BASE_OBJECT_CLASSES = ['Model', 'Dataset'] as const;
+export const KNOWN_BASE_OBJECT_CLASSES = [
+  'Model',
+  'Dataset',
+  'Evaluation',
+] as const;

From 6f6f75da01ed2c9c94deabb24a7d3c16eb442dfc Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 13:25:31 -0700
Subject: [PATCH 011/169] Human: Starting to extract real data from the system

---
 .../Home/Browse3/pages/LeaderboardPage/hooks.ts                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index 21513e8d192..ec4c1966ae0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -130,7 +130,7 @@ const useEvaluations = (
         if (!finalData.scores[modelName]) {
           finalData.scores[modelName] = {};
         }
-        finalData.scores[modelName][metric] = score;
+        finalData.scores[modelName][metricName] = score;
       });
     });
 

From c2714d5646e8009be6a8ad10cac7b6655ec4be2c Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 13:47:19 -0700
Subject: [PATCH 012/169] Human: Starting to extract real data from the system

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx | 73 +++++++++++++------
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  1 -
 .../Browse3/pages/LeaderboardPage/hooks.ts    |  5 +-
 3 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 4ca1107bc74..59e5b294b59 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -1,6 +1,7 @@
 import {Box} from '@mui/material';
 import {
   GridColDef,
+  GridColumnGroupingModel,
   GridPaginationModel,
   GridRenderCellParams,
 } from '@mui/x-data-grid-pro';
@@ -64,27 +65,39 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         headerName: metric,
         width: 130,
         flex: 1,
-        renderCell: (params: GridRenderCellParams) => (
-          <div
-            style={{
-              width: '100%',
-              height: '100%',
-              display: 'flex',
-              alignItems: 'center',
-              justifyContent: 'center',
-              backgroundColor: getColorForScore(metric, params.value),
-            }}
-            onClick={() =>
-              onCellClick(
-                params.row.model,
-                params.field as string,
-                params.value as number
-              )
-            }>
-            {params.value ?? '-'}
-            {/* {`${(params.value as number).toFixed(2)}%`} */}
-          </div>
-        ),
+        renderCell: (params: GridRenderCellParams) => {
+          let inner = params.value;
+          if (typeof inner === 'number') {
+            if (inner < 1) {
+              inner = `${(inner * 100).toFixed(2)}%`;
+            } else {
+              inner = `${inner.toFixed(2)}`;
+            }
+          }
+          // const value =
+          // <CellValue value={params.value} />
+          return (
+            <div
+              style={{
+                width: '100%',
+                height: '100%',
+                display: 'flex',
+                alignItems: 'center',
+                justifyContent: 'center',
+                backgroundColor: getColorForScore(metric, params.value),
+              }}
+              onClick={() =>
+                onCellClick(
+                  params.row.model,
+                  params.field as string,
+                  params.value as number
+                )
+              }>
+              {' '}
+              {inner}
+            </div>
+          );
+        },
       })),
     ],
     [data.metrics, getColorForScore, onCellClick]
@@ -100,6 +113,20 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     [data.models, data.scores]
   );
 
+  const groupingModel: GridColumnGroupingModel = useMemo(
+    () => {
+      return [
+        {
+          groupId: 'metrics',
+          children: Object.keys(data.metrics).map(metric => ({
+            field: metric,
+          })),
+        },
+      ];
+    },
+    [data.metrics]
+  );
+
   return (
     <Box
       sx={{
@@ -112,12 +139,13 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
       <StyledDataGrid
         rows={rows}
         columns={columns}
+        columnGroupingModel={groupingModel}
         pagination
         paginationModel={paginationModel}
         onPaginationModelChange={setPaginationModel}
         pageSizeOptions={[25]}
-        disableColumnMenu
         disableRowSelectionOnClick
+        hideFooterSelectedRowCount
         sx={{
           borderRadius: 0,
           '& .MuiDataGrid-footerContainer': {
@@ -139,6 +167,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
           },
         }}
         slots={{
+          // columnMenu: CallsCustomColumnMenu,
           pagination: PaginationButtons,
         }}
       />
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 06fa9e444eb..5bb4df0d2de 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -2,7 +2,6 @@ import {Box} from '@mui/material';
 import React, {useState} from 'react';
 
 import {EditableMarkdown} from './EditableMarkdown';
-import {fakeLeaderboardData} from './fakeData';
 import {useLeaderboardData} from './hooks';
 import {LeaderboardGrid} from './LeaderboardGrid';
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index ec4c1966ae0..b001ad96c98 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -1,4 +1,4 @@
-import {useMemo, useState} from 'react';
+import {useMemo} from 'react';
 
 import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
 import {parseRefMaybe} from '../../../Browse2/SmallRef';
@@ -8,7 +8,6 @@ import {
   objectVersionKeyToRefUri,
   opVersionKeyToRefUri,
 } from '../wfReactInterface/utilities';
-import {fakeLeaderboardData} from './fakeData';
 
 export type LeaderboardData = {
   metrics: {
@@ -33,7 +32,7 @@ const useEvaluations = (
     project,
     {
       baseObjectClasses: ['Evaluation'],
-      latestOnly: true,
+      // latestOnly: true,
     },
     100,
     true

From 91f56eaf82256c7e8413441ea98049ab3ade08d5 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 15:18:09 -0700
Subject: [PATCH 013/169] Human: Ok, getting to a checkpoint

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx | 167 ++++++++++++++----
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  54 +++---
 .../Browse3/pages/LeaderboardPage/hooks.ts    |  21 ++-
 .../Home/Browse3/pages/common/Links.tsx       |   1 +
 4 files changed, 171 insertions(+), 72 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 59e5b294b59..fd6c28795ec 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -1,29 +1,35 @@
 import {Box} from '@mui/material';
 import {
   GridColDef,
+  GridColumnGroup,
   GridColumnGroupingModel,
+  GridColumnNode,
   GridPaginationModel,
   GridRenderCellParams,
+  GridSortDirection,
 } from '@mui/x-data-grid-pro';
 import React, {useCallback, useMemo, useState} from 'react';
 
+import {parseRefMaybe, SmallRef} from '../../../Browse2/SmallRef';
 import {StyledDataGrid} from '../../StyledDataGrid';
 import {PaginationButtons} from '../CallsPage/CallsTableButtons';
+import {buildTree} from '../common/tabularListViews/buildTree';
 import {LeaderboardData} from './hooks';
 
 interface LeaderboardGridProps {
   data: LeaderboardData;
+  loading: boolean;
   onCellClick: (modelName: string, metricName: string, score: number) => void;
 }
 
-interface RowData {
+type RowData = {
   id: number;
   model: string;
-  [key: string]: number | string;
-}
+} & LeaderboardData['scores'][string];
 
 export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   data,
+  loading,
   onCellClick,
 }) => {
   const [paginationModel, setPaginationModel] = useState<GridPaginationModel>({
@@ -31,11 +37,18 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     page: 0,
   });
 
+  const orderedMetrics = useMemo(() => {
+    return Object.keys(data.metrics);
+    // return Object.keys(data.metrics).sort((a, b) => {
+    //   return a.localeCompare(b);
+    // });
+  }, [data.metrics]);
+
   const metricRanges = useMemo(() => {
     const ranges: {[key: string]: {min: number; max: number}} = {};
-    Object.keys(data.metrics).forEach(metric => {
+    orderedMetrics.forEach(metric => {
       const scores = data.models
-        .map(model => data.scores?.[model]?.[metric])
+        .map(model => data.scores?.[model]?.[metric]?.value)
         .filter(score => score !== undefined);
       ranges[metric] = {
         min: Math.min(...scores),
@@ -43,7 +56,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
       };
     });
     return ranges;
-  }, [data]);
+  }, [data.models, data.scores, orderedMetrics]);
 
   const getColorForScore = useCallback(
     (metric: string, score: number | undefined) => {
@@ -57,14 +70,74 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     [metricRanges]
   );
 
-  const columns: GridColDef[] = useMemo(
+  const rows: RowData[] = useMemo(
+    () =>
+      data.models.map((model, index) => ({
+        id: index,
+        model,
+        ...data.scores[model],
+      })) as RowData[],
+    [data.models, data.scores]
+  );
+
+  const columns: Array<GridColDef<RowData>> = useMemo(
     () => [
-      {field: 'model', headerName: 'Model', width: 200, flex: 1},
-      ...Object.keys(data.metrics).map(metric => ({
+      {
+        field: 'model',
+        headerName: 'Model',
+        minWidth: 200,
+        flex: 1,
+        renderCell: (params: GridRenderCellParams) => {
+          const modelRef = parseRefMaybe(params.value);
+          if (modelRef) {
+            return (
+              <div
+                style={{
+                  width: '100%',
+                  height: '100%',
+                  alignContent: 'center',
+                  display: 'flex',
+                  alignItems: 'center',
+                  justifyContent: 'center',
+                  lineHeight: '20px',
+                }}>
+                <SmallRef objRef={modelRef} />
+              </div>
+            );
+          }
+          return <div>{params.value}</div>;
+        },
+      },
+      ...orderedMetrics.map(metric => ({
         field: metric,
-        headerName: metric,
-        width: 130,
+        headerName: metric.split('.').pop(),
+        minWidth: 130,
         flex: 1,
+        valueGetter: (value: RowData) => {
+          return value?.value;
+        },
+        getSortComparator: (dir: GridSortDirection) => (a: any, b: any) => {
+          const aValue = a;
+          const bValue = b;
+          if (aValue == null && bValue == null) {
+            return 0;
+          }
+          // Ignoring direction here allows nulls to always sort to the end
+          if (aValue == null) {
+            return 1;
+          }
+          if (bValue == null) {
+            return -1;
+          }
+          if (typeof aValue === 'number' && typeof bValue === 'number') {
+            if (dir === 'asc') {
+              return aValue - bValue;
+            } else {
+              return bValue - aValue;
+            }
+          }
+          return aValue.localeCompare(bValue);
+        },
         renderCell: (params: GridRenderCellParams) => {
           let inner = params.value;
           if (typeof inner === 'number') {
@@ -100,32 +173,43 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         },
       })),
     ],
-    [data.metrics, getColorForScore, onCellClick]
+    [getColorForScore, onCellClick, orderedMetrics]
   );
 
-  const rows: RowData[] = useMemo(
-    () =>
-      data.models.map((model, index) => ({
-        id: index,
-        model,
-        ...data.scores[model],
-      })),
-    [data.models, data.scores]
-  );
+  const tree = buildTree([...Object.keys(data.metrics)]);
+  let groupingModel: GridColumnGroupingModel = tree.children.filter(
+    c => 'groupId' in c
+  ) as GridColumnGroup[];
+  groupingModel = walkGroupingModel(groupingModel, node => {
+    if ('groupId' in node) {
+      if (node.children.length === 1) {
+        if ('groupId' in node.children[0]) {
+          const currNode = node;
+          node = node.children[0];
+          node.headerName = currNode.headerName + '.' + node.headerName;
+        } else {
+          // pass
+          // node = node.children[0];
+        }
+      }
+    }
+    return node;
+  }) as GridColumnGroup[];
+  console.log(groupingModel);
 
-  const groupingModel: GridColumnGroupingModel = useMemo(
-    () => {
-      return [
-        {
-          groupId: 'metrics',
-          children: Object.keys(data.metrics).map(metric => ({
-            field: metric,
-          })),
-        },
-      ];
-    },
-    [data.metrics]
-  );
+  // const groupingModel: GridColumnGroupingModel = useMemo(
+  //   () => {
+  //     return [
+  //       {
+  //         groupId: 'metrics',
+  //         children: Object.keys(data.metrics).map(metric => ({
+  //           field: metric,
+  //         })),
+  //       },
+  //     ];
+  //   },
+  //   [data.metrics]
+  // );
 
   return (
     <Box
@@ -146,6 +230,9 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         pageSizeOptions={[25]}
         disableRowSelectionOnClick
         hideFooterSelectedRowCount
+        disableMultipleColumnsSorting={false}
+        columnHeaderHeight={40}
+        loading={loading}
         sx={{
           borderRadius: 0,
           '& .MuiDataGrid-footerContainer': {
@@ -174,3 +261,15 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     </Box>
   );
 };
+
+const walkGroupingModel = (
+  nodes: GridColumnNode[],
+  fn: (node: GridColumnNode) => GridColumnNode
+) => {
+  return nodes.map(node => {
+    if ('children' in node) {
+      node.children = walkGroupingModel(node.children, fn);
+    }
+    return fn(node);
+  });
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 5bb4df0d2de..b267371f939 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,10 +1,14 @@
 import {Box} from '@mui/material';
 import React, {useState} from 'react';
+import {useHistory} from 'react-router-dom';
 
+import {useWeaveflowRouteContext} from '../../context';
 import {EditableMarkdown} from './EditableMarkdown';
 import {useLeaderboardData} from './hooks';
 import {LeaderboardGrid} from './LeaderboardGrid';
 
+const USE_COMPARE_EVALUATIONS_PAGE = false;
+
 type LeaderboardPageProps = {
   entity: string;
   project: string;
@@ -17,49 +21,41 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
 };
 
 const DEFAULT_DESCRIPTION = `# Leaderboard`;
-const EXAMPLE_DESCRIPTION = `# Welcome to the Leaderboard!
-
-This leaderboard showcases the performance of various models across different metrics. Here's a quick guide:
-
-## Metrics Explained
-
-- **Accuracy**: Overall correctness of the model
-- **F1 Score**: Balanced measure of precision and recall
-- **Precision**: Ratio of true positives to all positive predictions
-- **Recall**: Ratio of true positives to all actual positives
-- **AUC-ROC**: Area Under the Receiver Operating Characteristic curve
-
-## How to Interpret the Results
-
-1. Higher scores are generally better for all metrics.
-2. Look for models that perform well across *multiple* metrics.
-3. Consider the trade-offs between different metrics based on your specific use case.
-
-> Note: Click on any cell in the grid to get more detailed information about that specific score.
-
-Happy analyzing!`;
 
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   const {entity, project} = props;
-  const [description, setDescription] = useState(EXAMPLE_DESCRIPTION);
-  const data = useLeaderboardData(entity, project);
+  const [description, setDescription] = useState("");
+  const {loading, data} = useLeaderboardData(entity, project);
 
   // const setDescription = useCallback((newDescription: string) => {
   //   setDescriptionRaw(newDescription.trim());
   // }, []);
 
+  const {peekingRouter} = useWeaveflowRouteContext();
+  const history = useHistory();
+
   const handleCellClick = (
     modelName: string,
     metricName: string,
     score: number
   ) => {
-    console.log(`Clicked on ${modelName} for ${metricName}: ${score}%`);
-    // TODO: Implement action on cell click
+    const sourceCallId = data.scores?.[modelName]?.[metricName]?.sourceCallId;
+    if (sourceCallId) {
+      let to: string;
+      if (USE_COMPARE_EVALUATIONS_PAGE) {
+        to = peekingRouter.compareEvaluationsUri(entity, project, [
+          sourceCallId,
+        ]);
+      } else {
+        to = peekingRouter.callUIUrl(entity, project, '', sourceCallId, null);
+      }
+      history.push(to);
+    }
   };
 
   return (
     <Box display="flex" flexDirection="column" height="100%">
-      <Box flexGrow={1} flexShrink={0} maxHeight="50%" overflow="auto">
+      <Box flexShrink={0} maxHeight="50%" overflow="auto">
         <EditableMarkdown
           value={description}
           onChange={setDescription}
@@ -72,7 +68,11 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
         flexDirection="column"
         overflow="hidden"
         minHeight="50%">
-        <LeaderboardGrid data={data} onCellClick={handleCellClick} />
+        <LeaderboardGrid
+          loading={loading}
+          data={data}
+          onCellClick={handleCellClick}
+        />
       </Box>
     </Box>
   );
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index b001ad96c98..4c0fa63202a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -17,10 +17,14 @@ export type LeaderboardData = {
     };
   };
   models: string[];
-  scores: {[modelId: string]: {[metricId: string]: number}};
+  scores: {
+    [modelId: string]: {
+      [metricId: string]: {value: number; sourceCallId: string};
+    };
+  };
 };
 
-const useEvaluations = (
+export const useLeaderboardData = (
   entity: string,
   project: string
 ): {loading: boolean; data: LeaderboardData} => {
@@ -129,7 +133,10 @@ const useEvaluations = (
         if (!finalData.scores[modelName]) {
           finalData.scores[modelName] = {};
         }
-        finalData.scores[modelName][metricName] = score;
+        finalData.scores[modelName][metricName] = {
+          value: score,
+          sourceCallId: r.callId,
+        };
       });
     });
 
@@ -145,11 +152,3 @@ const useEvaluations = (
 
   return results;
 };
-
-export const useLeaderboardData = (
-  entity: string,
-  project: string
-): LeaderboardData => {
-  const {loading, data} = useEvaluations(entity, project);
-  return data;
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
index 1cc0b7ee68a..b33d5f8d125 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
@@ -271,6 +271,7 @@ export const CallLink: React.FC<{
 }> = props => {
   const history = useHistory();
   const {peekingRouter} = useWeaveflowRouteContext();
+
   const opName = opNiceName(props.opName);
 
   // Custom logic to calculate path and tracetree here is not good. Shows

From a38694596921f23ae4da264722aae1dc5bde2299 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 15:40:49 -0700
Subject: [PATCH 014/169] Human: Cleanup

---
 .../LeaderboardPage/LeaderboardHeader.tsx     | 26 -------------------
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  2 +-
 .../Browse3/pages/LeaderboardPage/fakeData.ts | 22 ----------------
 3 files changed, 1 insertion(+), 49 deletions(-)
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardHeader.tsx
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/fakeData.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardHeader.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardHeader.tsx
deleted file mode 100644
index acf5a0af928..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardHeader.tsx
+++ /dev/null
@@ -1,26 +0,0 @@
-import React from 'react';
-import styled from 'styled-components';
-
-interface LeaderboardHeaderProps {
-  title: string;
-}
-
-export const LeaderboardHeader: React.FC<LeaderboardHeaderProps> = ({
-  title,
-}) => (
-  <HeaderContainer>
-    <Title>{title}</Title>
-  </HeaderContainer>
-);
-
-const HeaderContainer = styled.div`
-  background-color: #2c3e50;
-  color: white;
-  padding: 24px;
-  border-radius: 8px;
-`;
-
-const Title = styled.h1`
-  margin: 0;
-  font-size: 28px;
-`;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index b267371f939..838b3128b80 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -24,7 +24,7 @@ const DEFAULT_DESCRIPTION = `# Leaderboard`;
 
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   const {entity, project} = props;
-  const [description, setDescription] = useState("");
+  const [description, setDescription] = useState('');
   const {loading, data} = useLeaderboardData(entity, project);
 
   // const setDescription = useCallback((newDescription: string) => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/fakeData.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/fakeData.ts
deleted file mode 100644
index 172b9b68576..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/fakeData.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-const generateRandomScore = () =>
-  Math.round((Math.random() * 30 + 70) * 100) / 100;
-
-const metrics = ['Accuracy', 'F1 Score', 'Precision', 'Recall', 'AUC-ROC'];
-const models = Array.from({length: 200}, (_, i) => `Model ${i + 1}`);
-
-const generateScores = () => {
-  const scores: {[key: string]: number} = {};
-  metrics.forEach(metric => {
-    scores[metric] = generateRandomScore();
-  });
-  return scores;
-};
-
-export const fakeLeaderboardData = {
-  models,
-  metrics,
-  scores: models.reduce((acc, model) => {
-    acc[model] = generateScores();
-    return acc;
-  }, {} as {[key: string]: {[key: string]: number}}),
-};

From 35fed9148fb41c747d944291bb43ef267133ddb1 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 17:07:08 -0700
Subject: [PATCH 015/169] Human: more work

---
 .../components/FancyPage/useProjectSidebar.ts |  6 ++-
 .../pages/LeaderboardPage/LeaderboardGrid.tsx |  1 -
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 51 +++++++++++++++++--
 3 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index 258a251ceb9..cf0207d771f 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -10,7 +10,8 @@ export const useProjectSidebar = (
   hasModelsData: boolean,
   hasWeaveData: boolean,
   hasTraceBackend: boolean = true,
-  hasModelsAccess: boolean = true
+  hasModelsAccess: boolean = true,
+  viewerIsAdmin: boolean = false,
 ): FancyPageSidebarItem[] => {
   // Should show models sidebar items if we have models data or if we don't have a trace backend
   let showModelsSidebarItems = hasModelsData || !hasTraceBackend;
@@ -135,7 +136,8 @@ export const useProjectSidebar = (
             type: 'button' as const,
             name: 'Leaderboard',
             slug: 'weave/leaderboard',
-            isShown: showWeaveSidebarItems || isShowAll,
+            // TODO: remove this `true`
+            isShown: (viewerIsAdmin || true) && (showWeaveSidebarItems || isShowAll),
             iconName: IconNames.BenchmarkSquare,
           },
           {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index fd6c28795ec..393e2fe525b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -195,7 +195,6 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     }
     return node;
   }) as GridColumnGroup[];
-  console.log(groupingModel);
 
   // const groupingModel: GridColumnGroupingModel = useMemo(
   //   () => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 838b3128b80..720a3218e0a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,11 +1,13 @@
 import {Box} from '@mui/material';
-import React, {useState} from 'react';
+
+import React, {useContext, useEffect, useState} from 'react';
 import {useHistory} from 'react-router-dom';
 
-import {useWeaveflowRouteContext} from '../../context';
+import {usePeekLocation, useWeaveflowRouteContext, WeaveflowPeekContext} from '../../context';
 import {EditableMarkdown} from './EditableMarkdown';
 import {useLeaderboardData} from './hooks';
 import {LeaderboardGrid} from './LeaderboardGrid';
+import {Button} from '@wandb/weave/components/Button';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
@@ -34,6 +36,8 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   const {peekingRouter} = useWeaveflowRouteContext();
   const history = useHistory();
 
+  const [showConfig, setShowConfig] = useState(false);
+
   const handleCellClick = (
     modelName: string,
     metricName: string,
@@ -53,9 +57,16 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     }
   };
 
+
   return (
     <Box display="flex" flexDirection="column" height="100%">
-      <Box flexShrink={0} maxHeight="50%" overflow="auto">
+      <div style={{
+        position: 'absolute',
+        top: 20,
+        right: 24,
+      }}><ToggleLeaderboardConfig isOpen={showConfig} onClick={() => setShowConfig(c => !c)}/></div>
+
+      <Box flexShrink={0} maxHeight="35%" overflow="auto">
         <EditableMarkdown
           value={description}
           onChange={setDescription}
@@ -65,15 +76,45 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
       <Box
         flexGrow={1}
         display="flex"
-        flexDirection="column"
+        flexDirection="row"
         overflow="hidden"
-        minHeight="50%">
+        minHeight="65%">
         <LeaderboardGrid
           loading={loading}
           data={data}
           onCellClick={handleCellClick}
         />
+        {showConfig && <LeaderboardConfig />}
       </Box>
     </Box>
   );
 };
+
+export const ToggleLeaderboardConfig: React.FC<{
+  isOpen: boolean;
+  onClick: () => void;
+}> = ({isOpen, onClick}) => {
+  return (
+    <Box
+      sx={{
+        height: '100%',
+        display: 'flex',
+        alignItems: 'center',
+      }}>
+      <Button
+        variant="ghost"
+        size="medium"
+        onClick={onClick}
+        tooltip="Configure Leaderboard"
+        icon={isOpen ? "close" : "settings"}
+      />
+    </Box>
+  );
+};
+
+
+const LeaderboardConfig = () => {
+  return <Box sx={{
+    width: '100%'
+  }}>LeaderboardConfig</Box>;
+};

From 2754a0286f156fc5363ec3ffcbddff23c29d7d78 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 17:50:19 -0700
Subject: [PATCH 016/169] Human: getting the config going

---
 .../components/FancyPage/useProjectSidebar.ts |   5 +-
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  49 +++++---
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 118 ++++++++++++++++++
 3 files changed, 151 insertions(+), 21 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index cf0207d771f..b7025264826 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -11,7 +11,7 @@ export const useProjectSidebar = (
   hasWeaveData: boolean,
   hasTraceBackend: boolean = true,
   hasModelsAccess: boolean = true,
-  viewerIsAdmin: boolean = false,
+  viewerIsAdmin: boolean = false
 ): FancyPageSidebarItem[] => {
   // Should show models sidebar items if we have models data or if we don't have a trace backend
   let showModelsSidebarItems = hasModelsData || !hasTraceBackend;
@@ -137,7 +137,8 @@ export const useProjectSidebar = (
             name: 'Leaderboard',
             slug: 'weave/leaderboard',
             // TODO: remove this `true`
-            isShown: (viewerIsAdmin || true) && (showWeaveSidebarItems || isShowAll),
+            isShown:
+              (viewerIsAdmin || true) && (showWeaveSidebarItems || isShowAll),
             iconName: IconNames.BenchmarkSquare,
           },
           {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 720a3218e0a..73c13a61e46 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,13 +1,17 @@
 import {Box} from '@mui/material';
-
-import React, {useContext, useEffect, useState} from 'react';
+import {Button} from '@wandb/weave/components/Button';
+import React, {useState} from 'react';
 import {useHistory} from 'react-router-dom';
 
-import {usePeekLocation, useWeaveflowRouteContext, WeaveflowPeekContext} from '../../context';
+import {useWeaveflowRouteContext} from '../../context';
 import {EditableMarkdown} from './EditableMarkdown';
 import {useLeaderboardData} from './hooks';
 import {LeaderboardGrid} from './LeaderboardGrid';
-import {Button} from '@wandb/weave/components/Button';
+import {
+  LeaderboardConfig,
+  LeaderboardConfigType,
+  useCurrentLeaderboardConfig,
+} from './LeaderboardPageConfig';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
@@ -38,6 +42,10 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
 
   const [showConfig, setShowConfig] = useState(false);
 
+  const [currentConfig, setCurrentConfig] = useState<LeaderboardConfigType>(
+    useCurrentLeaderboardConfig()
+  );
+
   const handleCellClick = (
     modelName: string,
     metricName: string,
@@ -57,14 +65,19 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     }
   };
 
-
   return (
     <Box display="flex" flexDirection="column" height="100%">
-      <div style={{
-        position: 'absolute',
-        top: 20,
-        right: 24,
-      }}><ToggleLeaderboardConfig isOpen={showConfig} onClick={() => setShowConfig(c => !c)}/></div>
+      <div
+        style={{
+          position: 'absolute',
+          top: 20,
+          right: 24,
+        }}>
+        <ToggleLeaderboardConfig
+          isOpen={showConfig}
+          onClick={() => setShowConfig(c => !c)}
+        />
+      </div>
 
       <Box flexShrink={0} maxHeight="35%" overflow="auto">
         <EditableMarkdown
@@ -84,7 +97,12 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
           data={data}
           onCellClick={handleCellClick}
         />
-        {showConfig && <LeaderboardConfig />}
+        {showConfig && (
+          <LeaderboardConfig
+            currentConfig={currentConfig}
+            onConfigUpdate={setCurrentConfig}
+          />
+        )}
       </Box>
     </Box>
   );
@@ -106,15 +124,8 @@ export const ToggleLeaderboardConfig: React.FC<{
         size="medium"
         onClick={onClick}
         tooltip="Configure Leaderboard"
-        icon={isOpen ? "close" : "settings"}
+        icon={isOpen ? 'close' : 'settings'}
       />
     </Box>
   );
 };
-
-
-const LeaderboardConfig = () => {
-  return <Box sx={{
-    width: '100%'
-  }}>LeaderboardConfig</Box>;
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
new file mode 100644
index 00000000000..f43ed5ba663
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -0,0 +1,118 @@
+import {Box} from '@mui/material';
+import React, {useMemo} from 'react';
+
+type VersionSpec = string | 'latest' | 'all';
+type Direction = 'asc' | 'desc';
+
+export type LeaderboardConfigType = {
+  version: number;
+  config: {
+    columns: Array<{
+      dataset: {
+        name: string;
+        version: VersionSpec;
+      };
+      scores: Array<{
+        scorer: {
+          name: string;
+          version: VersionSpec;
+        };
+        metrics: Array<{
+          displayName: string;
+          path: string[];
+          minTrials?: number;
+          maxTrials?: number;
+          sort?: {
+            precedence: number;
+            direction: Direction;
+          };
+        }>;
+      }>;
+    }>;
+    models: Array<{
+      name: string;
+      version: VersionSpec;
+    }>;
+  };
+};
+
+export const LeaderboardConfig: React.FC<{
+  currentConfig: LeaderboardConfigType;
+  onConfigUpdate: (newConfig: LeaderboardConfigType) => void;
+}> = props => {
+  return (
+    <Box
+      sx={{
+        width: '100%',
+      }}>
+      LeaderboardConfig
+    </Box>
+  );
+};
+
+export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return {
+      version: 1,
+      config: {
+        columns: [],
+        models: [],
+      },
+    };
+  }, []);
+};
+
+const useDatasetNames = (): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [];
+  }, []);
+};
+
+const useDatasetVersionsForDatasetName = (datasetName: string): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [];
+  }, []);
+};
+
+const useScorerNamesForDataset = (
+  datasetName: string,
+  datasetVersion: string
+): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [];
+  }, []);
+};
+
+const useScorerVersionsForDatasetAndScorer = (
+  datasetName: string,
+  datasetVersion: string,
+  scorerName: string
+): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [];
+  }, []);
+};
+
+const useMetricPathsForDatasetAndScorer = (
+  datasetName: string,
+  datasetVersion: string,
+  scorerName: string,
+  scorerVersion: string
+): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [];
+  }, []);
+};
+
+const useModelNames = (): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [];
+  }, []);
+};

From c36415bd50524ade8b4745cb4491836ec8a07bd5 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 17:58:54 -0700
Subject: [PATCH 017/169] Human: finish the basic stubbing

---
 .../components/FancyPage/useProjectSidebar.ts |  7 +--
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 48 ++++++++++++++++---
 2 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index b7025264826..a38fec4d332 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -224,10 +224,11 @@ export const useProjectSidebar = (
     return onlyShownItems;
   }, [
     isLoading,
-    isModelsOnly,
-    isWeaveOnly,
-    showWeaveSidebarItems,
     isShowAll,
+    isWeaveOnly,
     viewingRestricted,
+    isModelsOnly,
+    viewerIsAdmin,
+    showWeaveSidebarItems,
   ]);
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index f43ed5ba663..0230251c8f5 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -66,14 +66,22 @@ export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
 const useDatasetNames = (): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [];
+    return [
+      'leaderboard-test-dataset',
+      'leaderboard-test-dataset-2',
+      'leaderboard-test-dataset-3',
+    ];
   }, []);
 };
 
 const useDatasetVersionsForDatasetName = (datasetName: string): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [];
+    return [
+      'dug657ioy8j1',
+      'dnkjubyhvasd',
+      'dadsgf3f451d',
+    ];
   }, []);
 };
 
@@ -83,7 +91,11 @@ const useScorerNamesForDataset = (
 ): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [];
+    return [
+      'scorer-1',
+      'scorer-2',
+      'scorer-3',
+    ];
   }, []);
 };
 
@@ -94,7 +106,11 @@ const useScorerVersionsForDatasetAndScorer = (
 ): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [];
+    return [
+      'sug657ioy8j1',
+      'snkjubyhvasd',
+      'sadsgf3f451d',
+    ];
   }, []);
 };
 
@@ -106,13 +122,33 @@ const useMetricPathsForDatasetAndScorer = (
 ): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [];
+    return [
+      'accuracy',
+      'f1.macro',
+      'precision.micro',
+      'recall.micro.data',
+    ];
   }, []);
 };
 
 const useModelNames = (): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [];
+    return [
+      'model-1',
+      'model-2',
+      'model-3',
+    ];
+  }, []);
+};
+
+const useModelVersionsForModelName = (modelName: string): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [
+      'mug657ioy8j1',
+      'mnkjubyhvasd',
+      'madsgf3f451d',
+    ];
   }, []);
 };

From 44d168ceb73da86b6afc3bb0d4b293c3159f5185 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 17:59:02 -0700
Subject: [PATCH 018/169] Human: finish the basic stubbing

---
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 37 +++----------------
 1 file changed, 6 insertions(+), 31 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 0230251c8f5..d85cfcea4b1 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -77,11 +77,7 @@ const useDatasetNames = (): string[] => {
 const useDatasetVersionsForDatasetName = (datasetName: string): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [
-      'dug657ioy8j1',
-      'dnkjubyhvasd',
-      'dadsgf3f451d',
-    ];
+    return ['dug657ioy8j1', 'dnkjubyhvasd', 'dadsgf3f451d'];
   }, []);
 };
 
@@ -91,11 +87,7 @@ const useScorerNamesForDataset = (
 ): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [
-      'scorer-1',
-      'scorer-2',
-      'scorer-3',
-    ];
+    return ['scorer-1', 'scorer-2', 'scorer-3'];
   }, []);
 };
 
@@ -106,11 +98,7 @@ const useScorerVersionsForDatasetAndScorer = (
 ): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [
-      'sug657ioy8j1',
-      'snkjubyhvasd',
-      'sadsgf3f451d',
-    ];
+    return ['sug657ioy8j1', 'snkjubyhvasd', 'sadsgf3f451d'];
   }, []);
 };
 
@@ -122,33 +110,20 @@ const useMetricPathsForDatasetAndScorer = (
 ): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [
-      'accuracy',
-      'f1.macro',
-      'precision.micro',
-      'recall.micro.data',
-    ];
+    return ['accuracy', 'f1.macro', 'precision.micro', 'recall.micro.data'];
   }, []);
 };
 
 const useModelNames = (): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [
-      'model-1',
-      'model-2',
-      'model-3',
-    ];
+    return ['model-1', 'model-2', 'model-3'];
   }, []);
 };
 
 const useModelVersionsForModelName = (modelName: string): string[] => {
   // TODO: Implement this
   return useMemo(() => {
-    return [
-      'mug657ioy8j1',
-      'mnkjubyhvasd',
-      'madsgf3f451d',
-    ];
+    return ['mug657ioy8j1', 'mnkjubyhvasd', 'madsgf3f451d'];
   }, []);
 };

From 6e2e58e709de70ab5ab70829fbe77f154c139523 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 18:19:15 -0700
Subject: [PATCH 019/169] Cursor: Template Config

---
 .../pages/LeaderboardPage/ColumnConfig.tsx    | 130 ++++++++++
 .../LeaderboardPage/LeaderboardConfigType.tsx |  34 +++
 .../pages/LeaderboardPage/LeaderboardPage.tsx |   8 +-
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 243 ++++++++++--------
 .../pages/LeaderboardPage/MetricConfig.tsx    | 142 ++++++++++
 .../pages/LeaderboardPage/ModelConfig.tsx     |  74 ++++++
 .../pages/LeaderboardPage/ScoreConfig.tsx     | 135 ++++++++++
 .../useCurrentLeaderboardConfig.tsx           |  90 +++++++
 8 files changed, 738 insertions(+), 118 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
new file mode 100644
index 00000000000..8f1f5714e44
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
@@ -0,0 +1,130 @@
+import AddIcon from '@mui/icons-material/Add';
+import DeleteIcon from '@mui/icons-material/Delete';
+import {
+  Box,
+  Button,
+  FormControl,
+  IconButton,
+  InputLabel,
+  MenuItem,
+  Select,
+  Stack,
+  Typography,
+} from '@mui/material';
+import React from 'react';
+
+import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
+import {ScoreConfig} from './ScoreConfig';
+import {
+  useDatasetNames,
+  useDatasetVersionsForDatasetName,
+} from './useCurrentLeaderboardConfig';
+
+export const ColumnConfig: React.FC<{
+  column: LeaderboardConfigType['config']['columns'][0];
+  onUpdate: (
+    updatedColumn: LeaderboardConfigType['config']['columns'][0]
+  ) => void;
+  onRemove: () => void;
+}> = ({column, onUpdate, onRemove}) => {
+  const datasetNames = useDatasetNames();
+  const datasetVersions = useDatasetVersionsForDatasetName(column.dataset.name);
+
+  const handleAddScore = () => {
+    onUpdate({
+      ...column,
+      scores: [
+        ...column.scores,
+        {scorer: {name: '', version: 'latest'}, metrics: []},
+      ],
+    });
+  };
+
+  const handleRemoveScore = (index: number) => {
+    onUpdate({
+      ...column,
+      scores: column.scores.filter((_, i) => i !== index),
+    });
+  };
+
+  const handleUpdateScore = (
+    index: number,
+    updatedScore: (typeof column.scores)[0]
+  ) => {
+    onUpdate({
+      ...column,
+      scores: column.scores.map((score, i) =>
+        i === index ? updatedScore : score
+      ),
+    });
+  };
+
+  return (
+    <Box sx={{mb: 2, p: 2, border: '1px solid #ccc', borderRadius: 1}}>
+      <Stack
+        direction="row"
+        justifyContent="space-between"
+        alignItems="center"
+        sx={{mb: 2}}>
+        <Typography variant="subtitle1">Column</Typography>
+        <IconButton onClick={onRemove} size="small">
+          <DeleteIcon />
+        </IconButton>
+      </Stack>
+      <FormControl fullWidth sx={{mb: 2}}>
+        <InputLabel>Dataset</InputLabel>
+        <Select
+          value={column.dataset.name}
+          label="Dataset"
+          onChange={e =>
+            onUpdate({
+              ...column,
+              dataset: {...column.dataset, name: e.target.value},
+            })
+          }>
+          {datasetNames.map(name => (
+            <MenuItem key={name} value={name}>
+              {name}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+      <FormControl fullWidth sx={{mb: 2}}>
+        <InputLabel>Dataset Version</InputLabel>
+        <Select
+          value={column.dataset.version}
+          label="Dataset Version"
+          onChange={e =>
+            onUpdate({
+              ...column,
+              dataset: {
+                ...column.dataset,
+                version: e.target.value as VersionSpec,
+              },
+            })
+          }>
+          <MenuItem value="latest">Latest</MenuItem>
+          <MenuItem value="all">All</MenuItem>
+          {datasetVersions.map(({version, versionIndex}) => (
+            <MenuItem key={version} value={version}>
+              Version {versionIndex}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+      {column.scores.map((score, index) => (
+        <ScoreConfig
+          key={index}
+          score={score}
+          datasetName={column.dataset.name}
+          datasetVersion={column.dataset.version}
+          onUpdate={updatedScore => handleUpdateScore(index, updatedScore)}
+          onRemove={() => handleRemoveScore(index)}
+        />
+      ))}
+      <Button startIcon={<AddIcon />} onClick={handleAddScore} sx={{mt: 1}}>
+        Add Score
+      </Button>
+    </Box>
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
new file mode 100644
index 00000000000..4ec72a63ad8
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
@@ -0,0 +1,34 @@
+export type VersionSpec = string | 'latest' | 'all';
+export type Direction = 'asc' | 'desc';
+
+export type LeaderboardConfigType = {
+  version: number;
+  config: {
+    columns: Array<{
+      dataset: {
+        name: string;
+        version: VersionSpec;
+      };
+      scores: Array<{
+        scorer: {
+          name: string;
+          version: VersionSpec;
+        };
+        metrics: Array<{
+          displayName: string;
+          path: string[];
+          minTrials?: number;
+          maxTrials?: number;
+          sort?: {
+            precedence: number;
+            direction: Direction;
+          };
+        }>;
+      }>;
+    }>;
+    models: Array<{
+      name: string;
+      version: VersionSpec;
+    }>;
+  };
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 73c13a61e46..5dd0b889cc3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -6,12 +6,10 @@ import {useHistory} from 'react-router-dom';
 import {useWeaveflowRouteContext} from '../../context';
 import {EditableMarkdown} from './EditableMarkdown';
 import {useLeaderboardData} from './hooks';
+import {LeaderboardConfigType} from './LeaderboardConfigType';
 import {LeaderboardGrid} from './LeaderboardGrid';
-import {
-  LeaderboardConfig,
-  LeaderboardConfigType,
-  useCurrentLeaderboardConfig,
-} from './LeaderboardPageConfig';
+import {LeaderboardConfig} from './LeaderboardPageConfig';
+import {useCurrentLeaderboardConfig} from './useCurrentLeaderboardConfig';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index d85cfcea4b1..759db6e8af6 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -1,129 +1,146 @@
-import {Box} from '@mui/material';
-import React, {useMemo} from 'react';
+import AddIcon from '@mui/icons-material/Add';
+import {Box, Button, Typography} from '@mui/material';
+import React, {useState} from 'react';
 
-type VersionSpec = string | 'latest' | 'all';
-type Direction = 'asc' | 'desc';
-
-export type LeaderboardConfigType = {
-  version: number;
-  config: {
-    columns: Array<{
-      dataset: {
-        name: string;
-        version: VersionSpec;
-      };
-      scores: Array<{
-        scorer: {
-          name: string;
-          version: VersionSpec;
-        };
-        metrics: Array<{
-          displayName: string;
-          path: string[];
-          minTrials?: number;
-          maxTrials?: number;
-          sort?: {
-            precedence: number;
-            direction: Direction;
-          };
-        }>;
-      }>;
-    }>;
-    models: Array<{
-      name: string;
-      version: VersionSpec;
-    }>;
-  };
-};
+import {ColumnConfig} from './ColumnConfig';
+import {LeaderboardConfigType} from './LeaderboardConfigType';
+import {ModelConfig} from './ModelConfig';
 
 export const LeaderboardConfig: React.FC<{
   currentConfig: LeaderboardConfigType;
   onConfigUpdate: (newConfig: LeaderboardConfigType) => void;
-}> = props => {
-  return (
-    <Box
-      sx={{
-        width: '100%',
-      }}>
-      LeaderboardConfig
-    </Box>
-  );
-};
+}> = ({currentConfig, onConfigUpdate}) => {
+  const [config, setConfig] = useState<LeaderboardConfigType>(currentConfig);
+
+  const handleSave = () => {
+    onConfigUpdate(config);
+  };
+
+  const handleCancel = () => {
+    setConfig(currentConfig);
+  };
 
-export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return {
-      version: 1,
+  const handleAddColumn = () => {
+    setConfig(prev => ({
+      ...prev,
       config: {
-        columns: [],
-        models: [],
+        ...prev.config,
+        columns: [
+          ...prev.config.columns,
+          {dataset: {name: '', version: 'latest'}, scores: []},
+        ],
       },
-    };
-  }, []);
-};
+    }));
+  };
 
-const useDatasetNames = (): string[] => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return [
-      'leaderboard-test-dataset',
-      'leaderboard-test-dataset-2',
-      'leaderboard-test-dataset-3',
-    ];
-  }, []);
-};
+  const handleRemoveColumn = (index: number) => {
+    setConfig(prev => ({
+      ...prev,
+      config: {
+        ...prev.config,
+        columns: prev.config.columns.filter((_, i) => i !== index),
+      },
+    }));
+  };
 
-const useDatasetVersionsForDatasetName = (datasetName: string): string[] => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return ['dug657ioy8j1', 'dnkjubyhvasd', 'dadsgf3f451d'];
-  }, []);
-};
+  const handleUpdateColumn = (
+    index: number,
+    updatedColumn: LeaderboardConfigType['config']['columns'][0]
+  ) => {
+    setConfig(prev => ({
+      ...prev,
+      config: {
+        ...prev.config,
+        columns: prev.config.columns.map((column, i) =>
+          i === index ? updatedColumn : column
+        ),
+      },
+    }));
+  };
 
-const useScorerNamesForDataset = (
-  datasetName: string,
-  datasetVersion: string
-): string[] => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return ['scorer-1', 'scorer-2', 'scorer-3'];
-  }, []);
-};
+  const handleAddModel = () => {
+    setConfig(prev => ({
+      ...prev,
+      config: {
+        ...prev.config,
+        models: [...prev.config.models, {name: '', version: 'latest'}],
+      },
+    }));
+  };
 
-const useScorerVersionsForDatasetAndScorer = (
-  datasetName: string,
-  datasetVersion: string,
-  scorerName: string
-): string[] => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return ['sug657ioy8j1', 'snkjubyhvasd', 'sadsgf3f451d'];
-  }, []);
-};
+  const handleRemoveModel = (index: number) => {
+    setConfig(prev => ({
+      ...prev,
+      config: {
+        ...prev.config,
+        models: prev.config.models.filter((_, i) => i !== index),
+      },
+    }));
+  };
 
-const useMetricPathsForDatasetAndScorer = (
-  datasetName: string,
-  datasetVersion: string,
-  scorerName: string,
-  scorerVersion: string
-): string[] => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return ['accuracy', 'f1.macro', 'precision.micro', 'recall.micro.data'];
-  }, []);
-};
+  const handleUpdateModel = (
+    index: number,
+    updatedModel: LeaderboardConfigType['config']['models'][0]
+  ) => {
+    setConfig(prev => ({
+      ...prev,
+      config: {
+        ...prev.config,
+        models: prev.config.models.map((model, i) =>
+          i === index ? updatedModel : model
+        ),
+      },
+    }));
+  };
 
-const useModelNames = (): string[] => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return ['model-1', 'model-2', 'model-3'];
-  }, []);
-};
+  return (
+    <Box sx={{width: '100%', p: 2}}>
+      <Typography variant="h5" gutterBottom>
+        Leaderboard Configuration
+      </Typography>
 
-const useModelVersionsForModelName = (modelName: string): string[] => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return ['mug657ioy8j1', 'mnkjubyhvasd', 'madsgf3f451d'];
-  }, []);
+      <Box sx={{mb: 4}}>
+        <Typography variant="h6" gutterBottom>
+          Columns
+        </Typography>
+        {config.config.columns.map((column, index) => (
+          <ColumnConfig
+            key={index}
+            column={column}
+            onUpdate={updatedColumn => handleUpdateColumn(index, updatedColumn)}
+            onRemove={() => handleRemoveColumn(index)}
+          />
+        ))}
+        <Button startIcon={<AddIcon />} onClick={handleAddColumn} sx={{mt: 2}}>
+          Add Column
+        </Button>
+      </Box>
+
+      <Box sx={{mb: 4}}>
+        <Typography variant="h6" gutterBottom>
+          Models
+        </Typography>
+        {config.config.models.map((model, index) => (
+          <ModelConfig
+            key={index}
+            model={model}
+            onUpdate={updatedModel => handleUpdateModel(index, updatedModel)}
+            onRemove={() => handleRemoveModel(index)}
+          />
+        ))}
+        <Button startIcon={<AddIcon />} onClick={handleAddModel} sx={{mt: 2}}>
+          Add Model
+        </Button>
+      </Box>
+
+      <Box sx={{display: 'flex', justifyContent: 'flex-end', mt: 4}}>
+        <Button variant="outlined" onClick={handleCancel} sx={{mr: 2}}>
+          Cancel
+        </Button>
+        <Button variant="contained" onClick={handleSave}>
+          Save
+        </Button>
+      </Box>
+    </Box>
+  );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
new file mode 100644
index 00000000000..26fa337002a
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
@@ -0,0 +1,142 @@
+import DeleteIcon from '@mui/icons-material/Delete';
+import {
+  Box,
+  FormControl,
+  IconButton,
+  InputLabel,
+  MenuItem,
+  Select,
+  Stack,
+  TextField,
+  Typography,
+} from '@mui/material';
+import React from 'react';
+
+import {Direction, LeaderboardConfigType} from './LeaderboardConfigType';
+import {useMetricPathsForDatasetAndScorer} from './useCurrentLeaderboardConfig';
+
+export const MetricConfig: React.FC<{
+  metric: LeaderboardConfigType['config']['columns'][0]['scores'][0]['metrics'][0];
+  datasetName: string;
+  datasetVersion: string;
+  scorerName: string;
+  scorerVersion: string;
+  onUpdate: (
+    updatedMetric: LeaderboardConfigType['config']['columns'][0]['scores'][0]['metrics'][0]
+  ) => void;
+  onRemove: () => void;
+}> = ({
+  metric,
+  datasetName,
+  datasetVersion,
+  scorerName,
+  scorerVersion,
+  onUpdate,
+  onRemove,
+}) => {
+  const metricPaths = useMetricPathsForDatasetAndScorer(
+    datasetName,
+    datasetVersion,
+    scorerName,
+    scorerVersion
+  );
+
+  return (
+    <Box sx={{mb: 2, p: 2, border: '1px solid #f5f5f5', borderRadius: 1}}>
+      <Stack
+        direction="row"
+        justifyContent="space-between"
+        alignItems="center"
+        sx={{mb: 2}}>
+        <Typography variant="subtitle2">Metric</Typography>
+        <IconButton onClick={onRemove} size="small">
+          <DeleteIcon />
+        </IconButton>
+      </Stack>
+      <TextField
+        fullWidth
+        label="Display Name"
+        value={metric.displayName}
+        onChange={e => onUpdate({...metric, displayName: e.target.value})}
+        sx={{mb: 2}}
+      />
+      <FormControl fullWidth sx={{mb: 2}}>
+        <InputLabel>Metric Path</InputLabel>
+        <Select
+          value={metric.path.join('.')}
+          label="Metric Path"
+          onChange={e =>
+            onUpdate({...metric, path: e.target.value.split('.')})
+          }>
+          {metricPaths.map(path => (
+            <MenuItem key={path} value={path}>
+              {path}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+      <TextField
+        fullWidth
+        label="Min Trials"
+        type="number"
+        value={metric.minTrials || ''}
+        onChange={e =>
+          onUpdate({
+            ...metric,
+            minTrials: parseInt(e.target.value, 10) || undefined,
+          })
+        }
+        sx={{mb: 2}}
+      />
+      <TextField
+        fullWidth
+        label="Max Trials"
+        type="number"
+        value={metric.maxTrials || ''}
+        onChange={e =>
+          onUpdate({
+            ...metric,
+            maxTrials: parseInt(e.target.value, 10) || undefined,
+          })
+        }
+        sx={{mb: 2}}
+      />
+      <FormControl fullWidth sx={{mb: 2}}>
+        <InputLabel>Sort Direction</InputLabel>
+        <Select
+          value={metric.sort?.direction || ''}
+          label="Sort Direction"
+          onChange={e => {
+            const direction = e.target.value as Direction | 'desc';
+            onUpdate({
+              ...metric,
+              sort: direction
+                ? {...(metric.sort || {precedence: 0}), direction}
+                : undefined,
+            });
+          }}>
+          <MenuItem value="">No sort</MenuItem>
+          <MenuItem value="asc">Ascending</MenuItem>
+          <MenuItem value="desc">Descending</MenuItem>
+        </Select>
+      </FormControl>
+      {metric.sort && (
+        <TextField
+          fullWidth
+          label="Sort Precedence"
+          type="number"
+          value={metric.sort.precedence}
+          onChange={e =>
+            onUpdate({
+              ...metric,
+              sort: {
+                ...metric.sort!,
+                precedence: parseInt(e.target.value, 10) || 0,
+              },
+            })
+          }
+        />
+      )}
+    </Box>
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
new file mode 100644
index 00000000000..d7e05fa7992
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
@@ -0,0 +1,74 @@
+import DeleteIcon from '@mui/icons-material/Delete';
+import {
+  Box,
+  FormControl,
+  IconButton,
+  InputLabel,
+  MenuItem,
+  Select,
+  Stack,
+  Typography,
+} from '@mui/material';
+import React from 'react';
+
+import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
+import {
+  useModelNames,
+  useModelVersionsForModelName,
+} from './useCurrentLeaderboardConfig';
+
+export const ModelConfig: React.FC<{
+  model: LeaderboardConfigType['config']['models'][0];
+  onUpdate: (
+    updatedModel: LeaderboardConfigType['config']['models'][0]
+  ) => void;
+  onRemove: () => void;
+}> = ({model, onUpdate, onRemove}) => {
+  const modelNames = useModelNames();
+  const modelVersions = useModelVersionsForModelName(model.name);
+
+  return (
+    <Box sx={{mb: 2, p: 2, border: '1px solid #ccc', borderRadius: 1}}>
+      <Stack
+        direction="row"
+        justifyContent="space-between"
+        alignItems="center"
+        sx={{mb: 2}}>
+        <Typography variant="subtitle1">Model</Typography>
+        <IconButton onClick={onRemove} size="small">
+          <DeleteIcon />
+        </IconButton>
+      </Stack>
+      <FormControl fullWidth sx={{mb: 2}}>
+        <InputLabel>Model Name</InputLabel>
+        <Select
+          value={model.name}
+          label="Model Name"
+          onChange={e => onUpdate({...model, name: e.target.value})}>
+          {modelNames.map(name => (
+            <MenuItem key={name} value={name}>
+              {name}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+      <FormControl fullWidth>
+        <InputLabel>Model Version</InputLabel>
+        <Select
+          value={model.version}
+          label="Model Version"
+          onChange={e =>
+            onUpdate({...model, version: e.target.value as VersionSpec})
+          }>
+          <MenuItem value="latest">Latest</MenuItem>
+          <MenuItem value="all">All</MenuItem>
+          {modelVersions.map(({version, versionIndex}) => (
+            <MenuItem key={version} value={version}>
+              Version {versionIndex}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+    </Box>
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
new file mode 100644
index 00000000000..752249dee56
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
@@ -0,0 +1,135 @@
+import AddIcon from '@mui/icons-material/Add';
+import DeleteIcon from '@mui/icons-material/Delete';
+import {
+  Box,
+  Button,
+  FormControl,
+  IconButton,
+  InputLabel,
+  MenuItem,
+  Select,
+  Stack,
+  Typography,
+} from '@mui/material';
+import React from 'react';
+
+import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
+import {MetricConfig} from './MetricConfig';
+import {
+  useScorerNamesForDataset,
+  useScorerVersionsForDatasetAndScorer,
+} from './useCurrentLeaderboardConfig';
+
+export const ScoreConfig: React.FC<{
+  score: LeaderboardConfigType['config']['columns'][0]['scores'][0];
+  datasetName: string;
+  datasetVersion: string;
+  onUpdate: (
+    updatedScore: LeaderboardConfigType['config']['columns'][0]['scores'][0]
+  ) => void;
+  onRemove: () => void;
+}> = ({score, datasetName, datasetVersion, onUpdate, onRemove}) => {
+  const scorerNames = useScorerNamesForDataset(datasetName, datasetVersion);
+  const scorerVersions = useScorerVersionsForDatasetAndScorer(
+    datasetName,
+    datasetVersion,
+    score.scorer.name
+  );
+
+  const handleAddMetric = () => {
+    onUpdate({
+      ...score,
+      metrics: [
+        ...score.metrics,
+        {displayName: '', path: [], minTrials: undefined, maxTrials: undefined},
+      ],
+    });
+  };
+
+  const handleRemoveMetric = (index: number) => {
+    onUpdate({
+      ...score,
+      metrics: score.metrics.filter((_, i) => i !== index),
+    });
+  };
+
+  const handleUpdateMetric = (
+    index: number,
+    updatedMetric: (typeof score.metrics)[0]
+  ) => {
+    onUpdate({
+      ...score,
+      metrics: score.metrics.map((metric, i) =>
+        i === index ? updatedMetric : metric
+      ),
+    });
+  };
+
+  return (
+    <Box sx={{mb: 2, p: 2, border: '1px solid #eee', borderRadius: 1}}>
+      <Stack
+        direction="row"
+        justifyContent="space-between"
+        alignItems="center"
+        sx={{mb: 2}}>
+        <Typography variant="subtitle2">Score</Typography>
+        <IconButton onClick={onRemove} size="small">
+          <DeleteIcon />
+        </IconButton>
+      </Stack>
+      <FormControl fullWidth sx={{mb: 2}}>
+        <InputLabel>Scorer</InputLabel>
+        <Select
+          value={score.scorer.name}
+          label="Scorer"
+          onChange={e =>
+            onUpdate({
+              ...score,
+              scorer: {...score.scorer, name: e.target.value},
+            })
+          }>
+          {scorerNames.map(name => (
+            <MenuItem key={name} value={name}>
+              {name}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+      <FormControl fullWidth sx={{mb: 2}}>
+        <InputLabel>Scorer Version</InputLabel>
+        <Select
+          value={score.scorer.version}
+          label="Scorer Version"
+          onChange={e =>
+            onUpdate({
+              ...score,
+              scorer: {...score.scorer, version: e.target.value as VersionSpec},
+            })
+          }>
+          <MenuItem value="latest">Latest</MenuItem>
+          <MenuItem value="all">All</MenuItem>
+          {scorerVersions.map(({version, versionIndex}) => (
+            <MenuItem key={version} value={version}>
+              Version {versionIndex}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+      {score.metrics.map((metric, index) => (
+        <MetricConfig
+          key={index}
+          metric={metric}
+          datasetName={datasetName}
+          datasetVersion={datasetVersion}
+          scorerName={score.scorer.name}
+          scorerVersion={score.scorer.version}
+          onUpdate={updatedMetric => handleUpdateMetric(index, updatedMetric)}
+          onRemove={() => handleRemoveMetric(index)}
+        />
+      ))}
+      <Button startIcon={<AddIcon />} onClick={handleAddMetric} sx={{mt: 1}}>
+        Add Metric
+      </Button>
+    </Box>
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx
new file mode 100644
index 00000000000..2039fee2d0b
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx
@@ -0,0 +1,90 @@
+import {useMemo} from 'react';
+
+import {LeaderboardConfigType} from './LeaderboardConfigType';
+
+export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return {
+      version: 1,
+      config: {
+        columns: [],
+        models: [],
+      },
+    };
+  }, []);
+};
+export const useDatasetNames = (): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [
+      'leaderboard-test-dataset',
+      'leaderboard-test-dataset-2',
+      'leaderboard-test-dataset-3',
+    ];
+  }, []);
+};
+export const useDatasetVersionsForDatasetName = (
+  datasetName: string
+): Array<{version: string; versionIndex: number}> => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [
+      {version: 'dug657ioy8j1', versionIndex: 0},
+      {version: 'dnkjubyhvasd', versionIndex: 1},
+      {version: 'dadsgf3f451d', versionIndex: 2},
+    ];
+  }, []);
+};
+export const useScorerNamesForDataset = (
+  datasetName: string,
+  datasetVersion: string
+): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return ['scorer-1', 'scorer-2', 'scorer-3'];
+  }, []);
+};
+export const useScorerVersionsForDatasetAndScorer = (
+  datasetName: string,
+  datasetVersion: string,
+  scorerName: string
+): Array<{version: string; versionIndex: number}> => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [
+      {version: 'sug657ioy8j1', versionIndex: 0},
+      {version: 'snkjubyhvasd', versionIndex: 1},
+      {version: 'sadsgf3f451d', versionIndex: 2},
+    ];
+  }, []);
+};
+export const useMetricPathsForDatasetAndScorer = (
+  datasetName: string,
+  datasetVersion: string,
+  scorerName: string,
+  scorerVersion: string
+): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return ['accuracy', 'f1.macro', 'precision.micro', 'recall.micro.data'];
+  }, []);
+};
+export const useModelNames = (): string[] => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return ['model-1', 'model-2', 'model-3'];
+  }, []);
+};
+export const useModelVersionsForModelName = (
+  modelName: string
+): Array<{version: string; versionIndex: number}> => {
+  // TODO: Implement this
+  return useMemo(() => {
+    return [
+      {version: 'mug657ioy8j1', versionIndex: 0},
+      {version: 'mnkjubyhvasd', versionIndex: 1},
+      {version: 'madsgf3f451d', versionIndex: 2},
+    ];
+  }, []);
+};

From 66510139895fec4914814b1867349374da4e60b2 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 18:32:03 -0700
Subject: [PATCH 020/169] Human + AI: Config editor

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 21 +++++++++++++++----
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 18 ++++++++++++++--
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 5dd0b889cc3..60ae23b9f94 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,4 +1,4 @@
-import {Box} from '@mui/material';
+import {Alert, Box, Typography} from '@mui/material';
 import {Button} from '@wandb/weave/components/Button';
 import React, {useState} from 'react';
 import {useHistory} from 'react-router-dom';
@@ -63,12 +63,15 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     }
   };
 
+  const [showingAlert, setShowingAlert] = useState(true);
+
   return (
     <Box display="flex" flexDirection="column" height="100%">
+      {showingAlert && <UnlistedAlert onClose={() => setShowingAlert(false)} />}
       <div
         style={{
           position: 'absolute',
-          top: 20,
+          top: 20 + (showingAlert ? 52 : 0),
           right: 24,
         }}>
         <ToggleLeaderboardConfig
@@ -76,7 +79,6 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
           onClick={() => setShowConfig(c => !c)}
         />
       </div>
-
       <Box flexShrink={0} maxHeight="35%" overflow="auto">
         <EditableMarkdown
           value={description}
@@ -121,9 +123,20 @@ export const ToggleLeaderboardConfig: React.FC<{
         variant="ghost"
         size="medium"
         onClick={onClick}
-        tooltip="Configure Leaderboard"
+        tooltip={isOpen ? 'Discard Changes' : 'Configure Leaderboard'}
         icon={isOpen ? 'close' : 'settings'}
       />
     </Box>
   );
 };
+
+const UnlistedAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
+  return (
+    <Alert severity="info" onClose={onClose}>
+      <Typography variant="body1">
+        You have found an internal, unlisted beta page! Please expect bugs and
+        incomplete features. Permilinks are not yet supported.
+      </Typography>
+    </Alert>
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 759db6e8af6..c4954c1bd1f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -1,5 +1,5 @@
 import AddIcon from '@mui/icons-material/Add';
-import {Box, Button, Typography} from '@mui/material';
+import {Alert, Box, Button, Typography} from '@mui/material';
 import React, {useState} from 'react';
 
 import {ColumnConfig} from './ColumnConfig';
@@ -93,8 +93,11 @@ export const LeaderboardConfig: React.FC<{
     }));
   };
 
+  const [showAlert, setShowAlert] = useState(true);
+
   return (
-    <Box sx={{width: '100%', p: 2}}>
+    <Box sx={{width: '100%', p: 2, borderLeft: '1px solid #e0e0e0'}}>
+      {showAlert && <TempAlert onClose={() => setShowAlert(false)} />}
       <Typography variant="h5" gutterBottom>
         Leaderboard Configuration
       </Typography>
@@ -144,3 +147,14 @@ export const LeaderboardConfig: React.FC<{
     </Box>
   );
 };
+
+const TempAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
+  return (
+    <Alert severity="info" onClose={onClose}>
+      <Typography variant="body1">
+        This configuration edtior is purely for internal exploration and not the
+        intended final product. It was 100% generated by Cursor.
+      </Typography>
+    </Alert>
+  );
+};

From f6f9db4a0fba8e662e50d191c7cc1e86b0fb9029 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 18:33:12 -0700
Subject: [PATCH 021/169] Cursor: Template Config

---
 .../Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index c4954c1bd1f..d987b44d691 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -96,7 +96,7 @@ export const LeaderboardConfig: React.FC<{
   const [showAlert, setShowAlert] = useState(true);
 
   return (
-    <Box sx={{width: '100%', p: 2, borderLeft: '1px solid #e0e0e0'}}>
+    <Box sx={{width: '100%', p: 2, border: '1px solid #e0e0e0'}}>
       {showAlert && <TempAlert onClose={() => setShowAlert(false)} />}
       <Typography variant="h5" gutterBottom>
         Leaderboard Configuration

From a2a945cf45250114db052cb0056b221cae4aebc4 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 18:50:47 -0700
Subject: [PATCH 022/169] Human + AI: Config Styling

---
 .../LeaderboardPage/LeaderboardConfigType.tsx |   1 +
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  33 ++++--
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 108 +++++++++++-------
 .../Browse3/pages/LeaderboardPage/hooks.ts    |   5 +-
 .../useCurrentLeaderboardConfig.tsx           |   7 ++
 5 files changed, 103 insertions(+), 51 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
index 4ec72a63ad8..62823cc9378 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
@@ -4,6 +4,7 @@ export type Direction = 'asc' | 'desc';
 export type LeaderboardConfigType = {
   version: number;
   config: {
+    description: string;
     columns: Array<{
       dataset: {
         name: string;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 60ae23b9f94..99bbadf3c83 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,6 +1,6 @@
 import {Alert, Box, Typography} from '@mui/material';
 import {Button} from '@wandb/weave/components/Button';
-import React, {useState} from 'react';
+import React, {useCallback, useState} from 'react';
 import {useHistory} from 'react-router-dom';
 
 import {useWeaveflowRouteContext} from '../../context';
@@ -9,7 +9,7 @@ import {useLeaderboardData} from './hooks';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
 import {LeaderboardGrid} from './LeaderboardGrid';
 import {LeaderboardConfig} from './LeaderboardPageConfig';
-import {useCurrentLeaderboardConfig} from './useCurrentLeaderboardConfig';
+import {persistLeaderboardConfig, useCurrentLeaderboardConfig} from './useCurrentLeaderboardConfig';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
@@ -26,23 +26,36 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
 
 const DEFAULT_DESCRIPTION = `# Leaderboard`;
 
+const usePersistedLeaderboardConfig = (): [LeaderboardConfigType, (updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType) => void] => {
+  const [config, setConfigLocal] = useState<LeaderboardConfigType>(
+    useCurrentLeaderboardConfig()
+  );
+
+  const setConfig = (updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType) => {
+    const newConfig = updater(config);
+    setConfigLocal(newConfig);
+    persistLeaderboardConfig(newConfig);
+  };
+
+  return [config, setConfig]
+};
+
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   const {entity, project} = props;
-  const [description, setDescription] = useState('');
-  const {loading, data} = useLeaderboardData(entity, project);
 
-  // const setDescription = useCallback((newDescription: string) => {
-  //   setDescriptionRaw(newDescription.trim());
-  // }, []);
 
   const {peekingRouter} = useWeaveflowRouteContext();
   const history = useHistory();
 
   const [showConfig, setShowConfig] = useState(false);
 
-  const [currentConfig, setCurrentConfig] = useState<LeaderboardConfigType>(
-    useCurrentLeaderboardConfig()
-  );
+  const [currentConfig, setCurrentConfig] = usePersistedLeaderboardConfig()
+  const description = currentConfig.config.description
+  const setDescription = useCallback((newDescription: string) => {
+    setCurrentConfig(newConfig => ({...newConfig, config: {...newConfig.config, description: newDescription}}))
+  }, [setCurrentConfig]); 
+
+  const {loading, data} = useLeaderboardData(entity, project, currentConfig);
 
   const handleCellClick = (
     modelName: string,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index d987b44d691..062d972de97 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -8,12 +8,14 @@ import {ModelConfig} from './ModelConfig';
 
 export const LeaderboardConfig: React.FC<{
   currentConfig: LeaderboardConfigType;
-  onConfigUpdate: (newConfig: LeaderboardConfigType) => void;
+  onConfigUpdate: (
+    updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType
+  ) => void;
 }> = ({currentConfig, onConfigUpdate}) => {
   const [config, setConfig] = useState<LeaderboardConfigType>(currentConfig);
 
   const handleSave = () => {
-    onConfigUpdate(config);
+    onConfigUpdate(prev => config);
   };
 
   const handleCancel = () => {
@@ -96,51 +98,77 @@ export const LeaderboardConfig: React.FC<{
   const [showAlert, setShowAlert] = useState(true);
 
   return (
-    <Box sx={{width: '100%', p: 2, border: '1px solid #e0e0e0'}}>
-      {showAlert && <TempAlert onClose={() => setShowAlert(false)} />}
-      <Typography variant="h5" gutterBottom>
-        Leaderboard Configuration
-      </Typography>
-
-      <Box sx={{mb: 4}}>
-        <Typography variant="h6" gutterBottom>
-          Columns
+    <Box
+      sx={{
+        width: '100%',
+        height: '100%',
+        display: 'flex',
+        flexDirection: 'column',
+        border: '1px solid #e0e0e0',
+      }}>
+      <Box
+        sx={{
+          flexGrow: 1,
+          overflowY: 'auto',
+          p: 2,
+        }}>
+        {showAlert && <TempAlert onClose={() => setShowAlert(false)} />}
+        <Typography variant="h5" gutterBottom>
+          Leaderboard Configuration
         </Typography>
-        {config.config.columns.map((column, index) => (
-          <ColumnConfig
-            key={index}
-            column={column}
-            onUpdate={updatedColumn => handleUpdateColumn(index, updatedColumn)}
-            onRemove={() => handleRemoveColumn(index)}
-          />
-        ))}
-        <Button startIcon={<AddIcon />} onClick={handleAddColumn} sx={{mt: 2}}>
-          Add Column
-        </Button>
-      </Box>
 
-      <Box sx={{mb: 4}}>
-        <Typography variant="h6" gutterBottom>
-          Models
-        </Typography>
-        {config.config.models.map((model, index) => (
-          <ModelConfig
-            key={index}
-            model={model}
-            onUpdate={updatedModel => handleUpdateModel(index, updatedModel)}
-            onRemove={() => handleRemoveModel(index)}
-          />
-        ))}
-        <Button startIcon={<AddIcon />} onClick={handleAddModel} sx={{mt: 2}}>
-          Add Model
-        </Button>
+        <Box sx={{mb: 4}}>
+          <Typography variant="h6" gutterBottom>
+            Columns
+          </Typography>
+          {config.config.columns.map((column, index) => (
+            <ColumnConfig
+              key={index}
+              column={column}
+              onUpdate={updatedColumn =>
+                handleUpdateColumn(index, updatedColumn)
+              }
+              onRemove={() => handleRemoveColumn(index)}
+            />
+          ))}
+          <Button
+            startIcon={<AddIcon />}
+            onClick={handleAddColumn}
+            sx={{mt: 2}}>
+            Add Column
+          </Button>
+        </Box>
+
+        <Box sx={{mb: 4}}>
+          <Typography variant="h6" gutterBottom>
+            Models
+          </Typography>
+          {config.config.models.map((model, index) => (
+            <ModelConfig
+              key={index}
+              model={model}
+              onUpdate={updatedModel => handleUpdateModel(index, updatedModel)}
+              onRemove={() => handleRemoveModel(index)}
+            />
+          ))}
+          <Button startIcon={<AddIcon />} onClick={handleAddModel} sx={{mt: 2}}>
+            Add Model
+          </Button>
+        </Box>
       </Box>
 
-      <Box sx={{display: 'flex', justifyContent: 'flex-end', mt: 4}}>
+      <Box
+        sx={{
+          display: 'flex',
+          justifyContent: 'flex-end',
+          height: '51px',
+          p: 1,
+          borderTop: '1px solid #e0e0e0',
+        }}>
         <Button variant="outlined" onClick={handleCancel} sx={{mr: 2}}>
           Cancel
         </Button>
-        <Button variant="contained" onClick={handleSave}>
+        <Button variant="contained" onClick={handleSave} sx={{mr: 2}}>
           Save
         </Button>
       </Box>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index 4c0fa63202a..3d8b757d76c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -8,6 +8,7 @@ import {
   objectVersionKeyToRefUri,
   opVersionKeyToRefUri,
 } from '../wfReactInterface/utilities';
+import { LeaderboardConfigType } from './LeaderboardConfigType';
 
 export type LeaderboardData = {
   metrics: {
@@ -26,8 +27,10 @@ export type LeaderboardData = {
 
 export const useLeaderboardData = (
   entity: string,
-  project: string
+  project: string,
+  config: LeaderboardConfigType
 ): {loading: boolean; data: LeaderboardData} => {
+  console.log('Fetching leaderboard data', config);
   const {useRootObjectVersions, useCalls} = useWFHooks();
 
   // Get the last 100 (latest) evaluation versions
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx
index 2039fee2d0b..4700e3c2fac 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx
@@ -4,16 +4,23 @@ import {LeaderboardConfigType} from './LeaderboardConfigType';
 
 export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
   // TODO: Implement this
+  console.log("Fetching current leaderboard config")
   return useMemo(() => {
     return {
       version: 1,
       config: {
+      description: '',
         columns: [],
         models: [],
       },
     };
   }, []);
 };
+export const persistLeaderboardConfig = (config: LeaderboardConfigType) => {
+  // TODO: Implement this
+  console.log('Persisting leaderboard config:', config);
+};
+
 export const useDatasetNames = (): string[] => {
   // TODO: Implement this
   return useMemo(() => {

From f1852976755c2b14bce3753000ab1b4373156590 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 18:50:57 -0700
Subject: [PATCH 023/169] Human + AI: Config Styling

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 33 +++++++++++++------
 .../Browse3/pages/LeaderboardPage/hooks.ts    |  2 +-
 .../useCurrentLeaderboardConfig.tsx           |  4 +--
 3 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 99bbadf3c83..3b4af45b58e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -9,7 +9,10 @@ import {useLeaderboardData} from './hooks';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
 import {LeaderboardGrid} from './LeaderboardGrid';
 import {LeaderboardConfig} from './LeaderboardPageConfig';
-import {persistLeaderboardConfig, useCurrentLeaderboardConfig} from './useCurrentLeaderboardConfig';
+import {
+  persistLeaderboardConfig,
+  useCurrentLeaderboardConfig,
+} from './useCurrentLeaderboardConfig';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
@@ -26,34 +29,44 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
 
 const DEFAULT_DESCRIPTION = `# Leaderboard`;
 
-const usePersistedLeaderboardConfig = (): [LeaderboardConfigType, (updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType) => void] => {
+const usePersistedLeaderboardConfig = (): [
+  LeaderboardConfigType,
+  (updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType) => void
+] => {
   const [config, setConfigLocal] = useState<LeaderboardConfigType>(
     useCurrentLeaderboardConfig()
   );
 
-  const setConfig = (updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType) => {
+  const setConfig = (
+    updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType
+  ) => {
     const newConfig = updater(config);
     setConfigLocal(newConfig);
     persistLeaderboardConfig(newConfig);
   };
 
-  return [config, setConfig]
+  return [config, setConfig];
 };
 
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   const {entity, project} = props;
 
-
   const {peekingRouter} = useWeaveflowRouteContext();
   const history = useHistory();
 
   const [showConfig, setShowConfig] = useState(false);
 
-  const [currentConfig, setCurrentConfig] = usePersistedLeaderboardConfig()
-  const description = currentConfig.config.description
-  const setDescription = useCallback((newDescription: string) => {
-    setCurrentConfig(newConfig => ({...newConfig, config: {...newConfig.config, description: newDescription}}))
-  }, [setCurrentConfig]); 
+  const [currentConfig, setCurrentConfig] = usePersistedLeaderboardConfig();
+  const description = currentConfig.config.description;
+  const setDescription = useCallback(
+    (newDescription: string) => {
+      setCurrentConfig(newConfig => ({
+        ...newConfig,
+        config: {...newConfig.config, description: newDescription},
+      }));
+    },
+    [setCurrentConfig]
+  );
 
   const {loading, data} = useLeaderboardData(entity, project, currentConfig);
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index 3d8b757d76c..13ec658a8ae 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -8,7 +8,7 @@ import {
   objectVersionKeyToRefUri,
   opVersionKeyToRefUri,
 } from '../wfReactInterface/utilities';
-import { LeaderboardConfigType } from './LeaderboardConfigType';
+import {LeaderboardConfigType} from './LeaderboardConfigType';
 
 export type LeaderboardData = {
   metrics: {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx
index 4700e3c2fac..a922eeeb3ed 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx
@@ -4,12 +4,12 @@ import {LeaderboardConfigType} from './LeaderboardConfigType';
 
 export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
   // TODO: Implement this
-  console.log("Fetching current leaderboard config")
+  console.log('Fetching current leaderboard config');
   return useMemo(() => {
     return {
       version: 1,
       config: {
-      description: '',
+        description: '',
         columns: [],
         models: [],
       },

From 9d3a40334ce155805d9b46c2b926208d344f7796 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 19:01:30 -0700
Subject: [PATCH 024/169] Human: Leaderboard state

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 51 +++++++++++--------
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 16 +++---
 2 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 3b4af45b58e..20639e59f14 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -29,23 +29,20 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
 
 const DEFAULT_DESCRIPTION = `# Leaderboard`;
 
-const usePersistedLeaderboardConfig = (): [
-  LeaderboardConfigType,
-  (updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType) => void
-] => {
-  const [config, setConfigLocal] = useState<LeaderboardConfigType>(
-    useCurrentLeaderboardConfig()
-  );
+const usePersistedLeaderboardConfig = () => {
+  const initialConfig = useCurrentLeaderboardConfig();
+  const [config, setConfigLocal] =
+    useState<LeaderboardConfigType>(initialConfig);
 
-  const setConfig = (
-    updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType
-  ) => {
-    const newConfig = updater(config);
-    setConfigLocal(newConfig);
-    persistLeaderboardConfig(newConfig);
-  };
+  const persistConfig = useCallback(() => {
+    persistLeaderboardConfig(config);
+  }, [config]);
+
+  const cancelChanges = useCallback(() => {
+    setConfigLocal(initialConfig);
+  }, [initialConfig]);
 
-  return [config, setConfig];
+  return {config, setConfigLocal, persistConfig, cancelChanges};
 };
 
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
@@ -56,16 +53,22 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
 
   const [showConfig, setShowConfig] = useState(false);
 
-  const [currentConfig, setCurrentConfig] = usePersistedLeaderboardConfig();
+  const {
+    config: currentConfig,
+    setConfigLocal,
+    persistConfig,
+    cancelChanges,
+  } = usePersistedLeaderboardConfig();
   const description = currentConfig.config.description;
   const setDescription = useCallback(
     (newDescription: string) => {
-      setCurrentConfig(newConfig => ({
+      setConfigLocal(newConfig => ({
         ...newConfig,
         config: {...newConfig.config, description: newDescription},
       }));
+      persistConfig();
     },
-    [setCurrentConfig]
+    [setConfigLocal, persistConfig]
   );
 
   const {loading, data} = useLeaderboardData(entity, project, currentConfig);
@@ -125,8 +128,16 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
         />
         {showConfig && (
           <LeaderboardConfig
-            currentConfig={currentConfig}
-            onConfigUpdate={setCurrentConfig}
+            config={currentConfig}
+            onCancel={() => {
+              cancelChanges();
+              setShowConfig(false);
+            }}
+            onPersist={() => {
+              persistConfig();
+              setShowConfig(false);
+            }}
+            setConfig={setConfigLocal}
           />
         )}
       </Box>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 062d972de97..3cb6db28a85 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -7,19 +7,19 @@ import {LeaderboardConfigType} from './LeaderboardConfigType';
 import {ModelConfig} from './ModelConfig';
 
 export const LeaderboardConfig: React.FC<{
-  currentConfig: LeaderboardConfigType;
-  onConfigUpdate: (
-    updater: (oldConfig: LeaderboardConfigType) => LeaderboardConfigType
+  config: LeaderboardConfigType;
+  onCancel: () => void;
+  onPersist: () => void;
+  setConfig: (
+    updater: (config: LeaderboardConfigType) => LeaderboardConfigType
   ) => void;
-}> = ({currentConfig, onConfigUpdate}) => {
-  const [config, setConfig] = useState<LeaderboardConfigType>(currentConfig);
-
+}> = ({config, setConfig, onPersist, onCancel}) => {
   const handleSave = () => {
-    onConfigUpdate(prev => config);
+    onPersist();
   };
 
   const handleCancel = () => {
-    setConfig(currentConfig);
+    onCancel();
   };
 
   const handleAddColumn = () => {

From c464fb4011db4f6a90f04fe8b81d8a9e7c60230f Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 19:02:45 -0700
Subject: [PATCH 025/169] Humnan: Rename file

---
 .../pages/LeaderboardPage/ColumnConfig.tsx    |  6 +--
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  8 ++--
 .../LeaderboardPageContent.tsx                | 28 ++++++++++++
 .../pages/LeaderboardPage/MetricConfig.tsx    |  2 +-
 .../pages/LeaderboardPage/ModelConfig.tsx     | 43 +++++++------------
 .../pages/LeaderboardPage/ScoreConfig.tsx     |  6 +--
 ...rdConfig.tsx => leaderboardConfigQuery.ts} |  0
 7 files changed, 54 insertions(+), 39 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageContent.tsx
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/{useCurrentLeaderboardConfig.tsx => leaderboardConfigQuery.ts} (100%)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
index 8f1f5714e44..2d3559c8a86 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
@@ -13,12 +13,12 @@ import {
 } from '@mui/material';
 import React from 'react';
 
-import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
-import {ScoreConfig} from './ScoreConfig';
 import {
   useDatasetNames,
   useDatasetVersionsForDatasetName,
-} from './useCurrentLeaderboardConfig';
+} from './leaderboardConfigQuery';
+import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
+import {ScoreConfig} from './ScoreConfig';
 
 export const ColumnConfig: React.FC<{
   column: LeaderboardConfigType['config']['columns'][0];
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 20639e59f14..f26d4c7a2b8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -6,13 +6,13 @@ import {useHistory} from 'react-router-dom';
 import {useWeaveflowRouteContext} from '../../context';
 import {EditableMarkdown} from './EditableMarkdown';
 import {useLeaderboardData} from './hooks';
-import {LeaderboardConfigType} from './LeaderboardConfigType';
-import {LeaderboardGrid} from './LeaderboardGrid';
-import {LeaderboardConfig} from './LeaderboardPageConfig';
 import {
   persistLeaderboardConfig,
   useCurrentLeaderboardConfig,
-} from './useCurrentLeaderboardConfig';
+} from './leaderboardConfigQuery';
+import {LeaderboardConfigType} from './LeaderboardConfigType';
+import {LeaderboardGrid} from './LeaderboardGrid';
+import {LeaderboardConfig} from './LeaderboardPageConfig';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageContent.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageContent.tsx
new file mode 100644
index 00000000000..5a1332444e5
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageContent.tsx
@@ -0,0 +1,28 @@
+import {Alert,Box, Typography} from '@mui/material';
+import React from 'react';
+
+import {useCurrentLeaderboardConfig} from './leaderboardConfigQuery';
+import {LeaderboardConfig} from './LeaderboardPageConfig';
+
+export const LeaderboardPageContent: React.FC = () => {
+  const currentConfig = useCurrentLeaderboardConfig();
+
+  const handleConfigUpdate = (newConfig: typeof currentConfig) => {
+    // TODO: Implement this
+    console.log('New config:', newConfig);
+  };
+
+  return (
+    <Box sx={{width: '100%', p: 2}}>
+      <Alert severity="info" sx={{mb: 3}}>
+        <Typography variant="body1">
+          This is an unlisted beta page. Features may be incomplete or subject to change.
+        </Typography>
+      </Alert>
+      <LeaderboardConfig
+        currentConfig={currentConfig}
+        onConfigUpdate={handleConfigUpdate}
+      />
+    </Box>
+  );
+};
\ No newline at end of file
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
index 26fa337002a..d4c3ecb020a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
@@ -12,8 +12,8 @@ import {
 } from '@mui/material';
 import React from 'react';
 
+import {useMetricPathsForDatasetAndScorer} from './leaderboardConfigQuery';
 import {Direction, LeaderboardConfigType} from './LeaderboardConfigType';
-import {useMetricPathsForDatasetAndScorer} from './useCurrentLeaderboardConfig';
 
 export const MetricConfig: React.FC<{
   metric: LeaderboardConfigType['config']['columns'][0]['scores'][0]['metrics'][0];
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
index d7e05fa7992..6f7f468943f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
@@ -11,44 +11,34 @@ import {
 } from '@mui/material';
 import React from 'react';
 
-import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
-import {
-  useModelNames,
-  useModelVersionsForModelName,
-} from './useCurrentLeaderboardConfig';
+import { useModelNames, useModelVersionsForModelName } from './leaderboardConfigQuery';
+import { LeaderboardConfigType, VersionSpec } from './LeaderboardConfigType';
 
 export const ModelConfig: React.FC<{
   model: LeaderboardConfigType['config']['models'][0];
-  onUpdate: (
-    updatedModel: LeaderboardConfigType['config']['models'][0]
-  ) => void;
+  onUpdate: (updatedModel: LeaderboardConfigType['config']['models'][0]) => void;
   onRemove: () => void;
-}> = ({model, onUpdate, onRemove}) => {
+}> = ({ model, onUpdate, onRemove }) => {
   const modelNames = useModelNames();
   const modelVersions = useModelVersionsForModelName(model.name);
 
   return (
-    <Box sx={{mb: 2, p: 2, border: '1px solid #ccc', borderRadius: 1}}>
-      <Stack
-        direction="row"
-        justifyContent="space-between"
-        alignItems="center"
-        sx={{mb: 2}}>
+    <Box sx={{ mb: 2, p: 2, border: '1px solid #ccc', borderRadius: 1 }}>
+      <Stack direction="row" justifyContent="space-between" alignItems="center" sx={{ mb: 2 }}>
         <Typography variant="subtitle1">Model</Typography>
         <IconButton onClick={onRemove} size="small">
           <DeleteIcon />
         </IconButton>
       </Stack>
-      <FormControl fullWidth sx={{mb: 2}}>
+      <FormControl fullWidth sx={{ mb: 2 }}>
         <InputLabel>Model Name</InputLabel>
         <Select
           value={model.name}
           label="Model Name"
-          onChange={e => onUpdate({...model, name: e.target.value})}>
+          onChange={(e) => onUpdate({ ...model, name: e.target.value })}
+        >
           {modelNames.map(name => (
-            <MenuItem key={name} value={name}>
-              {name}
-            </MenuItem>
+            <MenuItem key={name} value={name}>{name}</MenuItem>
           ))}
         </Select>
       </FormControl>
@@ -57,18 +47,15 @@ export const ModelConfig: React.FC<{
         <Select
           value={model.version}
           label="Model Version"
-          onChange={e =>
-            onUpdate({...model, version: e.target.value as VersionSpec})
-          }>
+          onChange={(e) => onUpdate({ ...model, version: e.target.value as VersionSpec })}
+        >
           <MenuItem value="latest">Latest</MenuItem>
           <MenuItem value="all">All</MenuItem>
-          {modelVersions.map(({version, versionIndex}) => (
-            <MenuItem key={version} value={version}>
-              Version {versionIndex}
-            </MenuItem>
+          {modelVersions.map(({ version, versionIndex }) => (
+            <MenuItem key={version} value={version}>Version {versionIndex}</MenuItem>
           ))}
         </Select>
       </FormControl>
     </Box>
   );
-};
+};
\ No newline at end of file
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
index 752249dee56..cf138908070 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
@@ -13,12 +13,12 @@ import {
 } from '@mui/material';
 import React from 'react';
 
-import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
-import {MetricConfig} from './MetricConfig';
 import {
   useScorerNamesForDataset,
   useScorerVersionsForDatasetAndScorer,
-} from './useCurrentLeaderboardConfig';
+} from './leaderboardConfigQuery';
+import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
+import {MetricConfig} from './MetricConfig';
 
 export const ScoreConfig: React.FC<{
   score: LeaderboardConfigType['config']['columns'][0]['scores'][0];
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
similarity index 100%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/useCurrentLeaderboardConfig.tsx
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts

From 3f5b11923f398028450ec74755c4c8edd72fe58d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 19:09:33 -0700
Subject: [PATCH 026/169] Lint

---
 .../pages/LeaderboardPage/ColumnConfig.tsx    |  6 ++-
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  2 +
 .../LeaderboardPage/LeaderboardPageConfig.tsx |  6 ++-
 .../LeaderboardPageContent.tsx                | 28 ------------
 .../pages/LeaderboardPage/ModelConfig.tsx     | 43 ++++++++++++-------
 .../LeaderboardPage/leaderboardConfigQuery.ts | 23 ++++++----
 6 files changed, 54 insertions(+), 54 deletions(-)
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageContent.tsx

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
index 2d3559c8a86..d3fc2a411ca 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
@@ -21,13 +21,15 @@ import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
 import {ScoreConfig} from './ScoreConfig';
 
 export const ColumnConfig: React.FC<{
+  entity: string;
+  project: string;
   column: LeaderboardConfigType['config']['columns'][0];
   onUpdate: (
     updatedColumn: LeaderboardConfigType['config']['columns'][0]
   ) => void;
   onRemove: () => void;
-}> = ({column, onUpdate, onRemove}) => {
-  const datasetNames = useDatasetNames();
+}> = ({entity, project, column, onUpdate, onRemove}) => {
+  const datasetNames = useDatasetNames(entity, project);
   const datasetVersions = useDatasetVersionsForDatasetName(column.dataset.name);
 
   const handleAddScore = () => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index f26d4c7a2b8..b1df55e8b47 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -128,6 +128,8 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
         />
         {showConfig && (
           <LeaderboardConfig
+            entity={entity}
+            project={project}
             config={currentConfig}
             onCancel={() => {
               cancelChanges();
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 3cb6db28a85..4746ab99a20 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -7,13 +7,15 @@ import {LeaderboardConfigType} from './LeaderboardConfigType';
 import {ModelConfig} from './ModelConfig';
 
 export const LeaderboardConfig: React.FC<{
+  entity: string;
+  project: string;
   config: LeaderboardConfigType;
   onCancel: () => void;
   onPersist: () => void;
   setConfig: (
     updater: (config: LeaderboardConfigType) => LeaderboardConfigType
   ) => void;
-}> = ({config, setConfig, onPersist, onCancel}) => {
+}> = ({entity, project, config, setConfig, onPersist, onCancel}) => {
   const handleSave = () => {
     onPersist();
   };
@@ -123,6 +125,8 @@ export const LeaderboardConfig: React.FC<{
           </Typography>
           {config.config.columns.map((column, index) => (
             <ColumnConfig
+              entity={entity}
+              project={project}
               key={index}
               column={column}
               onUpdate={updatedColumn =>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageContent.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageContent.tsx
deleted file mode 100644
index 5a1332444e5..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageContent.tsx
+++ /dev/null
@@ -1,28 +0,0 @@
-import {Alert,Box, Typography} from '@mui/material';
-import React from 'react';
-
-import {useCurrentLeaderboardConfig} from './leaderboardConfigQuery';
-import {LeaderboardConfig} from './LeaderboardPageConfig';
-
-export const LeaderboardPageContent: React.FC = () => {
-  const currentConfig = useCurrentLeaderboardConfig();
-
-  const handleConfigUpdate = (newConfig: typeof currentConfig) => {
-    // TODO: Implement this
-    console.log('New config:', newConfig);
-  };
-
-  return (
-    <Box sx={{width: '100%', p: 2}}>
-      <Alert severity="info" sx={{mb: 3}}>
-        <Typography variant="body1">
-          This is an unlisted beta page. Features may be incomplete or subject to change.
-        </Typography>
-      </Alert>
-      <LeaderboardConfig
-        currentConfig={currentConfig}
-        onConfigUpdate={handleConfigUpdate}
-      />
-    </Box>
-  );
-};
\ No newline at end of file
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
index 6f7f468943f..eff5ea4c943 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
@@ -11,34 +11,44 @@ import {
 } from '@mui/material';
 import React from 'react';
 
-import { useModelNames, useModelVersionsForModelName } from './leaderboardConfigQuery';
-import { LeaderboardConfigType, VersionSpec } from './LeaderboardConfigType';
+import {
+  useModelNames,
+  useModelVersionsForModelName,
+} from './leaderboardConfigQuery';
+import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
 
 export const ModelConfig: React.FC<{
   model: LeaderboardConfigType['config']['models'][0];
-  onUpdate: (updatedModel: LeaderboardConfigType['config']['models'][0]) => void;
+  onUpdate: (
+    updatedModel: LeaderboardConfigType['config']['models'][0]
+  ) => void;
   onRemove: () => void;
-}> = ({ model, onUpdate, onRemove }) => {
+}> = ({model, onUpdate, onRemove}) => {
   const modelNames = useModelNames();
   const modelVersions = useModelVersionsForModelName(model.name);
 
   return (
-    <Box sx={{ mb: 2, p: 2, border: '1px solid #ccc', borderRadius: 1 }}>
-      <Stack direction="row" justifyContent="space-between" alignItems="center" sx={{ mb: 2 }}>
+    <Box sx={{mb: 2, p: 2, border: '1px solid #ccc', borderRadius: 1}}>
+      <Stack
+        direction="row"
+        justifyContent="space-between"
+        alignItems="center"
+        sx={{mb: 2}}>
         <Typography variant="subtitle1">Model</Typography>
         <IconButton onClick={onRemove} size="small">
           <DeleteIcon />
         </IconButton>
       </Stack>
-      <FormControl fullWidth sx={{ mb: 2 }}>
+      <FormControl fullWidth sx={{mb: 2}}>
         <InputLabel>Model Name</InputLabel>
         <Select
           value={model.name}
           label="Model Name"
-          onChange={(e) => onUpdate({ ...model, name: e.target.value })}
-        >
+          onChange={e => onUpdate({...model, name: e.target.value})}>
           {modelNames.map(name => (
-            <MenuItem key={name} value={name}>{name}</MenuItem>
+            <MenuItem key={name} value={name}>
+              {name}
+            </MenuItem>
           ))}
         </Select>
       </FormControl>
@@ -47,15 +57,18 @@ export const ModelConfig: React.FC<{
         <Select
           value={model.version}
           label="Model Version"
-          onChange={(e) => onUpdate({ ...model, version: e.target.value as VersionSpec })}
-        >
+          onChange={e =>
+            onUpdate({...model, version: e.target.value as VersionSpec})
+          }>
           <MenuItem value="latest">Latest</MenuItem>
           <MenuItem value="all">All</MenuItem>
-          {modelVersions.map(({ version, versionIndex }) => (
-            <MenuItem key={version} value={version}>Version {versionIndex}</MenuItem>
+          {modelVersions.map(({version, versionIndex}) => (
+            <MenuItem key={version} value={version}>
+              Version {versionIndex}
+            </MenuItem>
           ))}
         </Select>
       </FormControl>
     </Box>
   );
-};
\ No newline at end of file
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
index a922eeeb3ed..e39be446d78 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
@@ -1,5 +1,6 @@
 import {useMemo} from 'react';
 
+import {useWFHooks} from '../wfReactInterface/context';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
 
 export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
@@ -21,15 +22,21 @@ export const persistLeaderboardConfig = (config: LeaderboardConfigType) => {
   console.log('Persisting leaderboard config:', config);
 };
 
-export const useDatasetNames = (): string[] => {
-  // TODO: Implement this
+export const useDatasetNames = (entity: string, project: string): string[] => {
+  const {useRootObjectVersions} = useWFHooks();
+  const names = useRootObjectVersions(
+    entity,
+    project,
+    {
+      baseObjectClasses: ['Dataset'],
+      latestOnly: true,
+    },
+    100,
+    true
+  );
   return useMemo(() => {
-    return [
-      'leaderboard-test-dataset',
-      'leaderboard-test-dataset-2',
-      'leaderboard-test-dataset-3',
-    ];
-  }, []);
+    return names.result?.map(obj => obj.objectId) ?? [];
+  }, [names]);
 };
 export const useDatasetVersionsForDatasetName = (
   datasetName: string

From 6e6029898dba3736d5181c32274f72aecbd85ddc Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 19:14:33 -0700
Subject: [PATCH 027/169] Human: dataset lookups

---
 .../pages/LeaderboardPage/ColumnConfig.tsx    |  8 +++--
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  2 ++
 .../LeaderboardPage/leaderboardConfigQuery.ts | 29 ++++++++++++-------
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
index d3fc2a411ca..e31d3a6e2ed 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
@@ -30,7 +30,11 @@ export const ColumnConfig: React.FC<{
   onRemove: () => void;
 }> = ({entity, project, column, onUpdate, onRemove}) => {
   const datasetNames = useDatasetNames(entity, project);
-  const datasetVersions = useDatasetVersionsForDatasetName(column.dataset.name);
+  const datasetVersions = useDatasetVersionsForDatasetName(
+    entity,
+    project,
+    column.dataset.name
+  );
 
   const handleAddScore = () => {
     onUpdate({
@@ -109,7 +113,7 @@ export const ColumnConfig: React.FC<{
           <MenuItem value="all">All</MenuItem>
           {datasetVersions.map(({version, versionIndex}) => (
             <MenuItem key={version} value={version}>
-              Version {versionIndex}
+              Version {versionIndex}: {version}
             </MenuItem>
           ))}
         </Select>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index b1df55e8b47..f7a5b7bd37f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,4 +1,5 @@
 import {Alert, Box, Typography} from '@mui/material';
+import { useTraceUpdate } from '@wandb/weave/common/util/hooks';
 import {Button} from '@wandb/weave/components/Button';
 import React, {useCallback, useState} from 'react';
 import {useHistory} from 'react-router-dom';
@@ -72,6 +73,7 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   );
 
   const {loading, data} = useLeaderboardData(entity, project, currentConfig);
+  useTraceUpdate('a', {entity, project, currentConfig});
 
   const handleCellClick = (
     modelName: string,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
index e39be446d78..fbd6d366b6c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
@@ -24,7 +24,7 @@ export const persistLeaderboardConfig = (config: LeaderboardConfigType) => {
 
 export const useDatasetNames = (entity: string, project: string): string[] => {
   const {useRootObjectVersions} = useWFHooks();
-  const names = useRootObjectVersions(
+  const query = useRootObjectVersions(
     entity,
     project,
     {
@@ -35,20 +35,29 @@ export const useDatasetNames = (entity: string, project: string): string[] => {
     true
   );
   return useMemo(() => {
-    return names.result?.map(obj => obj.objectId) ?? [];
-  }, [names]);
+    return query.result?.map(obj => obj.objectId) ?? [];
+  }, [query]);
 };
 export const useDatasetVersionsForDatasetName = (
+  entity: string,
+  project: string,
   datasetName: string
 ): Array<{version: string; versionIndex: number}> => {
-  // TODO: Implement this
+  const {useRootObjectVersions} = useWFHooks();
+  const query = useRootObjectVersions(
+    entity,
+    project,
+    {
+      baseObjectClasses: ['Dataset'],
+      objectIds: [datasetName],
+    },
+    100,
+    true
+  );
+
   return useMemo(() => {
-    return [
-      {version: 'dug657ioy8j1', versionIndex: 0},
-      {version: 'dnkjubyhvasd', versionIndex: 1},
-      {version: 'dadsgf3f451d', versionIndex: 2},
-    ];
-  }, []);
+    return query.result?.map(obj => ({version: obj.versionHash, versionIndex: obj.versionIndex})) ?? [];
+  }, [query]);
 };
 export const useScorerNamesForDataset = (
   datasetName: string,

From ce8cc448cd6880bd9a60a89bf065f765dfba9461 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 19:49:27 -0700
Subject: [PATCH 028/169] Human: Kind of a bad checkpoint, but need to pause

---
 .../pages/LeaderboardPage/ColumnConfig.tsx    |  2 +
 .../pages/LeaderboardPage/ScoreConfig.tsx     | 11 ++-
 .../LeaderboardPage/leaderboardConfigQuery.ts | 90 ++++++++++++++++---
 3 files changed, 88 insertions(+), 15 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
index e31d3a6e2ed..aeeb172d4ae 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
@@ -120,6 +120,8 @@ export const ColumnConfig: React.FC<{
       </FormControl>
       {column.scores.map((score, index) => (
         <ScoreConfig
+          entity={entity}
+          project={project}
           key={index}
           score={score}
           datasetName={column.dataset.name}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
index cf138908070..112c78b458d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
@@ -21,6 +21,8 @@ import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
 import {MetricConfig} from './MetricConfig';
 
 export const ScoreConfig: React.FC<{
+  entity: string;
+  project: string;
   score: LeaderboardConfigType['config']['columns'][0]['scores'][0];
   datasetName: string;
   datasetVersion: string;
@@ -28,8 +30,13 @@ export const ScoreConfig: React.FC<{
     updatedScore: LeaderboardConfigType['config']['columns'][0]['scores'][0]
   ) => void;
   onRemove: () => void;
-}> = ({score, datasetName, datasetVersion, onUpdate, onRemove}) => {
-  const scorerNames = useScorerNamesForDataset(datasetName, datasetVersion);
+}> = ({ entity, project, score, datasetName, datasetVersion, onUpdate, onRemove}) => {
+  const scorerNames = useScorerNamesForDataset(
+    entity,
+    project,
+    datasetName,
+    datasetVersion
+  );
   const scorerVersions = useScorerVersionsForDatasetAndScorer(
     datasetName,
     datasetVersion,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
index fbd6d366b6c..8184458af64 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
@@ -1,7 +1,8 @@
 import {useMemo} from 'react';
 
 import {useWFHooks} from '../wfReactInterface/context';
-import {LeaderboardConfigType} from './LeaderboardConfigType';
+import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
+import { parseRefMaybe } from '../../../Browse2/SmallRef';
 
 export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
   // TODO: Implement this
@@ -24,19 +25,20 @@ export const persistLeaderboardConfig = (config: LeaderboardConfigType) => {
 
 export const useDatasetNames = (entity: string, project: string): string[] => {
   const {useRootObjectVersions} = useWFHooks();
-  const query = useRootObjectVersions(
+  const evalQuery = useRootObjectVersions(
     entity,
     project,
     {
-      baseObjectClasses: ['Dataset'],
-      latestOnly: true,
+      baseObjectClasses: ['Evaluation'],
     },
+    // This 100 is very limited
     100,
-    true
   );
+
   return useMemo(() => {
-    return query.result?.map(obj => obj.objectId) ?? [];
-  }, [query]);
+    const datasets = (evalQuery.result ?? []).map(e => parseRefMaybe(e.val.dataset)?.artifactName).filter(name => !!name).sort().filter((name, index, self) => self.indexOf(name) === index) as string[];
+    return datasets;
+  }, [evalQuery.result]);
 };
 export const useDatasetVersionsForDatasetName = (
   entity: string,
@@ -55,18 +57,80 @@ export const useDatasetVersionsForDatasetName = (
     true
   );
 
-  return useMemo(() => {
-    return query.result?.map(obj => ({version: obj.versionHash, versionIndex: obj.versionIndex})) ?? [];
+  const allVersions = useMemo(() => {
+    return (query.result ?? []).map(obj => ({version: obj.versionHash, versionIndex: obj.versionIndex})) ?? [];
   }, [query]);
+
+  const evalQuery = useRootObjectVersions(
+    entity,
+    project,
+    {
+      baseObjectClasses: ['Evaluation'],
+    },
+    // This 100 is very limited
+    100,
+    false,
+    {skip: allVersions.length === 0}
+  );
+
+  return useMemo(() => {
+    
+    const datasets = (evalQuery.result ?? []).map(e => {
+      const ref = parseRefMaybe(e.val.dataset)
+      if (!ref) {
+        return null;
+      }
+      if (ref.artifactName !== datasetName) {
+        return null;
+      }
+      const match = allVersions.find(v => v.version === ref.artifactVersion)
+      return match
+    }).filter(version => !!version)
+    return datasets;
+  }, [allVersions, datasetName, evalQuery.result]);
 };
 export const useScorerNamesForDataset = (
+  entity: string,
+  project: string,
   datasetName: string,
-  datasetVersion: string
+  datasetVersion: VersionSpec
 ): string[] => {
-  // TODO: Implement this
+  // This one is a bit more involved:
+  // 1. Lookup all the evaluations that contain this dataset
+  // 2. Of each evaluation, get the scorer names
+
+  const {useRootObjectVersions} = useWFHooks();
+  const evalQuery = useRootObjectVersions(
+    entity,
+    project,
+    {
+      baseObjectClasses: ['Evaluation'],
+    },
+    // This 100 is very limited
+    100,
+  );
+
   return useMemo(() => {
-    return ['scorer-1', 'scorer-2', 'scorer-3'];
-  }, []);
+    const eval_results = evalQuery.result?.filter(obj => {
+      const ref = parseRefMaybe(obj.val.dataset ?? "")
+      if (!ref) {
+        return false;
+      }
+      if (ref.artifactName !== datasetName) {
+        return false;
+      }
+      if (datasetVersion === "latest" || datasetVersion === "all") {
+        return true;
+      }
+      return ref.artifactVersion === datasetVersion;
+    }) ?? [];
+    const res = eval_results.map(obj => obj.val.scorers ?? []).flat().map(scorer => parseRefMaybe(scorer)?.artifactName).filter(name => !!name).sort() // .filter((name, index, self) => self.indexOf(name) === index) as string[];
+
+    return res;
+  }, [datasetName, datasetVersion, evalQuery.result])
+  
+
+
 };
 export const useScorerVersionsForDatasetAndScorer = (
   datasetName: string,

From 4c781629f680fa648de60ea3be1cebc5bcebd429 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 21:42:03 -0700
Subject: [PATCH 029/169] Human: Making incremental progress

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx |   2 +-
 .../pages/LeaderboardPage/ScoreConfig.tsx     |  14 +-
 .../LeaderboardPage/leaderboardConfigQuery.ts | 174 +++++++++++++-----
 3 files changed, 145 insertions(+), 45 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index f7a5b7bd37f..c0be97f780e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,5 +1,5 @@
 import {Alert, Box, Typography} from '@mui/material';
-import { useTraceUpdate } from '@wandb/weave/common/util/hooks';
+import {useTraceUpdate} from '@wandb/weave/common/util/hooks';
 import {Button} from '@wandb/weave/components/Button';
 import React, {useCallback, useState} from 'react';
 import {useHistory} from 'react-router-dom';
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
index 112c78b458d..16a556fd401 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
@@ -30,7 +30,15 @@ export const ScoreConfig: React.FC<{
     updatedScore: LeaderboardConfigType['config']['columns'][0]['scores'][0]
   ) => void;
   onRemove: () => void;
-}> = ({ entity, project, score, datasetName, datasetVersion, onUpdate, onRemove}) => {
+}> = ({
+  entity,
+  project,
+  score,
+  datasetName,
+  datasetVersion,
+  onUpdate,
+  onRemove,
+}) => {
   const scorerNames = useScorerNamesForDataset(
     entity,
     project,
@@ -38,6 +46,8 @@ export const ScoreConfig: React.FC<{
     datasetVersion
   );
   const scorerVersions = useScorerVersionsForDatasetAndScorer(
+    entity,
+    project,
     datasetName,
     datasetVersion,
     score.scorer.name
@@ -117,7 +127,7 @@ export const ScoreConfig: React.FC<{
           <MenuItem value="all">All</MenuItem>
           {scorerVersions.map(({version, versionIndex}) => (
             <MenuItem key={version} value={version}>
-              Version {versionIndex}
+              Version {versionIndex}: {version}
             </MenuItem>
           ))}
         </Select>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
index 8184458af64..2d973ba7c2d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
@@ -1,8 +1,8 @@
 import {useMemo} from 'react';
 
+import {parseRefMaybe} from '../../../Browse2/SmallRef';
 import {useWFHooks} from '../wfReactInterface/context';
 import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
-import { parseRefMaybe } from '../../../Browse2/SmallRef';
 
 export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
   // TODO: Implement this
@@ -32,11 +32,15 @@ export const useDatasetNames = (entity: string, project: string): string[] => {
       baseObjectClasses: ['Evaluation'],
     },
     // This 100 is very limited
-    100,
+    100
   );
 
   return useMemo(() => {
-    const datasets = (evalQuery.result ?? []).map(e => parseRefMaybe(e.val.dataset)?.artifactName).filter(name => !!name).sort().filter((name, index, self) => self.indexOf(name) === index) as string[];
+    const datasets = (evalQuery.result ?? [])
+      .map(e => parseRefMaybe(e.val.dataset)?.artifactName)
+      .filter(name => !!name)
+      .sort()
+      .filter((name, index, self) => self.indexOf(name) === index) as string[];
     return datasets;
   }, [evalQuery.result]);
 };
@@ -58,7 +62,12 @@ export const useDatasetVersionsForDatasetName = (
   );
 
   const allVersions = useMemo(() => {
-    return (query.result ?? []).map(obj => ({version: obj.versionHash, versionIndex: obj.versionIndex})) ?? [];
+    return (
+      (query.result ?? []).map(obj => ({
+        version: obj.versionHash,
+        versionIndex: obj.versionIndex,
+      })) ?? []
+    );
   }, [query]);
 
   const evalQuery = useRootObjectVersions(
@@ -74,18 +83,22 @@ export const useDatasetVersionsForDatasetName = (
   );
 
   return useMemo(() => {
-    
-    const datasets = (evalQuery.result ?? []).map(e => {
-      const ref = parseRefMaybe(e.val.dataset)
-      if (!ref) {
-        return null;
-      }
-      if (ref.artifactName !== datasetName) {
-        return null;
-      }
-      const match = allVersions.find(v => v.version === ref.artifactVersion)
-      return match
-    }).filter(version => !!version)
+    const datasets = (evalQuery.result ?? [])
+      .map(e => {
+        const ref = parseRefMaybe(e.val.dataset);
+        if (!ref) {
+          return null;
+        }
+        if (ref.artifactName !== datasetName) {
+          return null;
+        }
+        const match = allVersions.find(v => v.version === ref.artifactVersion);
+        return match;
+      })
+      .filter(version => !!version) as Array<{
+      version: string;
+      versionIndex: number;
+    }>;
     return datasets;
   }, [allVersions, datasetName, evalQuery.result]);
 };
@@ -107,45 +120,120 @@ export const useScorerNamesForDataset = (
       baseObjectClasses: ['Evaluation'],
     },
     // This 100 is very limited
-    100,
+    100
   );
 
   return useMemo(() => {
-    const eval_results = evalQuery.result?.filter(obj => {
-      const ref = parseRefMaybe(obj.val.dataset ?? "")
-      if (!ref) {
-        return false;
-      }
-      if (ref.artifactName !== datasetName) {
-        return false;
-      }
-      if (datasetVersion === "latest" || datasetVersion === "all") {
-        return true;
-      }
-      return ref.artifactVersion === datasetVersion;
-    }) ?? [];
-    const res = eval_results.map(obj => obj.val.scorers ?? []).flat().map(scorer => parseRefMaybe(scorer)?.artifactName).filter(name => !!name).sort() // .filter((name, index, self) => self.indexOf(name) === index) as string[];
+    const evalResults =
+      evalQuery.result?.filter(obj => {
+        const ref = parseRefMaybe(obj.val.dataset ?? '');
+        if (!ref) {
+          return false;
+        }
+        if (ref.artifactName !== datasetName) {
+          return false;
+        }
+        if (datasetVersion === 'latest' || datasetVersion === 'all') {
+          return true;
+        }
+        return ref.artifactVersion === datasetVersion;
+      }) ?? [];
+    const res = evalResults
+      .map(obj => obj.val.scorers ?? [])
+      .flat()
+      .map(scorer => parseRefMaybe(scorer)?.artifactName)
+      .filter(name => !!name)
+      .sort() as string[]; // .filter((name, index, self) => self.indexOf(name) === index) as string[];
 
     return res;
-  }, [datasetName, datasetVersion, evalQuery.result])
-  
-
-
+  }, [datasetName, datasetVersion, evalQuery.result]);
 };
 export const useScorerVersionsForDatasetAndScorer = (
+  entity: string,
+  project: string,
   datasetName: string,
   datasetVersion: string,
   scorerName: string
 ): Array<{version: string; versionIndex: number}> => {
-  // TODO: Implement this
+  const {useRootObjectVersions, useOpVersions} = useWFHooks();
+  const query1 = useRootObjectVersions(
+    entity,
+    project,
+    {
+      objectIds: [scorerName],
+    },
+    100,
+    true
+  );
+
+  const query2 = useOpVersions(
+    entity,
+    project,
+    {
+      opIds: [scorerName],
+    },
+    100,
+    true
+  );
+
+  const allVersions = useMemo(() => {
+    const versions1 =
+      (query1.result ?? []).map(obj => ({
+        version: obj.versionHash,
+        versionIndex: obj.versionIndex,
+      })) ?? [];
+    const versions2 =
+      (query2.result ?? []).map(obj => ({
+        version: obj.versionHash,
+        versionIndex: obj.versionIndex,
+      })) ?? [];
+    return versions1.concat(versions2);
+  }, [query1, query2]);
+
+  const evalQuery = useRootObjectVersions(
+    entity,
+    project,
+    {
+      baseObjectClasses: ['Evaluation'],
+    },
+    // This 100 is very limited
+    100,
+    false,
+    {skip: allVersions.length === 0}
+  );
+
+  console.log(allVersions);
+
   return useMemo(() => {
-    return [
-      {version: 'sug657ioy8j1', versionIndex: 0},
-      {version: 'snkjubyhvasd', versionIndex: 1},
-      {version: 'sadsgf3f451d', versionIndex: 2},
-    ];
-  }, []);
+    const scorers = (evalQuery.result ?? [])
+      .map(e => {
+        const ref = parseRefMaybe(e.val.dataset);
+        if (!ref) {
+          return [];
+        }
+        if (
+          ref.artifactName !== datasetName ||
+          ref.artifactVersion !== datasetVersion
+        ) {
+          return [];
+        }
+        return (e.val.scorers ?? []).map((scorer: string) => {
+          const sRef = parseRefMaybe(scorer);
+          if (!sRef) {
+            return null;
+          }
+          const match = allVersions.find(
+            v => v.version === sRef.artifactVersion
+          );
+          return match;
+        });
+      })
+      .flat()
+      .filter(version => !!version);
+    return scorers;
+  }, [allVersions, datasetName, datasetVersion, evalQuery.result]);
 };
+
 export const useMetricPathsForDatasetAndScorer = (
   datasetName: string,
   datasetVersion: string,
@@ -157,12 +245,14 @@ export const useMetricPathsForDatasetAndScorer = (
     return ['accuracy', 'f1.macro', 'precision.micro', 'recall.micro.data'];
   }, []);
 };
+
 export const useModelNames = (): string[] => {
   // TODO: Implement this
   return useMemo(() => {
     return ['model-1', 'model-2', 'model-3'];
   }, []);
 };
+
 export const useModelVersionsForModelName = (
   modelName: string
 ): Array<{version: string; versionIndex: number}> => {

From 938dd9a06e51329d2cb204e9f709a0f7721f5d0d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 22:04:59 -0700
Subject: [PATCH 030/169] Human: Finished basic config

---
 .../LeaderboardPage/LeaderboardPageConfig.tsx |  3 +-
 .../pages/LeaderboardPage/MetricConfig.tsx    | 16 ++--
 .../pages/LeaderboardPage/ScoreConfig.tsx     |  2 +
 .../LeaderboardPage/leaderboardConfigQuery.ts | 73 ++++++++++++++++++-
 4 files changed, 84 insertions(+), 10 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 4746ab99a20..ae1e355d809 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -184,8 +184,7 @@ const TempAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
   return (
     <Alert severity="info" onClose={onClose}>
       <Typography variant="body1">
-        This configuration edtior is purely for internal exploration and not the
-        intended final product. It was 100% generated by Cursor.
+        Configuration edtior purely for internal exploration, not for production use.
       </Typography>
     </Alert>
   );
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
index d4c3ecb020a..53dceb058b8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
@@ -17,6 +17,8 @@ import {Direction, LeaderboardConfigType} from './LeaderboardConfigType';
 
 export const MetricConfig: React.FC<{
   metric: LeaderboardConfigType['config']['columns'][0]['scores'][0]['metrics'][0];
+  entity: string;
+  project: string;
   datasetName: string;
   datasetVersion: string;
   scorerName: string;
@@ -27,6 +29,8 @@ export const MetricConfig: React.FC<{
   onRemove: () => void;
 }> = ({
   metric,
+  entity,
+  project,
   datasetName,
   datasetVersion,
   scorerName,
@@ -35,6 +39,8 @@ export const MetricConfig: React.FC<{
   onRemove,
 }) => {
   const metricPaths = useMetricPathsForDatasetAndScorer(
+    entity,
+    project,
     datasetName,
     datasetVersion,
     scorerName,
@@ -53,13 +59,13 @@ export const MetricConfig: React.FC<{
           <DeleteIcon />
         </IconButton>
       </Stack>
-      <TextField
+      {/* <TextField
         fullWidth
         label="Display Name"
         value={metric.displayName}
         onChange={e => onUpdate({...metric, displayName: e.target.value})}
         sx={{mb: 2}}
-      />
+      /> */}
       <FormControl fullWidth sx={{mb: 2}}>
         <InputLabel>Metric Path</InputLabel>
         <Select
@@ -75,7 +81,7 @@ export const MetricConfig: React.FC<{
           ))}
         </Select>
       </FormControl>
-      <TextField
+      {/* <TextField
         fullWidth
         label="Min Trials"
         type="number"
@@ -100,7 +106,7 @@ export const MetricConfig: React.FC<{
           })
         }
         sx={{mb: 2}}
-      />
+      /> */}
       <FormControl fullWidth sx={{mb: 2}}>
         <InputLabel>Sort Direction</InputLabel>
         <Select
@@ -125,7 +131,7 @@ export const MetricConfig: React.FC<{
           fullWidth
           label="Sort Precedence"
           type="number"
-          value={metric.sort.precedence}
+          value={metric.sort.precedence === 0 ? '' : metric.sort.precedence}
           onChange={e =>
             onUpdate({
               ...metric,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
index 16a556fd401..d96e27c1c2f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
@@ -135,6 +135,8 @@ export const ScoreConfig: React.FC<{
       {score.metrics.map((metric, index) => (
         <MetricConfig
           key={index}
+          entity={entity}
+          project={project}
           metric={metric}
           datasetName={datasetName}
           datasetVersion={datasetVersion}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
index 2d973ba7c2d..d82169dfe5f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
@@ -1,7 +1,10 @@
 import {useMemo} from 'react';
 
+import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
 import {parseRefMaybe} from '../../../Browse2/SmallRef';
 import {useWFHooks} from '../wfReactInterface/context';
+import {objectVersionKeyToRefUri} from '../wfReactInterface/utilities';
+import {CallSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
 
 export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
@@ -235,15 +238,79 @@ export const useScorerVersionsForDatasetAndScorer = (
 };
 
 export const useMetricPathsForDatasetAndScorer = (
+  entity: string,
+  project: string,
   datasetName: string,
   datasetVersion: string,
   scorerName: string,
   scorerVersion: string
 ): string[] => {
-  // TODO: Implement this
+  const {useRootObjectVersions, useCalls} = useWFHooks();
+  const evalQuery = useRootObjectVersions(
+    entity,
+    project,
+    {
+      baseObjectClasses: ['Evaluation'],
+    },
+    // This 100 is very limited
+    100
+  );
+
+  const evals = useMemo(() => {
+    // Find the matching evals:
+    console.log(evalQuery.result, scorerName, scorerVersion);
+    return (evalQuery.result ?? []).filter(e => {
+      const match = (e.val.scorers ?? []).find((s: string) => {
+        const sRef = parseRefMaybe(s);
+        if (!sRef) {
+          return false;
+        }
+        console.log(sRef);
+        return (
+          sRef.artifactName === scorerName &&
+          sRef.artifactVersion === scorerVersion
+        );
+      });
+      return !!match;
+    });
+  }, [evalQuery.result, scorerName, scorerVersion]);
+
+  const evalCalls = useCalls(
+    entity,
+    project,
+    {
+      inputObjectVersionRefs: evals.map(objectVersionKeyToRefUri),
+    },
+    10,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    {
+      skip: evals.length === 0,
+    }
+  );
+
   return useMemo(() => {
-    return ['accuracy', 'f1.macro', 'precision.micro', 'recall.micro.data'];
-  }, []);
+    const foundCall = (evalCalls.result ?? []).find(
+      (call: CallSchema) =>
+        call.traceCall?.exception == null && call.traceCall?.ended_at != null
+    );
+    console.log(foundCall, evalCalls);
+    const output = foundCall?.traceCall?.output;
+    if (!output) {
+      return [];
+    }
+    if (typeof output !== 'object') {
+      return [];
+    }
+    return Object.keys(
+      flattenObjectPreservingWeaveTypes(
+        (output as {[key: string]: any})[scorerName] ?? {}
+      )
+    );
+  }, [evalCalls, scorerName]);
 };
 
 export const useModelNames = (): string[] => {

From cb86ea6933c4e3c9a70faf39beca9f417815522b Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 22:31:40 -0700
Subject: [PATCH 031/169] Human: Getting things wired uo

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx |   2 +
 .../LeaderboardPage/LeaderboardPageConfig.tsx |   3 +-
 .../Browse3/pages/LeaderboardPage/hooks.ts    | 100 +++++++--------
 .../LeaderboardPage/leaderboardConfigQuery.ts | 121 +++++++++++++++++-
 4 files changed, 172 insertions(+), 54 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 393e2fe525b..8c389bf2bd7 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -146,6 +146,8 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
             } else {
               inner = `${inner.toFixed(2)}`;
             }
+          } else {
+            inner = JSON.stringify(params.value);
           }
           // const value =
           // <CellValue value={params.value} />
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index ae1e355d809..0b1a9b5248c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -184,7 +184,8 @@ const TempAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
   return (
     <Alert severity="info" onClose={onClose}>
       <Typography variant="body1">
-        Configuration edtior purely for internal exploration, not for production use.
+        Configuration edtior purely for internal exploration, not for production
+        use.
       </Typography>
     </Alert>
   );
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index 13ec658a8ae..504bb8a8c8c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -8,6 +8,7 @@ import {
   objectVersionKeyToRefUri,
   opVersionKeyToRefUri,
 } from '../wfReactInterface/utilities';
+import { useEvalCallsForConfig } from './leaderboardConfigQuery';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
 
 export type LeaderboardData = {
@@ -30,54 +31,55 @@ export const useLeaderboardData = (
   project: string,
   config: LeaderboardConfigType
 ): {loading: boolean; data: LeaderboardData} => {
-  console.log('Fetching leaderboard data', config);
-  const {useRootObjectVersions, useCalls} = useWFHooks();
+  // console.log('Fetching leaderboard data', config);
+  // const {useRootObjectVersions, useCalls} = useWFHooks();
 
-  // Get the last 100 (latest) evaluation versions
-  const evaluationVersions = useRootObjectVersions(
-    entity,
-    project,
-    {
-      baseObjectClasses: ['Evaluation'],
-      // latestOnly: true,
-    },
-    100,
-    true
-  );
+  // // Get the last 100 (latest) evaluation versions
+  // const evaluationVersions = useRootObjectVersions(
+  //   entity,
+  //   project,
+  //   {
+  //     baseObjectClasses: ['Evaluation'],
+  //     // latestOnly: true,
+  //   },
+  //   100,
+  //   true
+  // );
 
-  // Unfortunately, the eval framework does not build models automatically!!
-  // const modelVersions = useRootObjectVersions(entity, project, {
-  //     baseObjectClasses: ['Model'],
-  //     latestOnly: true,
-  // }, 100, true)
+  // // Unfortunately, the eval framework does not build models automatically!!
+  // // const modelVersions = useRootObjectVersions(entity, project, {
+  // //     baseObjectClasses: ['Model'],
+  // //     latestOnly: true,
+  // // }, 100, true)
 
-  // Get the runs for these evaluation versions.
-  const evaluationVersionsResult = evaluationVersions?.result;
-  const evaluationRuns = useCalls(
-    entity,
-    project,
-    {
-      opVersionRefs: [
-        opVersionKeyToRefUri({
-          entity,
-          project,
-          opId: EVALUATE_OP_NAME_POST_PYDANTIC,
-          versionHash: '*',
-        }),
-      ],
-      traceRootsOnly: true,
-      inputObjectVersionRefs: (evaluationVersionsResult ?? []).map(version =>
-        objectVersionKeyToRefUri(version)
-      ),
-    },
-    100,
-    undefined,
-    undefined,
-    undefined,
-    undefined,
-    undefined,
-    {skip: !evaluationVersionsResult}
-  );
+  // // Get the runs for these evaluation versions.
+  // const evaluationVersionsResult = evaluationVersions?.result;
+  // const evaluationRuns = useCalls(
+  //   entity,
+  //   project,
+  //   {
+  //     opVersionRefs: [
+  //       opVersionKeyToRefUri({
+  //         entity,
+  //         project,
+  //         opId: EVALUATE_OP_NAME_POST_PYDANTIC,
+  //         versionHash: '*',
+  //       }),
+  //     ],
+  //     traceRootsOnly: true,
+  //     inputObjectVersionRefs: (evaluationVersionsResult ?? []).map(version =>
+  //       objectVersionKeyToRefUri(version)
+  //     ),
+  //   },
+  //   100,
+  //   undefined,
+  //   undefined,
+  //   undefined,
+  //   undefined,
+  //   undefined,
+  //   {skip: !evaluationVersionsResult}
+  // );
+  const evaluationRuns = useEvalCallsForConfig(entity, project, config);
 
   // Build the dataset
 
@@ -87,7 +89,7 @@ export const useLeaderboardData = (
       models: [],
       scores: {},
     };
-    if (evaluationVersions.loading || evaluationRuns.loading) {
+    if (evaluationRuns.loading) {
       return {
         loading: true,
         data: finalData,
@@ -147,11 +149,7 @@ export const useLeaderboardData = (
       loading: false,
       data: finalData,
     };
-  }, [
-    evaluationRuns.loading,
-    evaluationRuns.result,
-    evaluationVersions.loading,
-  ]);
+  }, [evaluationRuns.loading, evaluationRuns.result]);
 
   return results;
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
index d82169dfe5f..f62c0559c5a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
@@ -3,9 +3,13 @@ import {useMemo} from 'react';
 import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
 import {parseRefMaybe} from '../../../Browse2/SmallRef';
 import {useWFHooks} from '../wfReactInterface/context';
-import {objectVersionKeyToRefUri} from '../wfReactInterface/utilities';
-import {CallSchema} from '../wfReactInterface/wfDataModelHooksInterface';
+import {objectVersionKeyToRefUri, opVersionKeyToRefUri} from '../wfReactInterface/utilities';
+import {
+  CallSchema,
+  ObjectVersionSchema,
+} from '../wfReactInterface/wfDataModelHooksInterface';
 import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
+import { EVALUATE_OP_NAME_POST_PYDANTIC } from '../common/heuristics';
 
 export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
   // TODO: Implement this
@@ -332,3 +336,116 @@ export const useModelVersionsForModelName = (
     ];
   }, []);
 };
+
+export const useEvalObjsForConfig = (
+  entity: string,
+  project: string,
+  config: LeaderboardConfigType
+) => {
+  const {useRootObjectVersions} = useWFHooks();
+  // # TODO: handle latests
+  const evalQuery = useRootObjectVersions(
+    entity,
+    project,
+    {
+      baseObjectClasses: ['Evaluation'],
+    },
+    // This 100 is very limited
+    100
+  );
+  return useMemo(() => {
+    if (!evalQuery.result) {
+      return [];
+    }
+    const finalEvals: ObjectVersionSchema[] = [];
+    evalQuery.result.forEach(evaluation => {
+      const datasetRef = parseRefMaybe(evaluation.val.dataset);
+      if (!datasetRef) {
+        return;
+      }
+      if (config.config.columns.length === 0) {
+        finalEvals.push(evaluation);
+        return;
+      }
+      const datasetName = datasetRef.artifactName;
+      const datasetVersion = datasetRef.artifactVersion;
+      // Determine this dataset matches any of the config's datasets
+      let matched = false;
+      for (const column of config.config.columns) {
+        if (
+          datasetName === column.dataset.name &&
+          (column.dataset.version === datasetVersion ||
+            (column.dataset.version !== 'latest' &&
+              column.dataset.version !== 'all'))
+        ) {
+          if (column.scores.length === 0) {
+            finalEvals.push(evaluation);
+            break;
+          }
+          for (const scorer of evaluation.val.scorers ?? []) {
+            const sRef = parseRefMaybe(scorer);
+            if (!sRef) {
+              continue;
+            }
+            const scorerName = sRef.artifactName;
+            const scorerVersion = sRef.artifactVersion;
+            for (const score of column.scores) {
+              if (
+                scorerName === score.scorer.name &&
+                (scorerVersion === score.scorer.version ||
+                  (score.scorer.version !== 'latest' &&
+                    score.scorer.version !== 'all'))
+              ) {
+                finalEvals.push(evaluation);
+                matched = true;
+                break;
+              }
+            }
+          }
+          if (matched) {
+            break;
+          }
+        }
+      }
+    });
+
+    return finalEvals;
+  }, [config.config.columns, evalQuery.result]);
+};
+
+export const useEvalCallsForConfig = (
+  entity: string,
+  project: string,
+  config: LeaderboardConfigType
+) => {
+  const {useCalls} = useWFHooks();
+  // Step 1: Get the qualifying evaluation objects
+  const evals = useEvalObjsForConfig(entity, project, config);
+  // Step 2: Get the calls for each evaluation object
+  const calls = useCalls(
+    entity,
+    project,
+    {
+      opVersionRefs: [
+        opVersionKeyToRefUri({
+          entity,
+          project,
+          opId: EVALUATE_OP_NAME_POST_PYDANTIC,
+          versionHash: '*',
+        }),
+      ],
+      traceRootsOnly: true,
+      inputObjectVersionRefs: evals.map(objectVersionKeyToRefUri),
+    },
+    1000,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    {
+      skip: evals.length === 0,
+    }
+  );
+  return calls;
+};

From 5b0d9d420533c555c4593af611bb2083f5e77269 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Tue, 8 Oct 2024 23:54:28 -0700
Subject: [PATCH 032/169] Human: Ok, really forked hard

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx | 38 ++++++--
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  5 +-
 .../Browse3/pages/LeaderboardPage/hooks.ts    | 90 ++++++++++++++-----
 .../LeaderboardPage/leaderboardConfigQuery.ts | 34 ++++---
 4 files changed, 124 insertions(+), 43 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 8c389bf2bd7..1eccaa39d9d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -13,10 +13,13 @@ import React, {useCallback, useMemo, useState} from 'react';
 import {parseRefMaybe, SmallRef} from '../../../Browse2/SmallRef';
 import {StyledDataGrid} from '../../StyledDataGrid';
 import {PaginationButtons} from '../CallsPage/CallsTableButtons';
+import {ObjectVersionLink} from '../common/Links';
 import {buildTree} from '../common/tabularListViews/buildTree';
 import {LeaderboardData} from './hooks';
 
 interface LeaderboardGridProps {
+  entity: string;
+  project: string;
   data: LeaderboardData;
   loading: boolean;
   onCellClick: (modelName: string, metricName: string, score: number) => void;
@@ -28,12 +31,14 @@ type RowData = {
 } & LeaderboardData['scores'][string];
 
 export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
+  entity,
+  project,
   data,
   loading,
   onCellClick,
 }) => {
   const [paginationModel, setPaginationModel] = useState<GridPaginationModel>({
-    pageSize: 25,
+    pageSize: 50,
     page: 0,
   });
 
@@ -85,7 +90,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
       {
         field: 'model',
         headerName: 'Model',
-        minWidth: 200,
+        minWidth: 100,
         flex: 1,
         renderCell: (params: GridRenderCellParams) => {
           const modelRef = parseRefMaybe(params.value);
@@ -110,8 +115,8 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
       },
       ...orderedMetrics.map(metric => ({
         field: metric,
-        headerName: metric.split('.').pop(),
-        minWidth: 130,
+        headerName: data.metrics[metric].metricPath.split('.').pop(),
+        minWidth: 100,
         flex: 1,
         valueGetter: (value: RowData) => {
           return value?.value;
@@ -175,7 +180,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         },
       })),
     ],
-    [getColorForScore, onCellClick, orderedMetrics]
+    [data.metrics, getColorForScore, onCellClick, orderedMetrics]
   );
 
   const tree = buildTree([...Object.keys(data.metrics)]);
@@ -185,7 +190,11 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   groupingModel = walkGroupingModel(groupingModel, node => {
     if ('groupId' in node) {
       if (node.children.length === 1) {
-        if ('groupId' in node.children[0]) {
+        if (
+          'groupId' in node.children[0] &&
+          !node.headerName?.includes(':') &&
+          !node.children[0].headerName?.includes(':')
+        ) {
           const currNode = node;
           node = node.children[0];
           node.headerName = currNode.headerName + '.' + node.headerName;
@@ -197,6 +206,21 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     }
     return node;
   }) as GridColumnGroup[];
+  groupingModel = walkGroupingModel(groupingModel, node => {
+    if ('groupId' in node) {
+      node.renderHeaderGroup = params => {
+        const ref = parseRefMaybe(
+          `weave:///${entity}/${project}/object/${node.headerName}` ?? ''
+        );
+        // console.log(node.headerName, ref);
+        if (ref) {
+          return <SmallRef objRef={ref} />;
+        }
+        return <div>{node.headerName}</div>;
+      };
+    }
+    return node;
+  }) as GridColumnGroup[];
 
   // const groupingModel: GridColumnGroupingModel = useMemo(
   //   () => {
@@ -228,7 +252,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         pagination
         paginationModel={paginationModel}
         onPaginationModelChange={setPaginationModel}
-        pageSizeOptions={[25]}
+        pageSizeOptions={[50]}
         disableRowSelectionOnClick
         hideFooterSelectedRowCount
         disableMultipleColumnsSorting={false}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index c0be97f780e..774d7466055 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -80,7 +80,8 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     metricName: string,
     score: number
   ) => {
-    const sourceCallId = data.scores?.[modelName]?.[metricName]?.sourceCallId;
+    const sourceCallId =
+      data.scores?.[modelName]?.[metricName]?.sourceEvalCallId;
     if (sourceCallId) {
       let to: string;
       if (USE_COMPARE_EVALUATIONS_PAGE) {
@@ -124,6 +125,8 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
         overflow="hidden"
         minHeight="65%">
         <LeaderboardGrid
+          entity={entity}
+          project={project}
           loading={loading}
           data={data}
           onCellClick={handleCellClick}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index 504bb8a8c8c..d4c52880dec 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -8,20 +8,25 @@ import {
   objectVersionKeyToRefUri,
   opVersionKeyToRefUri,
 } from '../wfReactInterface/utilities';
-import { useEvalCallsForConfig } from './leaderboardConfigQuery';
+import {useEvalCallsForConfig} from './leaderboardConfigQuery';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
+import { ObjectRef } from '@wandb/weave/react';
 
 export type LeaderboardData = {
   metrics: {
     [metricId: string]: {
-      evaluationName: string;
+      // evaluationName: string;
+      datasetName: string;
+      datasetVersion: string;
+      scorerName: string;
+      scorerVersion: string;
       metricPath: string;
     };
   };
   models: string[];
   scores: {
     [modelId: string]: {
-      [metricId: string]: {value: number; sourceCallId: string};
+      [metricId: string]: {value: number; sourceEvalCallId: string};
     };
   };
 };
@@ -79,7 +84,8 @@ export const useLeaderboardData = (
   //   undefined,
   //   {skip: !evaluationVersionsResult}
   // );
-  const evaluationRuns = useEvalCallsForConfig(entity, project, config);
+  const {calls: evaluationRuns, evals: evaluationVersions} =
+    useEvalCallsForConfig(entity, project, config);
 
   // Build the dataset
 
@@ -113,43 +119,79 @@ export const useLeaderboardData = (
       if (!evaluationVersion) {
         return;
       }
-      const evalName = parseRefMaybe(evaluationVersion)?.artifactName;
-      if (!evalName) {
+      const evalVersion = parseRefMaybe(evaluationVersion)?.artifactVersion;
+      if (!evalVersion) {
+        return;
+      }
+      const evalObject = evaluationVersions.find(
+        e => e.versionHash === evalVersion
+      );
+      if (!evalObject) {
         return;
       }
       const outputSummary = r.traceCall?.output;
       if (!outputSummary) {
         return;
       }
+
+      const datasetRef = parseRefMaybe(evalObject.val.dataset);
+      if (!datasetRef) {
+        return;
+      }
       if (!finalData.models.includes(modelName)) {
         finalData.models.push(modelName);
-      }
-      const modelScores = flattenObjectPreservingWeaveTypes(
-        outputSummary ?? {}
-      );
-      Object.entries(modelScores).forEach(([metric, score]) => {
-        const metricName = `${evalName}.${metric}`;
-        if (!finalData.metrics[metricName]) {
-          finalData.metrics[metricName] = {
-            evaluationName: evalName,
-            metricPath: metric,
-          };
-        }
         if (!finalData.scores[modelName]) {
           finalData.scores[modelName] = {};
         }
-        finalData.scores[modelName][metricName] = {
-          value: score,
-          sourceCallId: r.callId,
-        };
-      });
+      }
+
+      const datasetName = datasetRef.artifactName;
+      const datasetVersion = datasetRef.artifactVersion;
+      const scorerNameMap = Object.fromEntries(
+        (evalObject.val.scorers ?? [])
+          .map(parseRefMaybe)
+          .filter(Boolean)
+          .map((s: ObjectRef) => [s.artifactName, s.artifactVersion])
+      );
+
+      Object.entries(outputSummary ?? {}).forEach(
+        ([scorerName, scorerMetricsVal]) => {
+          const scorerVersion = scorerNameMap[scorerName];
+          if (!scorerVersion) {
+            return;
+          }
+          if (
+            scorerMetricsVal == null ||
+            typeof scorerMetricsVal !== 'object'
+          ) {
+            return;
+          }
+          const flattened = flattenObjectPreservingWeaveTypes(scorerMetricsVal);
+          Object.entries(flattened).forEach(([metricPath, metricValue]) => {
+            const metricId = `${datasetName}:${datasetVersion}.${scorerName}:${scorerVersion}.${metricPath}`;
+            if (!finalData.metrics[metricId]) {
+              finalData.metrics[metricId] = {
+                datasetName,
+                datasetVersion,
+                scorerName,
+                scorerVersion,
+                metricPath,
+              };
+            }
+            finalData.scores[modelName][metricId] = {
+              value: metricValue,
+              sourceEvalCallId: r.callId,
+            };
+          });
+        }
+      );
     });
 
     return {
       loading: false,
       data: finalData,
     };
-  }, [evaluationRuns.loading, evaluationRuns.result]);
+  }, [evaluationRuns.loading, evaluationRuns.result, evaluationVersions]);
 
   return results;
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
index f62c0559c5a..7c50c6a62a0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
@@ -2,18 +2,21 @@ import {useMemo} from 'react';
 
 import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
 import {parseRefMaybe} from '../../../Browse2/SmallRef';
+import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
 import {useWFHooks} from '../wfReactInterface/context';
-import {objectVersionKeyToRefUri, opVersionKeyToRefUri} from '../wfReactInterface/utilities';
+import {
+  objectVersionKeyToRefUri,
+  opVersionKeyToRefUri,
+} from '../wfReactInterface/utilities';
 import {
   CallSchema,
   ObjectVersionSchema,
 } from '../wfReactInterface/wfDataModelHooksInterface';
 import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
-import { EVALUATE_OP_NAME_POST_PYDANTIC } from '../common/heuristics';
 
 export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
   // TODO: Implement this
-  console.log('Fetching current leaderboard config');
+  // console.log('Fetching current leaderboard config');
   return useMemo(() => {
     return {
       version: 1,
@@ -27,7 +30,7 @@ export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
 };
 export const persistLeaderboardConfig = (config: LeaderboardConfigType) => {
   // TODO: Implement this
-  console.log('Persisting leaderboard config:', config);
+  // console.log('Persisting leaderboard config:', config);
 };
 
 export const useDatasetNames = (entity: string, project: string): string[] => {
@@ -145,6 +148,7 @@ export const useScorerNamesForDataset = (
         }
         return ref.artifactVersion === datasetVersion;
       }) ?? [];
+
     const res = evalResults
       .map(obj => obj.val.scorers ?? [])
       .flat()
@@ -209,7 +213,7 @@ export const useScorerVersionsForDatasetAndScorer = (
     {skip: allVersions.length === 0}
   );
 
-  console.log(allVersions);
+  // console.log(allVersions);
 
   return useMemo(() => {
     const scorers = (evalQuery.result ?? [])
@@ -262,14 +266,14 @@ export const useMetricPathsForDatasetAndScorer = (
 
   const evals = useMemo(() => {
     // Find the matching evals:
-    console.log(evalQuery.result, scorerName, scorerVersion);
+    // console.log(evalQuery.result, scorerName, scorerVersion);
     return (evalQuery.result ?? []).filter(e => {
       const match = (e.val.scorers ?? []).find((s: string) => {
         const sRef = parseRefMaybe(s);
         if (!sRef) {
           return false;
         }
-        console.log(sRef);
+        // console.log(sRef);
         return (
           sRef.artifactName === scorerName &&
           sRef.artifactVersion === scorerVersion
@@ -299,9 +303,11 @@ export const useMetricPathsForDatasetAndScorer = (
   return useMemo(() => {
     const foundCall = (evalCalls.result ?? []).find(
       (call: CallSchema) =>
-        call.traceCall?.exception == null && call.traceCall?.ended_at != null
+        call.traceCall?.exception == null &&
+        call.traceCall?.ended_at != null &&
+        (call?.traceCall?.output as any)?.[scorerName] != null
     );
-    console.log(foundCall, evalCalls);
+    // console.log(foundCall, evalCalls);
     const output = foundCall?.traceCall?.output;
     if (!output) {
       return [];
@@ -309,11 +315,13 @@ export const useMetricPathsForDatasetAndScorer = (
     if (typeof output !== 'object') {
       return [];
     }
-    return Object.keys(
+    const res = Object.keys(
       flattenObjectPreservingWeaveTypes(
         (output as {[key: string]: any})[scorerName] ?? {}
       )
     );
+    console.log(foundCall, scorerName, res);
+    return res;
   }, [evalCalls, scorerName]);
 };
 
@@ -368,6 +376,10 @@ export const useEvalObjsForConfig = (
         return;
       }
       const datasetName = datasetRef.artifactName;
+      // if (datasetName === '') {
+      //   finalEvals.push(evaluation);
+      //   return;
+      // }
       const datasetVersion = datasetRef.artifactVersion;
       // Determine this dataset matches any of the config's datasets
       let matched = false;
@@ -447,5 +459,5 @@ export const useEvalCallsForConfig = (
       skip: evals.length === 0,
     }
   );
-  return calls;
+  return {calls, evals};
 };

From 30483c6f9814a651cd8000793f31d7ede96e73c2 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 9 Oct 2024 00:04:20 -0700
Subject: [PATCH 033/169] Human: Ok, really forked hard

---
 .../LeaderboardPage/LeaderboardPageConfig.tsx |  2 +-
 .../Browse3/pages/LeaderboardPage/hooks.ts    | 39 +++++++++++++++----
 .../LeaderboardPage/leaderboardConfigQuery.ts | 17 ++++----
 3 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 0b1a9b5248c..66dcacc2197 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -102,7 +102,7 @@ export const LeaderboardConfig: React.FC<{
   return (
     <Box
       sx={{
-        width: '100%',
+        width: '50%',
         height: '100%',
         display: 'flex',
         flexDirection: 'column',
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index d4c52880dec..d2354395c95 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -1,3 +1,4 @@
+import {ObjectRef} from '@wandb/weave/react';
 import {useMemo} from 'react';
 
 import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
@@ -10,7 +11,6 @@ import {
 } from '../wfReactInterface/utilities';
 import {useEvalCallsForConfig} from './leaderboardConfigQuery';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
-import { ObjectRef } from '@wandb/weave/react';
 
 export type LeaderboardData = {
   metrics: {
@@ -138,12 +138,6 @@ export const useLeaderboardData = (
       if (!datasetRef) {
         return;
       }
-      if (!finalData.models.includes(modelName)) {
-        finalData.models.push(modelName);
-        if (!finalData.scores[modelName]) {
-          finalData.scores[modelName] = {};
-        }
-      }
 
       const datasetName = datasetRef.artifactName;
       const datasetVersion = datasetRef.artifactVersion;
@@ -152,8 +146,37 @@ export const useLeaderboardData = (
           .map(parseRefMaybe)
           .filter(Boolean)
           .map((s: ObjectRef) => [s.artifactName, s.artifactVersion])
+          .filter(([name, version]: [string, string]) => {
+            return (
+              config.config.columns.length === 0 ||
+              config.config.columns.some(dc => {
+                return (
+                  (dc.dataset.name === '' || dc.dataset.name === datasetName) &&
+                  (dc.dataset.version === 'all' ||
+                    dc.dataset.version === 'latest' ||
+                    dc.dataset.version === datasetVersion) &&
+                  (dc.scores.length === 0 ||
+                    dc.scores.some(s => {
+                      return (
+                        (s.scorer.name === '' || s.scorer.name === name) &&
+                        (s.scorer.version === 'all' ||
+                          s.scorer.version === 'latest' ||
+                          s.scorer.version === version)
+                      );
+                    }))
+                );
+              })
+            );
+          })
       );
 
+      if (!finalData.models.includes(modelName)) {
+        finalData.models.push(modelName);
+        if (!finalData.scores[modelName]) {
+          finalData.scores[modelName] = {};
+        }
+      }
+
       Object.entries(outputSummary ?? {}).forEach(
         ([scorerName, scorerMetricsVal]) => {
           const scorerVersion = scorerNameMap[scorerName];
@@ -191,7 +214,7 @@ export const useLeaderboardData = (
       loading: false,
       data: finalData,
     };
-  }, [evaluationRuns.loading, evaluationRuns.result, evaluationVersions]);
+  }, [config.config.columns, evaluationRuns.loading, evaluationRuns.result, evaluationVersions]);
 
   return results;
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
index 7c50c6a62a0..905aca976b2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
@@ -376,19 +376,15 @@ export const useEvalObjsForConfig = (
         return;
       }
       const datasetName = datasetRef.artifactName;
-      // if (datasetName === '') {
-      //   finalEvals.push(evaluation);
-      //   return;
-      // }
       const datasetVersion = datasetRef.artifactVersion;
       // Determine this dataset matches any of the config's datasets
       let matched = false;
       for (const column of config.config.columns) {
         if (
-          datasetName === column.dataset.name &&
+          (datasetName === column.dataset.name || column.dataset.name === '') &&
           (column.dataset.version === datasetVersion ||
-            (column.dataset.version !== 'latest' &&
-              column.dataset.version !== 'all'))
+            column.dataset.version === 'latest' ||
+            column.dataset.version === 'all')
         ) {
           if (column.scores.length === 0) {
             finalEvals.push(evaluation);
@@ -403,10 +399,11 @@ export const useEvalObjsForConfig = (
             const scorerVersion = sRef.artifactVersion;
             for (const score of column.scores) {
               if (
-                scorerName === score.scorer.name &&
+                (scorerName === score.scorer.name ||
+                  score.scorer.name === '') &&
                 (scorerVersion === score.scorer.version ||
-                  (score.scorer.version !== 'latest' &&
-                    score.scorer.version !== 'all'))
+                  score.scorer.version === 'latest' ||
+                  score.scorer.version === 'all')
               ) {
                 finalEvals.push(evaluation);
                 matched = true;

From 6c368d57f3d59dacbfca6df5792aec0cfa8888c8 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 9 Oct 2024 22:35:10 -0700
Subject: [PATCH 034/169] addressed comments

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  42 +-
 .../Browse3/pages/LeaderboardPage/hooks.ts    |  66 ++-
 .../leaderboardServerInterface.ts             | 442 ++++++++++++++++++
 .../traceServerClientTypes.ts                 |   8 +-
 4 files changed, 518 insertions(+), 40 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 774d7466055..d2463e8ba0f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,19 +1,19 @@
-import {Alert, Box, Typography} from '@mui/material';
-import {useTraceUpdate} from '@wandb/weave/common/util/hooks';
-import {Button} from '@wandb/weave/components/Button';
-import React, {useCallback, useState} from 'react';
-import {useHistory} from 'react-router-dom';
-
-import {useWeaveflowRouteContext} from '../../context';
-import {EditableMarkdown} from './EditableMarkdown';
-import {useLeaderboardData} from './hooks';
+import { Alert, Box, Typography } from '@mui/material';
+import { useTraceUpdate } from '@wandb/weave/common/util/hooks';
+import { Button } from '@wandb/weave/components/Button';
+import React, { useCallback, useState } from 'react';
+import { useHistory } from 'react-router-dom';
+
+import { useWeaveflowRouteContext } from '../../context';
+import { EditableMarkdown } from './EditableMarkdown';
+import { useLeaderboardData, useLeaderboardData2 } from './hooks';
 import {
   persistLeaderboardConfig,
   useCurrentLeaderboardConfig,
 } from './leaderboardConfigQuery';
-import {LeaderboardConfigType} from './LeaderboardConfigType';
-import {LeaderboardGrid} from './LeaderboardGrid';
-import {LeaderboardConfig} from './LeaderboardPageConfig';
+import { LeaderboardConfigType } from './LeaderboardConfigType';
+import { LeaderboardGrid } from './LeaderboardGrid';
+import { LeaderboardConfig } from './LeaderboardPageConfig';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
@@ -43,13 +43,13 @@ const usePersistedLeaderboardConfig = () => {
     setConfigLocal(initialConfig);
   }, [initialConfig]);
 
-  return {config, setConfigLocal, persistConfig, cancelChanges};
+  return { config, setConfigLocal, persistConfig, cancelChanges };
 };
 
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
-  const {entity, project} = props;
+  const { entity, project } = props;
 
-  const {peekingRouter} = useWeaveflowRouteContext();
+  const { peekingRouter } = useWeaveflowRouteContext();
   const history = useHistory();
 
   const [showConfig, setShowConfig] = useState(false);
@@ -65,15 +65,15 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     (newDescription: string) => {
       setConfigLocal(newConfig => ({
         ...newConfig,
-        config: {...newConfig.config, description: newDescription},
+        config: { ...newConfig.config, description: newDescription },
       }));
       persistConfig();
     },
     [setConfigLocal, persistConfig]
   );
-
-  const {loading, data} = useLeaderboardData(entity, project, currentConfig);
-  useTraceUpdate('a', {entity, project, currentConfig});
+  useLeaderboardData2(entity, project)
+  const { loading, data } = useLeaderboardData(entity, project, currentConfig);
+  useTraceUpdate('a', { entity, project, currentConfig });
 
   const handleCellClick = (
     modelName: string,
@@ -155,7 +155,7 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
 export const ToggleLeaderboardConfig: React.FC<{
   isOpen: boolean;
   onClick: () => void;
-}> = ({isOpen, onClick}) => {
+}> = ({ isOpen, onClick }) => {
   return (
     <Box
       sx={{
@@ -174,7 +174,7 @@ export const ToggleLeaderboardConfig: React.FC<{
   );
 };
 
-const UnlistedAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
+const UnlistedAlert: React.FC<{ onClose: () => void }> = ({ onClose }) => {
   return (
     <Alert severity="info" onClose={onClose}>
       <Typography variant="body1">
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index d2354395c95..7c630745517 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -1,16 +1,12 @@
-import {ObjectRef} from '@wandb/weave/react';
-import {useMemo} from 'react';
-
-import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
-import {parseRefMaybe} from '../../../Browse2/SmallRef';
-import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
-import {useWFHooks} from '../wfReactInterface/context';
-import {
-  objectVersionKeyToRefUri,
-  opVersionKeyToRefUri,
-} from '../wfReactInterface/utilities';
-import {useEvalCallsForConfig} from './leaderboardConfigQuery';
-import {LeaderboardConfigType} from './LeaderboardConfigType';
+import { ObjectRef } from '@wandb/weave/react';
+import { useEffect, useMemo, useState, } from 'react';
+
+import { flattenObjectPreservingWeaveTypes } from '../../../Browse2/browse2Util';
+import { parseRefMaybe } from '../../../Browse2/SmallRef';
+import { useGetTraceServerClientContext } from '../wfReactInterface/traceServerClientContext';
+import { useEvalCallsForConfig } from './leaderboardConfigQuery';
+import { LeaderboardConfigType } from './LeaderboardConfigType';
+import { getLeaderboardData, LeaderboardData2 } from './leaderboardServerInterface';
 
 export type LeaderboardData = {
   metrics: {
@@ -26,7 +22,7 @@ export type LeaderboardData = {
   models: string[];
   scores: {
     [modelId: string]: {
-      [metricId: string]: {value: number; sourceEvalCallId: string};
+      [metricId: string]: { value: number; sourceEvalCallId: string };
     };
   };
 };
@@ -35,7 +31,7 @@ export const useLeaderboardData = (
   entity: string,
   project: string,
   config: LeaderboardConfigType
-): {loading: boolean; data: LeaderboardData} => {
+): { loading: boolean; data: LeaderboardData } => {
   // console.log('Fetching leaderboard data', config);
   // const {useRootObjectVersions, useCalls} = useWFHooks();
 
@@ -84,12 +80,12 @@ export const useLeaderboardData = (
   //   undefined,
   //   {skip: !evaluationVersionsResult}
   // );
-  const {calls: evaluationRuns, evals: evaluationVersions} =
+  const { calls: evaluationRuns, evals: evaluationVersions } =
     useEvalCallsForConfig(entity, project, config);
 
   // Build the dataset
 
-  const results: {loading: boolean; data: LeaderboardData} = useMemo(() => {
+  const results: { loading: boolean; data: LeaderboardData } = useMemo(() => {
     const finalData: LeaderboardData = {
       metrics: {},
       models: [],
@@ -218,3 +214,39 @@ export const useLeaderboardData = (
 
   return results;
 };
+
+
+export const useLeaderboardData2 = (
+  entity: string, project: string
+) => {
+  const getTraceServerClient = useGetTraceServerClientContext();
+  const [state, setState] = useState<{ loading: boolean; data: LeaderboardData2 }>({ loading: true, data: [] });
+  useEffect(() => {
+    let mounted = true;
+    getLeaderboardData(
+      getTraceServerClient(), entity, project, {
+      datasets: [{
+        name: 'SWEBenchVerified-shuffle808-50',
+        version: '*'
+      }],
+      models: [{
+        name: '*',
+        version: '*',
+        splitByVersion: false,
+      }, {
+        name: "reproProblem",
+        version: '*',
+        splitByVersion: true,
+      }]
+    }
+    ).then(data => {
+      if (mounted) {
+        setState({ loading: false, data });
+      }
+    });
+    return () => {
+      mounted = false;
+    };
+  }, [entity, project, getTraceServerClient]);
+  return state;
+}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
new file mode 100644
index 00000000000..723db9f2075
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -0,0 +1,442 @@
+
+// type EvaluationMatrix = {
+//     datasets: {
+//         datasetName: string;
+//         versions: {
+
+import { isWeaveObjectRef } from "@wandb/weave/react";
+
+import { flattenObjectPreservingWeaveTypes } from "../../../Browse2/browse2Util";
+import { parseRefMaybe } from "../../../Browse2/SmallRef";
+import { EVALUATE_OP_NAME_POST_PYDANTIC } from "../common/heuristics";
+import { TraceServerClient } from "../wfReactInterface/traceServerClient";
+import { TraceObjSchema } from "../wfReactInterface/traceServerClientTypes";
+import { convertISOToDate, projectIdFromParts } from "../wfReactInterface/tsDataModelHooks";
+import { opVersionKeyToRefUri } from "../wfReactInterface/utilities";
+import _ from "lodash";
+
+//         }
+
+//     }
+// };
+
+// const getProjectEvaluationMatrix = () => { }
+
+
+// type EvaluationResultRecord = {
+//     datasetName: string;
+//     datasetVersion: string;
+//     scorerName: string;
+//     scorerVersion: string;
+//     metricPath: string;
+//     metricValue: number | string | boolean | null;
+//     modelName: string;
+//     modelVersion: string;
+//     modelType: 'object' | 'op';
+//     trials: number;
+//     sourceEvaluationCallId: string;
+//     sourceEvaluationObjectRef: string;
+// }
+
+// type EvaluationDefinition = {
+//     sourceEvaluationObjectRef: string;
+//     datasetName: string;
+//     datasetVersion: string;
+//     scorers: Array<{
+//         scorerName: string;
+//         scorerVersion: string;
+//     }>
+// }
+
+// type ScorerDefinition = {
+//     scorerName: string;
+//     scorerVersion: string;
+//     type: 'object' | 'op'
+//     sampleResult: { [key: string]: any }
+// }
+
+// // 
+
+// type EvaluationObjectLeaderboardSpec = {
+//     evaluationObjectVersionRefUri: string;
+// }
+
+// ----
+
+type LeaderboardSpec = {
+    columnGroups: LeaderboardDatasetColumnGroupSpec;
+    modelSpec?: ExplicitModelSpec
+}
+
+type ExplicitModelSpec = {
+    modelType: 'explicit';
+    models: Array<{
+        modelName: string;
+        modelVersion?: string;
+    }>
+}
+
+
+type LeaderboardDatasetColumnGroupSpec = {
+    datasetName: string;
+    datasetVersion?: string;
+    metricSpecs: MetricSpec[];
+    // TODO: min/max trials
+}
+
+type ScorerMetricSpec = {
+    metricType: 'scorerMetric';
+    scorerName: string;
+    scorerVersion?: string;
+    scorerType: 'object' | 'op';
+    metrics: Array<{
+        metricPath: string;
+        shouldMinimize?: boolean;
+    }>;
+}
+
+type ModelMetricSpec = {
+    metricType: 'modelLatency' | 'modelCost' | 'modelTokens' | 'modelErrors';
+}
+
+type MetricSpec = ScorerMetricSpec | ModelMetricSpec;
+
+type LeaderboardValueRecord = {
+    datasetName: string;
+    datasetVersion: string;
+    metricType: 'scorerMetric' | 'modelLatency' | 'modelCost' | 'modelTokens' | 'modelErrors';
+    scorerName: string; // modelMetrics repeat the type here
+    scorerVersion: string; // modelMetrics repeat the type here
+    metricPath: string; // modelMetrics repeat the type here
+    metricValue: number | string | boolean | null;
+    modelName: string;
+    modelVersion: string;
+    modelType: 'object' | 'op';
+    trials: number;
+    createdAt: Date;
+    sourceEvaluationCallId: string;
+    sourceEvaluationObjectRef: string;
+}
+
+export type LeaderboardData2 = Array<LeaderboardValueRecord>;
+
+export type FilterAndGroupSpec = {
+    datasets?: Array<{
+        name: string; // "*" means all
+        version: string; // "*" means all
+        splitByVersion?: boolean;
+        scorers?: Array<{
+            name: string; // "*" means all
+            version: string; // "*" means all
+            splitByVersion?: boolean;
+            metrics?: Array<{
+                path: string; // "*" means all
+                shouldMinimize?: boolean;
+            }>// null is all
+        }>// null is all
+    }> // null is all
+    models?: Array<{
+        name: string; // "*" means all
+        version: string; // "*" means all
+        splitByVersion?: boolean;
+    }> // null is all
+}
+
+export const getLeaderboardData = async (client: TraceServerClient, entity: string, project: string, spec: FilterAndGroupSpec = {}): Promise<LeaderboardData2> => {
+    // get all the evaluations
+    const allEvaluationObjectsProm = client.objsQuery(
+        {
+            project_id: projectIdFromParts({ entity, project }),
+            filter: {
+                base_object_classes: ["Evaluation"],
+                is_op: false,
+            }
+        }
+    )
+    const allEvaluationCallsProm = client.callsStreamQuery(
+        {
+            project_id: projectIdFromParts({ entity, project }),
+            filter: {
+                op_names: [opVersionKeyToRefUri({
+                    entity,
+                    project,
+                    opId: EVALUATE_OP_NAME_POST_PYDANTIC,
+                    versionHash: '*',
+                }),]
+            }
+        }
+    )
+
+    const allEvaluationObjectsRes = await allEvaluationObjectsProm;
+    const evaluationObjectDigestMap = new Map<string, { versions: Map<string, TraceObjSchema>, versionOrder: string[] }>();
+    allEvaluationObjectsRes.objs.forEach(obj => {
+        const outerKey = obj.object_id;
+        const innerKey = obj.digest;
+        if (!evaluationObjectDigestMap.has(outerKey)) {
+            evaluationObjectDigestMap.set(outerKey, { versions: new Map(), versionOrder: [] });
+        }
+        evaluationObjectDigestMap.get(outerKey)!.versions.set(innerKey, obj);
+    });
+    evaluationObjectDigestMap.forEach((value, key) => {
+        value.versionOrder = Array.from(value.versions.entries()).sort((a, b) => a[1].version_index - b[1].version_index).map(entry => entry[0]);
+    });
+
+    const allEvaluationCallsRes = await allEvaluationCallsProm;
+    const data: LeaderboardData2 = [];
+    allEvaluationCallsRes.calls.forEach(call => {
+        const evalObjectRefUri = call.inputs.self;
+        const evalObjectRef = parseRefMaybe(evalObjectRefUri ?? "");
+        const modelObjectOrOpRef = parseRefMaybe(call.inputs.model ?? "");
+
+        if (!evalObjectRef || !modelObjectOrOpRef) {
+            console.warn("Skipping evaluation call with missing eval object ref", call);
+            return;
+        }
+
+        const evalObjectName = evalObjectRef.artifactName
+        const evalObjectVersion = evalObjectRef.artifactVersion
+        const evalObject = evaluationObjectDigestMap.get(evalObjectName)?.versions.get(evalObjectVersion);
+        if (!evalObject) {
+            console.warn("Skipping evaluation call with missing eval object", call);
+            return;
+        }
+
+        const datasetRef = parseRefMaybe(evalObject.val.dataset ?? "");
+        if (!datasetRef) {
+            console.warn("Skipping evaluation call with missing dataset ref", call);
+            return;
+        }
+        const datasetName = datasetRef.artifactName;
+        const datasetVersion = datasetRef.artifactVersion;
+
+        const modelName = modelObjectOrOpRef.artifactName;
+        const modelVersion = modelObjectOrOpRef.artifactVersion;
+        if (!isWeaveObjectRef(modelObjectOrOpRef)) {
+            console.warn("Skipping evaluation call with invalid model ref", call);
+            return;
+        }
+        const modelType = modelObjectOrOpRef.weaveKind === 'op' ? 'op' : 'object';
+        const trials = (evalObject.val.trials ?? call.inputs.trials) ?? 1;
+
+        const recordPartial: Omit<LeaderboardValueRecord, 'metricType' | 'scorerName' | 'scorerVersion' | 'metricPath' | 'metricValue'> = {
+            datasetName,
+            datasetVersion,
+            modelName,
+            modelVersion,
+            modelType,
+            trials,
+            createdAt: convertISOToDate(call.started_at),
+            sourceEvaluationCallId: call.id,
+            sourceEvaluationObjectRef: evalObjectRefUri
+        }
+
+        const modelLatency = (call.output as any)?.model_latency?.mean;
+        if (modelLatency == null) {
+            console.warn("Skipping model latency", call);
+        } else {
+            const modelLatencyRecord: LeaderboardValueRecord = {
+                ...recordPartial,
+                metricType: 'modelLatency',
+                scorerName: 'modelLatency',
+                scorerVersion: 'modelLatency',
+                metricPath: 'model_latency.mean',
+                metricValue: modelLatency,
+            }
+            data.push(modelLatencyRecord);
+        }
+
+        // TODO: add modelCost, modelTokens, modelErrors
+
+        const scorerRefUris = (evalObject.val.scorers ?? []) as Array<string>
+        scorerRefUris.forEach(scorerRefUri => {
+            const scorerRef = parseRefMaybe(scorerRefUri);
+            if (!scorerRef || !isWeaveObjectRef(scorerRef)) {
+                console.warn("Skipping scorer ref", scorerRefUri);
+                return;
+            }
+            const scorerName = scorerRef.artifactName;
+            const scorerVersion = scorerRef.artifactVersion;
+            // const scorerType = scorerRef.weaveKind === 'op' ? 'op' : 'object';
+            const scorePayload = (call.output as any)?.[scorerName];
+            if (typeof scorePayload !== 'object' || scorePayload == null) {
+                console.warn("Skipping scorer call with invalid score payload", scorerName, scorerVersion, call);
+                return;
+            }
+            const flatScorePayload = flattenObjectPreservingWeaveTypes(scorePayload);
+            Object.entries(flatScorePayload).forEach(([metricPath, metricValue]) => {
+                const scoreRecord: LeaderboardValueRecord = {
+                    ...recordPartial,
+                    metricType: 'scorerMetric',
+                    scorerName,
+                    scorerVersion,
+                    metricPath: metricPath,
+                    metricValue,
+                }
+                data.push(scoreRecord);
+            })
+        })
+    });
+
+    // console.table(data);
+
+
+    // First, apply the filters. Filter can work by:
+    // Datasets:
+    //    Take All
+    //       Option; should split by version
+    //    Allow-list of Names
+    //       Option (per name); should split by version
+    //    Allow-list of Names + Versions
+    // Scorers (specified per dataset spec)
+    //    Take All
+    //       Option; should split by version
+    //    Allow-list of Names
+    //       Option (per name); should split by version
+    //    Allow-list of Names + Versions
+    // Metrics (specified per scorer spec)
+    //    Take All
+    //    Allow-list of Metric Paths
+    // Models
+    //    Take All
+    //       Option; should split by version
+    //    Allow-list of Names
+    //       Option (per name); should split by version
+    //    Allow-list of Names + Versions
+
+    const filterableGroupableData = data.map(row => {
+        const groupableRow = {
+            datasetGroup: row.datasetName,
+            scorerGroup: row.scorerName,
+            modelGroup: row.modelName,
+            metricPathGroup: row.metricPath,
+            sortKey: -row.createdAt.getTime(),
+            row,
+        }
+
+        if (!spec.datasets) {
+            return { include: true, groupableRow }
+        }
+        if (spec.datasets.length === 0) {
+            return { include: true, groupableRow }
+        }
+        if (spec.datasets.some(dataset => dataset.name === '*')) {
+            return { include: true, groupableRow }
+        }
+
+        let datasetSpec = spec.datasets.find(dataset => (dataset.name === row.datasetName) && (dataset.version === row.datasetVersion));
+        datasetSpec = datasetSpec || spec.datasets.find(dataset => (dataset.name === row.datasetName) && (dataset.version === '*' || dataset.version === row.datasetVersion));
+        datasetSpec = datasetSpec || spec.datasets.find(dataset => (dataset.name === '*' || dataset.name === row.datasetName) && (dataset.version === '*' || dataset.version === row.datasetVersion));
+        if (!datasetSpec) {
+            return { include: false, groupableRow };
+        }
+        if (datasetSpec.splitByVersion) {
+            groupableRow.datasetGroup += `${row.datasetName}:${row.datasetVersion}`;
+        }
+        if (datasetSpec.scorers) {
+
+            let scorerSpec = datasetSpec.scorers.find(scorer => (scorer.name === row.scorerName) && (scorer.version === row.scorerVersion));
+            scorerSpec = scorerSpec || datasetSpec.scorers.find(scorer => (scorer.name === row.scorerName) && (scorer.version === '*' || scorer.version === row.scorerVersion));
+            scorerSpec = scorerSpec || datasetSpec.scorers.find(scorer => (scorer.name === '*' || scorer.name === row.scorerName) && (scorer.version === '*' || scorer.version === row.scorerVersion));
+            if (!scorerSpec) {
+                return { include: false, groupableRow };
+            }
+            if (scorerSpec.splitByVersion) {
+                groupableRow.scorerGroup += `${row.scorerName}:${row.scorerVersion}`;
+            }
+            if (scorerSpec.metrics) {
+                const metricSpec = scorerSpec.metrics.find(metric => (metric.path === '*' || metric.path === row.metricPath));
+                if (!metricSpec) {
+                    return { include: false, groupableRow };
+                }
+            }
+        }
+        if (spec.models) {
+            let modelSpec = spec.models.find(model => (model.name === row.modelName) && (model.version === row.modelVersion));
+            modelSpec = modelSpec || spec.models.find(model => (model.name === row.modelName) && (model.version === '*' || model.version === row.modelVersion));
+            modelSpec = modelSpec || spec.models.find(model => (model.name === '*' || model.name === row.modelName) && (model.version === '*' || model.version === row.modelVersion));
+            if (!modelSpec) {
+                return { include: false, groupableRow };
+            }
+            if (modelSpec.splitByVersion) {
+                groupableRow.modelGroup += `${row.modelName}:${row.modelVersion}`;
+            }
+        }
+        return { include: true, groupableRow };
+    })
+
+    const groupableData = filterableGroupableData.filter(entry => entry.include).map(entry => entry.groupableRow);
+
+    const finalData = [];
+    const groupData = (data: { [key: string]: any } & { sortKey: number }[], fields: string[]): any => {
+        if (fields.length === 0) {
+            // Sort by created at descending and return the most recent record
+            // Would be better to use some form of latest.
+            const res = data.sort((a, b) => a.sortKey - b.sortKey)[0];
+            finalData.push(res);
+            return res;
+        }
+
+        const [currentField, ...remainingFields] = fields;
+        return _.mapValues(_.groupBy(data, currentField), (groupedData) =>
+            groupData(groupedData, remainingFields)
+        );
+    };
+
+    const groupedData = groupData(groupableData, ['datasetGroup', 'scorerGroup', 'metricPathGroup', 'modelGroup']);
+    console.log(groupedData);
+    console.table(finalData)
+
+    return data;
+}
+
+
+
+// type LeaderboardSpec = {
+//     columnGroups: LeaderboardDatasetColumnGroupSpec;
+//     modelSpec?: ExplicitModelSpec
+// }
+
+// type ExplicitModelSpec = {
+//     modelType: 'explicit';
+//     models: Array<{
+//         modelName: string;
+//         modelVersion?: string;
+//     }>
+// }
+
+
+// type LeaderboardDatasetColumnGroupSpec = {
+//     datasetName: string;
+//     datasetVersion?: string;
+//     metricSpecs: MetricSpec[];
+//     // TODO: min/max trials
+// }
+
+// type ScorerMetricSpec = {
+//     metricType: 'scorerMetric';
+//     scorerName: string;
+//     scorerVersion?: string;
+//     scorerType: 'object' | 'op';
+//     metrics: Array<{
+//         metricPath: string;
+//         shouldMinimize?: boolean;
+//     }>;
+// }
+
+// const groupedData = _.mapValues(_.groupBy(data, 'datasetName'), (data) => {
+//     return _.mapValues(_.groupBy(data, 'datasetVersion'), (data) => {
+//         return _.mapValues(_.groupBy(data, 'scorerName'), (data) => {
+//             return _.mapValues(_.groupBy(data, 'scorerVersion'), (data) => {
+//                 return _.mapValues(_.groupBy(data, 'metricPath'), (data) => {
+//                     return _.mapValues(_.groupBy(data, 'modelName'), (data) => {
+//                         return _.mapValues(_.groupBy(data, 'modelVersion'), (data) => {
+//                             // sort by created at descending
+//                             // Opportunity to aggregate based on the metricType (ex. avg)
+//                             return data.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())[0]
+//                         });
+//                     });
+//                 });
+//             });
+//         });
+//     });
+// });
\ No newline at end of file
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
index 6286e76d963..979665db900 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
@@ -1,4 +1,4 @@
-import {Query} from './traceServerClientInterface/query';
+import { Query } from './traceServerClientInterface/query';
 
 export type KeyedDictType = {
   [key: string]: any;
@@ -49,7 +49,7 @@ interface TraceCallsFilter {
   wb_user_ids?: string[];
 }
 
-export type SortBy = {field: string; direction: 'asc' | 'desc'};
+export type SortBy = { field: string; direction: 'asc' | 'desc' };
 
 export type TraceCallsQueryReq = {
   project_id: string;
@@ -148,6 +148,10 @@ interface TraceObjectsFilter {
 export type TraceObjQueryReq = {
   project_id: string;
   filter?: TraceObjectsFilter;
+  limit?: number;
+  offset?: number;
+  sort_by?: SortBy[];
+  metadata_only?: boolean;
 };
 
 export interface TraceObjSchema {

From 17077d3219d0866add318e17e9f7cf16cd8908cd Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 9 Oct 2024 22:38:20 -0700
Subject: [PATCH 035/169] addressed comments

---
 .../leaderboardServerInterface.ts             | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 723db9f2075..96815c797d9 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -118,6 +118,16 @@ type LeaderboardValueRecord = {
     sourceEvaluationObjectRef: string;
 }
 
+type GroupableLeaderboardValueRecord = {
+    datasetGroup: string;
+    scorerGroup: string;
+    metricPathGroup: string;
+    modelGroup: string;
+    sortKey: number;
+    row: LeaderboardValueRecord;
+}
+
+export type GroupedLeaderboardData2 = Array<GroupableLeaderboardValueRecord>;
 export type LeaderboardData2 = Array<LeaderboardValueRecord>;
 
 export type FilterAndGroupSpec = {
@@ -142,7 +152,7 @@ export type FilterAndGroupSpec = {
     }> // null is all
 }
 
-export const getLeaderboardData = async (client: TraceServerClient, entity: string, project: string, spec: FilterAndGroupSpec = {}): Promise<LeaderboardData2> => {
+export const getLeaderboardData = async (client: TraceServerClient, entity: string, project: string, spec: FilterAndGroupSpec = {}): Promise<GroupedLeaderboardData2> => {
     // get all the evaluations
     const allEvaluationObjectsProm = client.objsQuery(
         {
@@ -304,7 +314,7 @@ export const getLeaderboardData = async (client: TraceServerClient, entity: stri
     //    Allow-list of Names + Versions
 
     const filterableGroupableData = data.map(row => {
-        const groupableRow = {
+        const groupableRow: GroupableLeaderboardValueRecord = {
             datasetGroup: row.datasetName,
             scorerGroup: row.scorerName,
             modelGroup: row.modelName,
@@ -366,8 +376,8 @@ export const getLeaderboardData = async (client: TraceServerClient, entity: stri
 
     const groupableData = filterableGroupableData.filter(entry => entry.include).map(entry => entry.groupableRow);
 
-    const finalData = [];
-    const groupData = (data: { [key: string]: any } & { sortKey: number }[], fields: string[]): any => {
+    const finalData: GroupedLeaderboardData2 = [];
+    const groupData = (data: GroupableLeaderboardValueRecord[], fields: string[]): any => {
         if (fields.length === 0) {
             // Sort by created at descending and return the most recent record
             // Would be better to use some form of latest.
@@ -386,7 +396,7 @@ export const getLeaderboardData = async (client: TraceServerClient, entity: stri
     console.log(groupedData);
     console.table(finalData)
 
-    return data;
+    return finalData;
 }
 
 

From 13f9a76c4b23ad5646eab71875ba01f607e4177b Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 9 Oct 2024 22:50:48 -0700
Subject: [PATCH 036/169] lint city

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx |   2 +-
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  42 +-
 .../Browse3/pages/LeaderboardPage/hooks.ts    |  85 +-
 .../leaderboardServerInterface.ts             | 760 ++++++++++--------
 .../Browse3/pages/common/SimplePageLayout.tsx |   2 +-
 .../traceServerClientTypes.ts                 |   4 +-
 6 files changed, 496 insertions(+), 399 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 1eccaa39d9d..f366bc7f6d0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -13,7 +13,7 @@ import React, {useCallback, useMemo, useState} from 'react';
 import {parseRefMaybe, SmallRef} from '../../../Browse2/SmallRef';
 import {StyledDataGrid} from '../../StyledDataGrid';
 import {PaginationButtons} from '../CallsPage/CallsTableButtons';
-import {ObjectVersionLink} from '../common/Links';
+// import {ObjectVersionLink} from '../common/Links';
 import {buildTree} from '../common/tabularListViews/buildTree';
 import {LeaderboardData} from './hooks';
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index d2463e8ba0f..a3a3996360d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,19 +1,19 @@
-import { Alert, Box, Typography } from '@mui/material';
-import { useTraceUpdate } from '@wandb/weave/common/util/hooks';
-import { Button } from '@wandb/weave/components/Button';
-import React, { useCallback, useState } from 'react';
-import { useHistory } from 'react-router-dom';
-
-import { useWeaveflowRouteContext } from '../../context';
-import { EditableMarkdown } from './EditableMarkdown';
-import { useLeaderboardData, useLeaderboardData2 } from './hooks';
+import {Alert, Box, Typography} from '@mui/material';
+import {useTraceUpdate} from '@wandb/weave/common/util/hooks';
+import {Button} from '@wandb/weave/components/Button';
+import React, {useCallback, useState} from 'react';
+import {useHistory} from 'react-router-dom';
+
+import {useWeaveflowRouteContext} from '../../context';
+import {EditableMarkdown} from './EditableMarkdown';
+import {useLeaderboardData, useLeaderboardData2} from './hooks';
 import {
   persistLeaderboardConfig,
   useCurrentLeaderboardConfig,
 } from './leaderboardConfigQuery';
-import { LeaderboardConfigType } from './LeaderboardConfigType';
-import { LeaderboardGrid } from './LeaderboardGrid';
-import { LeaderboardConfig } from './LeaderboardPageConfig';
+import {LeaderboardConfigType} from './LeaderboardConfigType';
+import {LeaderboardGrid} from './LeaderboardGrid';
+import {LeaderboardConfig} from './LeaderboardPageConfig';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
@@ -43,13 +43,13 @@ const usePersistedLeaderboardConfig = () => {
     setConfigLocal(initialConfig);
   }, [initialConfig]);
 
-  return { config, setConfigLocal, persistConfig, cancelChanges };
+  return {config, setConfigLocal, persistConfig, cancelChanges};
 };
 
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
-  const { entity, project } = props;
+  const {entity, project} = props;
 
-  const { peekingRouter } = useWeaveflowRouteContext();
+  const {peekingRouter} = useWeaveflowRouteContext();
   const history = useHistory();
 
   const [showConfig, setShowConfig] = useState(false);
@@ -65,15 +65,15 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     (newDescription: string) => {
       setConfigLocal(newConfig => ({
         ...newConfig,
-        config: { ...newConfig.config, description: newDescription },
+        config: {...newConfig.config, description: newDescription},
       }));
       persistConfig();
     },
     [setConfigLocal, persistConfig]
   );
-  useLeaderboardData2(entity, project)
-  const { loading, data } = useLeaderboardData(entity, project, currentConfig);
-  useTraceUpdate('a', { entity, project, currentConfig });
+  useLeaderboardData2(entity, project);
+  const {loading, data} = useLeaderboardData(entity, project, currentConfig);
+  useTraceUpdate('a', {entity, project, currentConfig});
 
   const handleCellClick = (
     modelName: string,
@@ -155,7 +155,7 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
 export const ToggleLeaderboardConfig: React.FC<{
   isOpen: boolean;
   onClick: () => void;
-}> = ({ isOpen, onClick }) => {
+}> = ({isOpen, onClick}) => {
   return (
     <Box
       sx={{
@@ -174,7 +174,7 @@ export const ToggleLeaderboardConfig: React.FC<{
   );
 };
 
-const UnlistedAlert: React.FC<{ onClose: () => void }> = ({ onClose }) => {
+const UnlistedAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
   return (
     <Alert severity="info" onClose={onClose}>
       <Typography variant="body1">
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index 7c630745517..306f0e61879 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -1,12 +1,15 @@
-import { ObjectRef } from '@wandb/weave/react';
-import { useEffect, useMemo, useState, } from 'react';
+import {ObjectRef} from '@wandb/weave/react';
+import {useEffect, useMemo, useState} from 'react';
 
-import { flattenObjectPreservingWeaveTypes } from '../../../Browse2/browse2Util';
-import { parseRefMaybe } from '../../../Browse2/SmallRef';
-import { useGetTraceServerClientContext } from '../wfReactInterface/traceServerClientContext';
-import { useEvalCallsForConfig } from './leaderboardConfigQuery';
-import { LeaderboardConfigType } from './LeaderboardConfigType';
-import { getLeaderboardData, LeaderboardData2 } from './leaderboardServerInterface';
+import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
+import {parseRefMaybe} from '../../../Browse2/SmallRef';
+import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
+import {useEvalCallsForConfig} from './leaderboardConfigQuery';
+import {LeaderboardConfigType} from './LeaderboardConfigType';
+import {
+  getLeaderboardData,
+  GroupedLeaderboardData2,
+} from './leaderboardServerInterface';
 
 export type LeaderboardData = {
   metrics: {
@@ -22,7 +25,7 @@ export type LeaderboardData = {
   models: string[];
   scores: {
     [modelId: string]: {
-      [metricId: string]: { value: number; sourceEvalCallId: string };
+      [metricId: string]: {value: number; sourceEvalCallId: string};
     };
   };
 };
@@ -31,7 +34,7 @@ export const useLeaderboardData = (
   entity: string,
   project: string,
   config: LeaderboardConfigType
-): { loading: boolean; data: LeaderboardData } => {
+): {loading: boolean; data: LeaderboardData} => {
   // console.log('Fetching leaderboard data', config);
   // const {useRootObjectVersions, useCalls} = useWFHooks();
 
@@ -80,12 +83,12 @@ export const useLeaderboardData = (
   //   undefined,
   //   {skip: !evaluationVersionsResult}
   // );
-  const { calls: evaluationRuns, evals: evaluationVersions } =
+  const {calls: evaluationRuns, evals: evaluationVersions} =
     useEvalCallsForConfig(entity, project, config);
 
   // Build the dataset
 
-  const results: { loading: boolean; data: LeaderboardData } = useMemo(() => {
+  const results: {loading: boolean; data: LeaderboardData} = useMemo(() => {
     const finalData: LeaderboardData = {
       metrics: {},
       models: [],
@@ -210,38 +213,46 @@ export const useLeaderboardData = (
       loading: false,
       data: finalData,
     };
-  }, [config.config.columns, evaluationRuns.loading, evaluationRuns.result, evaluationVersions]);
+  }, [
+    config.config.columns,
+    evaluationRuns.loading,
+    evaluationRuns.result,
+    evaluationVersions,
+  ]);
 
   return results;
 };
 
-
-export const useLeaderboardData2 = (
-  entity: string, project: string
-) => {
+export const useLeaderboardData2 = (entity: string, project: string) => {
   const getTraceServerClient = useGetTraceServerClientContext();
-  const [state, setState] = useState<{ loading: boolean; data: LeaderboardData2 }>({ loading: true, data: [] });
+  const [state, setState] = useState<{
+    loading: boolean;
+    data: GroupedLeaderboardData2;
+  }>({loading: true, data: []});
   useEffect(() => {
     let mounted = true;
-    getLeaderboardData(
-      getTraceServerClient(), entity, project, {
-      datasets: [{
-        name: 'SWEBenchVerified-shuffle808-50',
-        version: '*'
-      }],
-      models: [{
-        name: '*',
-        version: '*',
-        splitByVersion: false,
-      }, {
-        name: "reproProblem",
-        version: '*',
-        splitByVersion: true,
-      }]
-    }
-    ).then(data => {
+    getLeaderboardData(getTraceServerClient(), entity, project, {
+      datasets: [
+        {
+          name: 'SWEBenchVerified-shuffle808-50',
+          version: '*',
+        },
+      ],
+      models: [
+        {
+          name: '*',
+          version: '*',
+          splitByVersion: false,
+        },
+        {
+          name: 'reproProblem',
+          version: '*',
+          splitByVersion: true,
+        },
+      ],
+    }).then(data => {
       if (mounted) {
-        setState({ loading: false, data });
+        setState({loading: false, data});
       }
     });
     return () => {
@@ -249,4 +260,4 @@ export const useLeaderboardData2 = (
     };
   }, [entity, project, getTraceServerClient]);
   return state;
-}
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 96815c797d9..798b4994654 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -1,19 +1,21 @@
-
 // type EvaluationMatrix = {
 //     datasets: {
 //         datasetName: string;
 //         versions: {
 
-import { isWeaveObjectRef } from "@wandb/weave/react";
+import {isWeaveObjectRef} from '@wandb/weave/react';
+import _ from 'lodash';
 
-import { flattenObjectPreservingWeaveTypes } from "../../../Browse2/browse2Util";
-import { parseRefMaybe } from "../../../Browse2/SmallRef";
-import { EVALUATE_OP_NAME_POST_PYDANTIC } from "../common/heuristics";
-import { TraceServerClient } from "../wfReactInterface/traceServerClient";
-import { TraceObjSchema } from "../wfReactInterface/traceServerClientTypes";
-import { convertISOToDate, projectIdFromParts } from "../wfReactInterface/tsDataModelHooks";
-import { opVersionKeyToRefUri } from "../wfReactInterface/utilities";
-import _ from "lodash";
+import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
+import {parseRefMaybe} from '../../../Browse2/SmallRef';
+import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
+import {TraceServerClient} from '../wfReactInterface/traceServerClient';
+import {TraceObjSchema} from '../wfReactInterface/traceServerClientTypes';
+import {
+  convertISOToDate,
+  projectIdFromParts,
+} from '../wfReactInterface/tsDataModelHooks';
+import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
 
 //         }
 
@@ -22,7 +24,6 @@ import _ from "lodash";
 
 // const getProjectEvaluationMatrix = () => { }
 
-
 // type EvaluationResultRecord = {
 //     datasetName: string;
 //     datasetVersion: string;
@@ -55,7 +56,7 @@ import _ from "lodash";
 //     sampleResult: { [key: string]: any }
 // }
 
-// // 
+// //
 
 // type EvaluationObjectLeaderboardSpec = {
 //     evaluationObjectVersionRefUri: string;
@@ -63,343 +64,429 @@ import _ from "lodash";
 
 // ----
 
-type LeaderboardSpec = {
-    columnGroups: LeaderboardDatasetColumnGroupSpec;
-    modelSpec?: ExplicitModelSpec
-}
-
-type ExplicitModelSpec = {
-    modelType: 'explicit';
-    models: Array<{
-        modelName: string;
-        modelVersion?: string;
-    }>
-}
-
-
-type LeaderboardDatasetColumnGroupSpec = {
-    datasetName: string;
-    datasetVersion?: string;
-    metricSpecs: MetricSpec[];
-    // TODO: min/max trials
-}
-
-type ScorerMetricSpec = {
-    metricType: 'scorerMetric';
-    scorerName: string;
-    scorerVersion?: string;
-    scorerType: 'object' | 'op';
-    metrics: Array<{
-        metricPath: string;
-        shouldMinimize?: boolean;
-    }>;
-}
-
-type ModelMetricSpec = {
-    metricType: 'modelLatency' | 'modelCost' | 'modelTokens' | 'modelErrors';
-}
-
-type MetricSpec = ScorerMetricSpec | ModelMetricSpec;
+// type LeaderboardSpec = {
+//     columnGroups: LeaderboardDatasetColumnGroupSpec;
+//     modelSpec?: ExplicitModelSpec
+// }
 
-type LeaderboardValueRecord = {
-    datasetName: string;
-    datasetVersion: string;
-    metricType: 'scorerMetric' | 'modelLatency' | 'modelCost' | 'modelTokens' | 'modelErrors';
-    scorerName: string; // modelMetrics repeat the type here
-    scorerVersion: string; // modelMetrics repeat the type here
-    metricPath: string; // modelMetrics repeat the type here
-    metricValue: number | string | boolean | null;
-    modelName: string;
-    modelVersion: string;
-    modelType: 'object' | 'op';
-    trials: number;
-    createdAt: Date;
-    sourceEvaluationCallId: string;
-    sourceEvaluationObjectRef: string;
-}
+// type ExplicitModelSpec = {
+//     modelType: 'explicit';
+//     models: Array<{
+//         modelName: string;
+//         modelVersion?: string;
+//     }>
+// }
 
-type GroupableLeaderboardValueRecord = {
-    datasetGroup: string;
-    scorerGroup: string;
-    metricPathGroup: string;
-    modelGroup: string;
-    sortKey: number;
-    row: LeaderboardValueRecord;
-}
+// type LeaderboardDatasetColumnGroupSpec = {
+//     datasetName: string;
+//     datasetVersion?: string;
+//     metricSpecs: MetricSpec[];
+//     // TODO: min/max trials
+// }
 
-export type GroupedLeaderboardData2 = Array<GroupableLeaderboardValueRecord>;
-export type LeaderboardData2 = Array<LeaderboardValueRecord>;
+// type ScorerMetricSpec = {
+//     metricType: 'scorerMetric';
+//     scorerName: string;
+//     scorerVersion?: string;
+//     scorerType: 'object' | 'op';
+//     metrics: Array<{
+//         metricPath: string;
+//         shouldMinimize?: boolean;
+//     }>;
+// }
 
-export type FilterAndGroupSpec = {
-    datasets?: Array<{
-        name: string; // "*" means all
-        version: string; // "*" means all
-        splitByVersion?: boolean;
-        scorers?: Array<{
-            name: string; // "*" means all
-            version: string; // "*" means all
-            splitByVersion?: boolean;
-            metrics?: Array<{
-                path: string; // "*" means all
-                shouldMinimize?: boolean;
-            }>// null is all
-        }>// null is all
-    }> // null is all
-    models?: Array<{
-        name: string; // "*" means all
-        version: string; // "*" means all
-        splitByVersion?: boolean;
-    }> // null is all
-}
-
-export const getLeaderboardData = async (client: TraceServerClient, entity: string, project: string, spec: FilterAndGroupSpec = {}): Promise<GroupedLeaderboardData2> => {
-    // get all the evaluations
-    const allEvaluationObjectsProm = client.objsQuery(
-        {
-            project_id: projectIdFromParts({ entity, project }),
-            filter: {
-                base_object_classes: ["Evaluation"],
-                is_op: false,
-            }
-        }
-    )
-    const allEvaluationCallsProm = client.callsStreamQuery(
-        {
-            project_id: projectIdFromParts({ entity, project }),
-            filter: {
-                op_names: [opVersionKeyToRefUri({
-                    entity,
-                    project,
-                    opId: EVALUATE_OP_NAME_POST_PYDANTIC,
-                    versionHash: '*',
-                }),]
-            }
-        }
-    )
-
-    const allEvaluationObjectsRes = await allEvaluationObjectsProm;
-    const evaluationObjectDigestMap = new Map<string, { versions: Map<string, TraceObjSchema>, versionOrder: string[] }>();
-    allEvaluationObjectsRes.objs.forEach(obj => {
-        const outerKey = obj.object_id;
-        const innerKey = obj.digest;
-        if (!evaluationObjectDigestMap.has(outerKey)) {
-            evaluationObjectDigestMap.set(outerKey, { versions: new Map(), versionOrder: [] });
-        }
-        evaluationObjectDigestMap.get(outerKey)!.versions.set(innerKey, obj);
-    });
-    evaluationObjectDigestMap.forEach((value, key) => {
-        value.versionOrder = Array.from(value.versions.entries()).sort((a, b) => a[1].version_index - b[1].version_index).map(entry => entry[0]);
-    });
+// type ModelMetricSpec = {
+//     metricType: 'modelLatency' | 'modelCost' | 'modelTokens' | 'modelErrors';
+// }
 
-    const allEvaluationCallsRes = await allEvaluationCallsProm;
-    const data: LeaderboardData2 = [];
-    allEvaluationCallsRes.calls.forEach(call => {
-        const evalObjectRefUri = call.inputs.self;
-        const evalObjectRef = parseRefMaybe(evalObjectRefUri ?? "");
-        const modelObjectOrOpRef = parseRefMaybe(call.inputs.model ?? "");
+// type MetricSpec = ScorerMetricSpec | ModelMetricSpec;
 
-        if (!evalObjectRef || !modelObjectOrOpRef) {
-            console.warn("Skipping evaluation call with missing eval object ref", call);
-            return;
-        }
+type LeaderboardValueRecord = {
+  datasetName: string;
+  datasetVersion: string;
+  metricType:
+    | 'scorerMetric'
+    | 'modelLatency'
+    | 'modelCost'
+    | 'modelTokens'
+    | 'modelErrors';
+  scorerName: string; // modelMetrics repeat the type here
+  scorerVersion: string; // modelMetrics repeat the type here
+  metricPath: string; // modelMetrics repeat the type here
+  metricValue: number | string | boolean | null;
+  modelName: string;
+  modelVersion: string;
+  modelType: 'object' | 'op';
+  trials: number;
+  createdAt: Date;
+  sourceEvaluationCallId: string;
+  sourceEvaluationObjectRef: string;
+};
 
-        const evalObjectName = evalObjectRef.artifactName
-        const evalObjectVersion = evalObjectRef.artifactVersion
-        const evalObject = evaluationObjectDigestMap.get(evalObjectName)?.versions.get(evalObjectVersion);
-        if (!evalObject) {
-            console.warn("Skipping evaluation call with missing eval object", call);
-            return;
-        }
+type GroupableLeaderboardValueRecord = {
+  datasetGroup: string;
+  scorerGroup: string;
+  metricPathGroup: string;
+  modelGroup: string;
+  sortKey: number;
+  row: LeaderboardValueRecord;
+};
 
-        const datasetRef = parseRefMaybe(evalObject.val.dataset ?? "");
-        if (!datasetRef) {
-            console.warn("Skipping evaluation call with missing dataset ref", call);
-            return;
-        }
-        const datasetName = datasetRef.artifactName;
-        const datasetVersion = datasetRef.artifactVersion;
-
-        const modelName = modelObjectOrOpRef.artifactName;
-        const modelVersion = modelObjectOrOpRef.artifactVersion;
-        if (!isWeaveObjectRef(modelObjectOrOpRef)) {
-            console.warn("Skipping evaluation call with invalid model ref", call);
-            return;
-        }
-        const modelType = modelObjectOrOpRef.weaveKind === 'op' ? 'op' : 'object';
-        const trials = (evalObject.val.trials ?? call.inputs.trials) ?? 1;
-
-        const recordPartial: Omit<LeaderboardValueRecord, 'metricType' | 'scorerName' | 'scorerVersion' | 'metricPath' | 'metricValue'> = {
-            datasetName,
-            datasetVersion,
-            modelName,
-            modelVersion,
-            modelType,
-            trials,
-            createdAt: convertISOToDate(call.started_at),
-            sourceEvaluationCallId: call.id,
-            sourceEvaluationObjectRef: evalObjectRefUri
-        }
+export type GroupedLeaderboardData2 = GroupableLeaderboardValueRecord[];
+export type LeaderboardData2 = LeaderboardValueRecord[];
 
-        const modelLatency = (call.output as any)?.model_latency?.mean;
-        if (modelLatency == null) {
-            console.warn("Skipping model latency", call);
-        } else {
-            const modelLatencyRecord: LeaderboardValueRecord = {
-                ...recordPartial,
-                metricType: 'modelLatency',
-                scorerName: 'modelLatency',
-                scorerVersion: 'modelLatency',
-                metricPath: 'model_latency.mean',
-                metricValue: modelLatency,
-            }
-            data.push(modelLatencyRecord);
-        }
+export type FilterAndGroupSpec = {
+  datasets?: Array<{
+    name: string; // "*" means all
+    version: string; // "*" means all
+    splitByVersion?: boolean;
+    scorers?: Array<{
+      name: string; // "*" means all
+      version: string; // "*" means all
+      splitByVersion?: boolean;
+      metrics?: Array<{
+        path: string; // "*" means all
+        shouldMinimize?: boolean;
+      }>; // null is all
+    }>; // null is all
+  }>; // null is all
+  models?: Array<{
+    name: string; // "*" means all
+    version: string; // "*" means all
+    splitByVersion?: boolean;
+  }>; // null is all
+};
+
+export const getLeaderboardData = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
+  spec: FilterAndGroupSpec = {}
+): Promise<GroupedLeaderboardData2> => {
+  // get all the evaluations
+  const allEvaluationObjectsProm = client.objsQuery({
+    project_id: projectIdFromParts({entity, project}),
+    filter: {
+      base_object_classes: ['Evaluation'],
+      is_op: false,
+    },
+  });
+  const allEvaluationCallsProm = client.callsStreamQuery({
+    project_id: projectIdFromParts({entity, project}),
+    filter: {
+      op_names: [
+        opVersionKeyToRefUri({
+          entity,
+          project,
+          opId: EVALUATE_OP_NAME_POST_PYDANTIC,
+          versionHash: '*',
+        }),
+      ],
+    },
+  });
+
+  const allEvaluationObjectsRes = await allEvaluationObjectsProm;
+  const evaluationObjectDigestMap = new Map<
+    string,
+    {versions: Map<string, TraceObjSchema>; versionOrder: string[]}
+  >();
+  allEvaluationObjectsRes.objs.forEach(obj => {
+    const outerKey = obj.object_id;
+    const innerKey = obj.digest;
+    if (!evaluationObjectDigestMap.has(outerKey)) {
+      evaluationObjectDigestMap.set(outerKey, {
+        versions: new Map(),
+        versionOrder: [],
+      });
+    }
+    evaluationObjectDigestMap.get(outerKey)!.versions.set(innerKey, obj);
+  });
+  evaluationObjectDigestMap.forEach((value, key) => {
+    value.versionOrder = Array.from(value.versions.entries())
+      .sort((a, b) => a[1].version_index - b[1].version_index)
+      .map(entry => entry[0]);
+  });
+
+  const allEvaluationCallsRes = await allEvaluationCallsProm;
+  const data: LeaderboardData2 = [];
+  allEvaluationCallsRes.calls.forEach(call => {
+    const evalObjectRefUri = call.inputs.self;
+    const evalObjectRef = parseRefMaybe(evalObjectRefUri ?? '');
+    const modelObjectOrOpRef = parseRefMaybe(call.inputs.model ?? '');
+
+    if (!evalObjectRef || !modelObjectOrOpRef) {
+      console.warn(
+        'Skipping evaluation call with missing eval object ref',
+        call
+      );
+      return;
+    }
+
+    const evalObjectName = evalObjectRef.artifactName;
+    const evalObjectVersion = evalObjectRef.artifactVersion;
+    const evalObject = evaluationObjectDigestMap
+      .get(evalObjectName)
+      ?.versions.get(evalObjectVersion);
+    if (!evalObject) {
+      console.warn('Skipping evaluation call with missing eval object', call);
+      return;
+    }
+
+    const datasetRef = parseRefMaybe(evalObject.val.dataset ?? '');
+    if (!datasetRef) {
+      console.warn('Skipping evaluation call with missing dataset ref', call);
+      return;
+    }
+    const datasetName = datasetRef.artifactName;
+    const datasetVersion = datasetRef.artifactVersion;
+
+    const modelName = modelObjectOrOpRef.artifactName;
+    const modelVersion = modelObjectOrOpRef.artifactVersion;
+    if (!isWeaveObjectRef(modelObjectOrOpRef)) {
+      console.warn('Skipping evaluation call with invalid model ref', call);
+      return;
+    }
+    const modelType = modelObjectOrOpRef.weaveKind === 'op' ? 'op' : 'object';
+    const trials = evalObject.val.trials ?? call.inputs.trials ?? 1;
+
+    const recordPartial: Omit<
+      LeaderboardValueRecord,
+      | 'metricType'
+      | 'scorerName'
+      | 'scorerVersion'
+      | 'metricPath'
+      | 'metricValue'
+    > = {
+      datasetName,
+      datasetVersion,
+      modelName,
+      modelVersion,
+      modelType,
+      trials,
+      createdAt: convertISOToDate(call.started_at),
+      sourceEvaluationCallId: call.id,
+      sourceEvaluationObjectRef: evalObjectRefUri,
+    };
 
-        // TODO: add modelCost, modelTokens, modelErrors
-
-        const scorerRefUris = (evalObject.val.scorers ?? []) as Array<string>
-        scorerRefUris.forEach(scorerRefUri => {
-            const scorerRef = parseRefMaybe(scorerRefUri);
-            if (!scorerRef || !isWeaveObjectRef(scorerRef)) {
-                console.warn("Skipping scorer ref", scorerRefUri);
-                return;
-            }
-            const scorerName = scorerRef.artifactName;
-            const scorerVersion = scorerRef.artifactVersion;
-            // const scorerType = scorerRef.weaveKind === 'op' ? 'op' : 'object';
-            const scorePayload = (call.output as any)?.[scorerName];
-            if (typeof scorePayload !== 'object' || scorePayload == null) {
-                console.warn("Skipping scorer call with invalid score payload", scorerName, scorerVersion, call);
-                return;
-            }
-            const flatScorePayload = flattenObjectPreservingWeaveTypes(scorePayload);
-            Object.entries(flatScorePayload).forEach(([metricPath, metricValue]) => {
-                const scoreRecord: LeaderboardValueRecord = {
-                    ...recordPartial,
-                    metricType: 'scorerMetric',
-                    scorerName,
-                    scorerVersion,
-                    metricPath: metricPath,
-                    metricValue,
-                }
-                data.push(scoreRecord);
-            })
-        })
+    const modelLatency = (call.output as any)?.model_latency?.mean;
+    if (modelLatency == null) {
+      console.warn('Skipping model latency', call);
+    } else {
+      const modelLatencyRecord: LeaderboardValueRecord = {
+        ...recordPartial,
+        metricType: 'modelLatency',
+        scorerName: 'modelLatency',
+        scorerVersion: 'modelLatency',
+        metricPath: 'model_latency.mean',
+        metricValue: modelLatency,
+      };
+      data.push(modelLatencyRecord);
+    }
+
+    // TODO: add modelCost, modelTokens, modelErrors
+
+    const scorerRefUris = (evalObject.val.scorers ?? []) as string[];
+    scorerRefUris.forEach(scorerRefUri => {
+      const scorerRef = parseRefMaybe(scorerRefUri);
+      if (!scorerRef || !isWeaveObjectRef(scorerRef)) {
+        console.warn('Skipping scorer ref', scorerRefUri);
+        return;
+      }
+      const scorerName = scorerRef.artifactName;
+      const scorerVersion = scorerRef.artifactVersion;
+      // const scorerType = scorerRef.weaveKind === 'op' ? 'op' : 'object';
+      const scorePayload = (call.output as any)?.[scorerName];
+      if (typeof scorePayload !== 'object' || scorePayload == null) {
+        console.warn(
+          'Skipping scorer call with invalid score payload',
+          scorerName,
+          scorerVersion,
+          call
+        );
+        return;
+      }
+      const flatScorePayload = flattenObjectPreservingWeaveTypes(scorePayload);
+      Object.entries(flatScorePayload).forEach(([metricPath, metricValue]) => {
+        const scoreRecord: LeaderboardValueRecord = {
+          ...recordPartial,
+          metricType: 'scorerMetric',
+          scorerName,
+          scorerVersion,
+          metricPath,
+          metricValue,
+        };
+        data.push(scoreRecord);
+      });
     });
+  });
+
+  // console.table(data);
+
+  // First, apply the filters. Filter can work by:
+  // Datasets:
+  //    Take All
+  //       Option; should split by version
+  //    Allow-list of Names
+  //       Option (per name); should split by version
+  //    Allow-list of Names + Versions
+  // Scorers (specified per dataset spec)
+  //    Take All
+  //       Option; should split by version
+  //    Allow-list of Names
+  //       Option (per name); should split by version
+  //    Allow-list of Names + Versions
+  // Metrics (specified per scorer spec)
+  //    Take All
+  //    Allow-list of Metric Paths
+  // Models
+  //    Take All
+  //       Option; should split by version
+  //    Allow-list of Names
+  //       Option (per name); should split by version
+  //    Allow-list of Names + Versions
+
+  const filterableGroupableData = data.map(row => {
+    const groupableRow: GroupableLeaderboardValueRecord = {
+      datasetGroup: row.datasetName,
+      scorerGroup: row.scorerName,
+      modelGroup: row.modelName,
+      metricPathGroup: row.metricPath,
+      sortKey: -row.createdAt.getTime(),
+      row,
+    };
 
-    // console.table(data);
-
-
-    // First, apply the filters. Filter can work by:
-    // Datasets:
-    //    Take All
-    //       Option; should split by version
-    //    Allow-list of Names
-    //       Option (per name); should split by version
-    //    Allow-list of Names + Versions
-    // Scorers (specified per dataset spec)
-    //    Take All
-    //       Option; should split by version
-    //    Allow-list of Names
-    //       Option (per name); should split by version
-    //    Allow-list of Names + Versions
-    // Metrics (specified per scorer spec)
-    //    Take All
-    //    Allow-list of Metric Paths
-    // Models
-    //    Take All
-    //       Option; should split by version
-    //    Allow-list of Names
-    //       Option (per name); should split by version
-    //    Allow-list of Names + Versions
-
-    const filterableGroupableData = data.map(row => {
-        const groupableRow: GroupableLeaderboardValueRecord = {
-            datasetGroup: row.datasetName,
-            scorerGroup: row.scorerName,
-            modelGroup: row.modelName,
-            metricPathGroup: row.metricPath,
-            sortKey: -row.createdAt.getTime(),
-            row,
-        }
-
-        if (!spec.datasets) {
-            return { include: true, groupableRow }
-        }
-        if (spec.datasets.length === 0) {
-            return { include: true, groupableRow }
-        }
-        if (spec.datasets.some(dataset => dataset.name === '*')) {
-            return { include: true, groupableRow }
-        }
-
-        let datasetSpec = spec.datasets.find(dataset => (dataset.name === row.datasetName) && (dataset.version === row.datasetVersion));
-        datasetSpec = datasetSpec || spec.datasets.find(dataset => (dataset.name === row.datasetName) && (dataset.version === '*' || dataset.version === row.datasetVersion));
-        datasetSpec = datasetSpec || spec.datasets.find(dataset => (dataset.name === '*' || dataset.name === row.datasetName) && (dataset.version === '*' || dataset.version === row.datasetVersion));
-        if (!datasetSpec) {
-            return { include: false, groupableRow };
-        }
-        if (datasetSpec.splitByVersion) {
-            groupableRow.datasetGroup += `${row.datasetName}:${row.datasetVersion}`;
-        }
-        if (datasetSpec.scorers) {
-
-            let scorerSpec = datasetSpec.scorers.find(scorer => (scorer.name === row.scorerName) && (scorer.version === row.scorerVersion));
-            scorerSpec = scorerSpec || datasetSpec.scorers.find(scorer => (scorer.name === row.scorerName) && (scorer.version === '*' || scorer.version === row.scorerVersion));
-            scorerSpec = scorerSpec || datasetSpec.scorers.find(scorer => (scorer.name === '*' || scorer.name === row.scorerName) && (scorer.version === '*' || scorer.version === row.scorerVersion));
-            if (!scorerSpec) {
-                return { include: false, groupableRow };
-            }
-            if (scorerSpec.splitByVersion) {
-                groupableRow.scorerGroup += `${row.scorerName}:${row.scorerVersion}`;
-            }
-            if (scorerSpec.metrics) {
-                const metricSpec = scorerSpec.metrics.find(metric => (metric.path === '*' || metric.path === row.metricPath));
-                if (!metricSpec) {
-                    return { include: false, groupableRow };
-                }
-            }
-        }
-        if (spec.models) {
-            let modelSpec = spec.models.find(model => (model.name === row.modelName) && (model.version === row.modelVersion));
-            modelSpec = modelSpec || spec.models.find(model => (model.name === row.modelName) && (model.version === '*' || model.version === row.modelVersion));
-            modelSpec = modelSpec || spec.models.find(model => (model.name === '*' || model.name === row.modelName) && (model.version === '*' || model.version === row.modelVersion));
-            if (!modelSpec) {
-                return { include: false, groupableRow };
-            }
-            if (modelSpec.splitByVersion) {
-                groupableRow.modelGroup += `${row.modelName}:${row.modelVersion}`;
-            }
-        }
-        return { include: true, groupableRow };
-    })
-
-    const groupableData = filterableGroupableData.filter(entry => entry.include).map(entry => entry.groupableRow);
-
-    const finalData: GroupedLeaderboardData2 = [];
-    const groupData = (data: GroupableLeaderboardValueRecord[], fields: string[]): any => {
-        if (fields.length === 0) {
-            // Sort by created at descending and return the most recent record
-            // Would be better to use some form of latest.
-            const res = data.sort((a, b) => a.sortKey - b.sortKey)[0];
-            finalData.push(res);
-            return res;
+    if (!spec.datasets) {
+      return {include: true, groupableRow};
+    }
+    if (spec.datasets.length === 0) {
+      return {include: true, groupableRow};
+    }
+    if (spec.datasets.some(dataset => dataset.name === '*')) {
+      return {include: true, groupableRow};
+    }
+
+    let datasetSpec = spec.datasets.find(
+      dataset =>
+        dataset.name === row.datasetName &&
+        dataset.version === row.datasetVersion
+    );
+    datasetSpec =
+      datasetSpec ||
+      spec.datasets.find(
+        dataset =>
+          dataset.name === row.datasetName &&
+          (dataset.version === '*' || dataset.version === row.datasetVersion)
+      );
+    datasetSpec =
+      datasetSpec ||
+      spec.datasets.find(
+        dataset =>
+          (dataset.name === '*' || dataset.name === row.datasetName) &&
+          (dataset.version === '*' || dataset.version === row.datasetVersion)
+      );
+    if (!datasetSpec) {
+      return {include: false, groupableRow};
+    }
+    if (datasetSpec.splitByVersion) {
+      groupableRow.datasetGroup += `${row.datasetName}:${row.datasetVersion}`;
+    }
+    if (datasetSpec.scorers) {
+      let scorerSpec = datasetSpec.scorers.find(
+        scorer =>
+          scorer.name === row.scorerName && scorer.version === row.scorerVersion
+      );
+      scorerSpec =
+        scorerSpec ||
+        datasetSpec.scorers.find(
+          scorer =>
+            scorer.name === row.scorerName &&
+            (scorer.version === '*' || scorer.version === row.scorerVersion)
+        );
+      scorerSpec =
+        scorerSpec ||
+        datasetSpec.scorers.find(
+          scorer =>
+            (scorer.name === '*' || scorer.name === row.scorerName) &&
+            (scorer.version === '*' || scorer.version === row.scorerVersion)
+        );
+      if (!scorerSpec) {
+        return {include: false, groupableRow};
+      }
+      if (scorerSpec.splitByVersion) {
+        groupableRow.scorerGroup += `${row.scorerName}:${row.scorerVersion}`;
+      }
+      if (scorerSpec.metrics) {
+        const metricSpec = scorerSpec.metrics.find(
+          metric => metric.path === '*' || metric.path === row.metricPath
+        );
+        if (!metricSpec) {
+          return {include: false, groupableRow};
         }
-
-        const [currentField, ...remainingFields] = fields;
-        return _.mapValues(_.groupBy(data, currentField), (groupedData) =>
-            groupData(groupedData, remainingFields)
+      }
+    }
+    if (spec.models) {
+      let modelSpec = spec.models.find(
+        model =>
+          model.name === row.modelName && model.version === row.modelVersion
+      );
+      modelSpec =
+        modelSpec ||
+        spec.models.find(
+          model =>
+            model.name === row.modelName &&
+            (model.version === '*' || model.version === row.modelVersion)
         );
-    };
-
-    const groupedData = groupData(groupableData, ['datasetGroup', 'scorerGroup', 'metricPathGroup', 'modelGroup']);
-    console.log(groupedData);
-    console.table(finalData)
-
-    return finalData;
-}
-
-
+      modelSpec =
+        modelSpec ||
+        spec.models.find(
+          model =>
+            (model.name === '*' || model.name === row.modelName) &&
+            (model.version === '*' || model.version === row.modelVersion)
+        );
+      if (!modelSpec) {
+        return {include: false, groupableRow};
+      }
+      if (modelSpec.splitByVersion) {
+        groupableRow.modelGroup += `${row.modelName}:${row.modelVersion}`;
+      }
+    }
+    return {include: true, groupableRow};
+  });
+
+  const groupableData = filterableGroupableData
+    .filter(entry => entry.include)
+    .map(entry => entry.groupableRow);
+
+  const finalData: GroupedLeaderboardData2 = [];
+  const groupData = (
+    recordList: GroupableLeaderboardValueRecord[],
+    fields: string[]
+  ): any => {
+    if (fields.length === 0) {
+      // Sort by created at descending and return the most recent record
+      // Would be better to use some form of latest.
+      const res = recordList.sort((a, b) => a.sortKey - b.sortKey)[0];
+      finalData.push(res);
+      return res;
+    }
+
+    const [currentField, ...remainingFields] = fields;
+    return _.mapValues(_.groupBy(recordList, currentField), groupedRecords =>
+      groupData(groupedRecords, remainingFields)
+    );
+  };
+
+  const groupedData = groupData(groupableData, [
+    'datasetGroup',
+    'scorerGroup',
+    'metricPathGroup',
+    'modelGroup',
+  ]);
+  console.log(groupedData);
+  console.table(finalData);
+
+  return finalData;
+};
 
 // type LeaderboardSpec = {
 //     columnGroups: LeaderboardDatasetColumnGroupSpec;
@@ -414,7 +501,6 @@ export const getLeaderboardData = async (client: TraceServerClient, entity: stri
 //     }>
 // }
 
-
 // type LeaderboardDatasetColumnGroupSpec = {
 //     datasetName: string;
 //     datasetVersion?: string;
@@ -449,4 +535,4 @@ export const getLeaderboardData = async (client: TraceServerClient, entity: stri
 //             });
 //         });
 //     });
-// });
\ No newline at end of file
+// });
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx
index 89fbccd4be2..b8e308e51f2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx
@@ -24,7 +24,7 @@ export const SimplePageLayoutContext =
   createContext<SimplePageLayoutContextType>({});
 
 export const SimplePageLayout: FC<{
-  title: string;
+  title: ReactNode;
   tabs: Array<{
     label: string;
     content: ReactNode;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
index 979665db900..5ccc3f1d10b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
@@ -1,4 +1,4 @@
-import { Query } from './traceServerClientInterface/query';
+import {Query} from './traceServerClientInterface/query';
 
 export type KeyedDictType = {
   [key: string]: any;
@@ -49,7 +49,7 @@ interface TraceCallsFilter {
   wb_user_ids?: string[];
 }
 
-export type SortBy = { field: string; direction: 'asc' | 'desc' };
+export type SortBy = {field: string; direction: 'asc' | 'desc'};
 
 export type TraceCallsQueryReq = {
   project_id: string;

From 0537bd5efc2139ffe4c924d545d41f90d36b9107 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 9 Oct 2024 22:56:04 -0700
Subject: [PATCH 037/169] REFACTOR START

---
 .../LeaderboardPage/LeaderboardConfigType.tsx |   2 +-
 .../LeaderboardPage/LeaderboardPageConfig.tsx |  12 +-
 .../Browse3/pages/LeaderboardPage/hooks.ts    |   6 +-
 .../LeaderboardPage/leaderboardConfigQuery.ts |   6 +-
 .../leaderboardServerInterface.ts             | 174 +-----------------
 5 files changed, 17 insertions(+), 183 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
index 62823cc9378..791400f3105 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
@@ -5,7 +5,7 @@ export type LeaderboardConfigType = {
   version: number;
   config: {
     description: string;
-    columns: Array<{
+    datasets: Array<{
       dataset: {
         name: string;
         version: VersionSpec;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 66dcacc2197..8d289b4c6f0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -29,8 +29,8 @@ export const LeaderboardConfig: React.FC<{
       ...prev,
       config: {
         ...prev.config,
-        columns: [
-          ...prev.config.columns,
+        datasets: [
+          ...prev.config.datasets,
           {dataset: {name: '', version: 'latest'}, scores: []},
         ],
       },
@@ -42,20 +42,20 @@ export const LeaderboardConfig: React.FC<{
       ...prev,
       config: {
         ...prev.config,
-        columns: prev.config.columns.filter((_, i) => i !== index),
+        datasets: prev.config.datasets.filter((_, i) => i !== index),
       },
     }));
   };
 
   const handleUpdateColumn = (
     index: number,
-    updatedColumn: LeaderboardConfigType['config']['columns'][0]
+    updatedColumn: LeaderboardConfigType['config']['datasets'][0]
   ) => {
     setConfig(prev => ({
       ...prev,
       config: {
         ...prev.config,
-        columns: prev.config.columns.map((column, i) =>
+        datasets: prev.config.datasets.map((column, i) =>
           i === index ? updatedColumn : column
         ),
       },
@@ -123,7 +123,7 @@ export const LeaderboardConfig: React.FC<{
           <Typography variant="h6" gutterBottom>
             Columns
           </Typography>
-          {config.config.columns.map((column, index) => (
+          {config.config.datasets.map((column, index) => (
             <ColumnConfig
               entity={entity}
               project={project}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index 306f0e61879..87c51f5c0c1 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -147,8 +147,8 @@ export const useLeaderboardData = (
           .map((s: ObjectRef) => [s.artifactName, s.artifactVersion])
           .filter(([name, version]: [string, string]) => {
             return (
-              config.config.columns.length === 0 ||
-              config.config.columns.some(dc => {
+              config.config.datasets.length === 0 ||
+              config.config.datasets.some(dc => {
                 return (
                   (dc.dataset.name === '' || dc.dataset.name === datasetName) &&
                   (dc.dataset.version === 'all' ||
@@ -214,7 +214,7 @@ export const useLeaderboardData = (
       data: finalData,
     };
   }, [
-    config.config.columns,
+    config.config.datasets,
     evaluationRuns.loading,
     evaluationRuns.result,
     evaluationVersions,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
index 905aca976b2..bf918e31008 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
@@ -371,7 +371,7 @@ export const useEvalObjsForConfig = (
       if (!datasetRef) {
         return;
       }
-      if (config.config.columns.length === 0) {
+      if (config.config.datasets.length === 0) {
         finalEvals.push(evaluation);
         return;
       }
@@ -379,7 +379,7 @@ export const useEvalObjsForConfig = (
       const datasetVersion = datasetRef.artifactVersion;
       // Determine this dataset matches any of the config's datasets
       let matched = false;
-      for (const column of config.config.columns) {
+      for (const column of config.config.datasets) {
         if (
           (datasetName === column.dataset.name || column.dataset.name === '') &&
           (column.dataset.version === datasetVersion ||
@@ -419,7 +419,7 @@ export const useEvalObjsForConfig = (
     });
 
     return finalEvals;
-  }, [config.config.columns, evalQuery.result]);
+  }, [config.config.datasets, evalQuery.result]);
 };
 
 export const useEvalCallsForConfig = (
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 798b4994654..18fcd88a571 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -1,8 +1,3 @@
-// type EvaluationMatrix = {
-//     datasets: {
-//         datasetName: string;
-//         versions: {
-
 import {isWeaveObjectRef} from '@wandb/weave/react';
 import _ from 'lodash';
 
@@ -17,90 +12,6 @@ import {
 } from '../wfReactInterface/tsDataModelHooks';
 import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
 
-//         }
-
-//     }
-// };
-
-// const getProjectEvaluationMatrix = () => { }
-
-// type EvaluationResultRecord = {
-//     datasetName: string;
-//     datasetVersion: string;
-//     scorerName: string;
-//     scorerVersion: string;
-//     metricPath: string;
-//     metricValue: number | string | boolean | null;
-//     modelName: string;
-//     modelVersion: string;
-//     modelType: 'object' | 'op';
-//     trials: number;
-//     sourceEvaluationCallId: string;
-//     sourceEvaluationObjectRef: string;
-// }
-
-// type EvaluationDefinition = {
-//     sourceEvaluationObjectRef: string;
-//     datasetName: string;
-//     datasetVersion: string;
-//     scorers: Array<{
-//         scorerName: string;
-//         scorerVersion: string;
-//     }>
-// }
-
-// type ScorerDefinition = {
-//     scorerName: string;
-//     scorerVersion: string;
-//     type: 'object' | 'op'
-//     sampleResult: { [key: string]: any }
-// }
-
-// //
-
-// type EvaluationObjectLeaderboardSpec = {
-//     evaluationObjectVersionRefUri: string;
-// }
-
-// ----
-
-// type LeaderboardSpec = {
-//     columnGroups: LeaderboardDatasetColumnGroupSpec;
-//     modelSpec?: ExplicitModelSpec
-// }
-
-// type ExplicitModelSpec = {
-//     modelType: 'explicit';
-//     models: Array<{
-//         modelName: string;
-//         modelVersion?: string;
-//     }>
-// }
-
-// type LeaderboardDatasetColumnGroupSpec = {
-//     datasetName: string;
-//     datasetVersion?: string;
-//     metricSpecs: MetricSpec[];
-//     // TODO: min/max trials
-// }
-
-// type ScorerMetricSpec = {
-//     metricType: 'scorerMetric';
-//     scorerName: string;
-//     scorerVersion?: string;
-//     scorerType: 'object' | 'op';
-//     metrics: Array<{
-//         metricPath: string;
-//         shouldMinimize?: boolean;
-//     }>;
-// }
-
-// type ModelMetricSpec = {
-//     metricType: 'modelLatency' | 'modelCost' | 'modelTokens' | 'modelErrors';
-// }
-
-// type MetricSpec = ScorerMetricSpec | ModelMetricSpec;
-
 type LeaderboardValueRecord = {
   datasetName: string;
   datasetVersion: string;
@@ -320,31 +231,6 @@ export const getLeaderboardData = async (
     });
   });
 
-  // console.table(data);
-
-  // First, apply the filters. Filter can work by:
-  // Datasets:
-  //    Take All
-  //       Option; should split by version
-  //    Allow-list of Names
-  //       Option (per name); should split by version
-  //    Allow-list of Names + Versions
-  // Scorers (specified per dataset spec)
-  //    Take All
-  //       Option; should split by version
-  //    Allow-list of Names
-  //       Option (per name); should split by version
-  //    Allow-list of Names + Versions
-  // Metrics (specified per scorer spec)
-  //    Take All
-  //    Allow-list of Metric Paths
-  // Models
-  //    Take All
-  //       Option; should split by version
-  //    Allow-list of Names
-  //       Option (per name); should split by version
-  //    Allow-list of Names + Versions
-
   const filterableGroupableData = data.map(row => {
     const groupableRow: GroupableLeaderboardValueRecord = {
       datasetGroup: row.datasetName,
@@ -458,7 +344,7 @@ export const getLeaderboardData = async (
     .map(entry => entry.groupableRow);
 
   const finalData: GroupedLeaderboardData2 = [];
-  const groupData = (
+  const groupedVisitor = (
     recordList: GroupableLeaderboardValueRecord[],
     fields: string[]
   ): any => {
@@ -467,72 +353,20 @@ export const getLeaderboardData = async (
       // Would be better to use some form of latest.
       const res = recordList.sort((a, b) => a.sortKey - b.sortKey)[0];
       finalData.push(res);
-      return res;
     }
 
     const [currentField, ...remainingFields] = fields;
-    return _.mapValues(_.groupBy(recordList, currentField), groupedRecords =>
-      groupData(groupedRecords, remainingFields)
+    return _.values(_.groupBy(recordList, currentField)).forEach(groupedRecords =>
+      groupedVisitor(groupedRecords, remainingFields)
     );
   };
 
-  const groupedData = groupData(groupableData, [
+  groupedVisitor(groupableData, [
     'datasetGroup',
     'scorerGroup',
     'metricPathGroup',
     'modelGroup',
   ]);
-  console.log(groupedData);
-  console.table(finalData);
 
   return finalData;
 };
-
-// type LeaderboardSpec = {
-//     columnGroups: LeaderboardDatasetColumnGroupSpec;
-//     modelSpec?: ExplicitModelSpec
-// }
-
-// type ExplicitModelSpec = {
-//     modelType: 'explicit';
-//     models: Array<{
-//         modelName: string;
-//         modelVersion?: string;
-//     }>
-// }
-
-// type LeaderboardDatasetColumnGroupSpec = {
-//     datasetName: string;
-//     datasetVersion?: string;
-//     metricSpecs: MetricSpec[];
-//     // TODO: min/max trials
-// }
-
-// type ScorerMetricSpec = {
-//     metricType: 'scorerMetric';
-//     scorerName: string;
-//     scorerVersion?: string;
-//     scorerType: 'object' | 'op';
-//     metrics: Array<{
-//         metricPath: string;
-//         shouldMinimize?: boolean;
-//     }>;
-// }
-
-// const groupedData = _.mapValues(_.groupBy(data, 'datasetName'), (data) => {
-//     return _.mapValues(_.groupBy(data, 'datasetVersion'), (data) => {
-//         return _.mapValues(_.groupBy(data, 'scorerName'), (data) => {
-//             return _.mapValues(_.groupBy(data, 'scorerVersion'), (data) => {
-//                 return _.mapValues(_.groupBy(data, 'metricPath'), (data) => {
-//                     return _.mapValues(_.groupBy(data, 'modelName'), (data) => {
-//                         return _.mapValues(_.groupBy(data, 'modelVersion'), (data) => {
-//                             // sort by created at descending
-//                             // Opportunity to aggregate based on the metricType (ex. avg)
-//                             return data.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())[0]
-//                         });
-//                     });
-//                 });
-//             });
-//         });
-//     });
-// });

From c8f542e2b95f063959b8f405761f8d7a91e36e5c Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 9 Oct 2024 23:44:52 -0700
Subject: [PATCH 038/169] Massive refactor step 1

---
 .../pages/LeaderboardPage/ColumnConfig.tsx    | 138 -----
 .../LeaderboardPage/LeaderboardConfigType.tsx |  57 +-
 .../pages/LeaderboardPage/LeaderboardGrid.tsx | 499 +++++++++---------
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  32 +-
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 117 ----
 .../pages/LeaderboardPage/MetricConfig.tsx    | 148 ------
 .../pages/LeaderboardPage/ModelConfig.tsx     |  74 ---
 .../pages/LeaderboardPage/ScoreConfig.tsx     | 154 ------
 .../Browse3/pages/LeaderboardPage/hooks.ts    | 263 +--------
 .../LeaderboardPage/leaderboardConfigQuery.ts | 460 ----------------
 .../leaderboardServerInterface.ts             | 101 ++--
 11 files changed, 364 insertions(+), 1679 deletions(-)
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
deleted file mode 100644
index aeeb172d4ae..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ColumnConfig.tsx
+++ /dev/null
@@ -1,138 +0,0 @@
-import AddIcon from '@mui/icons-material/Add';
-import DeleteIcon from '@mui/icons-material/Delete';
-import {
-  Box,
-  Button,
-  FormControl,
-  IconButton,
-  InputLabel,
-  MenuItem,
-  Select,
-  Stack,
-  Typography,
-} from '@mui/material';
-import React from 'react';
-
-import {
-  useDatasetNames,
-  useDatasetVersionsForDatasetName,
-} from './leaderboardConfigQuery';
-import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
-import {ScoreConfig} from './ScoreConfig';
-
-export const ColumnConfig: React.FC<{
-  entity: string;
-  project: string;
-  column: LeaderboardConfigType['config']['columns'][0];
-  onUpdate: (
-    updatedColumn: LeaderboardConfigType['config']['columns'][0]
-  ) => void;
-  onRemove: () => void;
-}> = ({entity, project, column, onUpdate, onRemove}) => {
-  const datasetNames = useDatasetNames(entity, project);
-  const datasetVersions = useDatasetVersionsForDatasetName(
-    entity,
-    project,
-    column.dataset.name
-  );
-
-  const handleAddScore = () => {
-    onUpdate({
-      ...column,
-      scores: [
-        ...column.scores,
-        {scorer: {name: '', version: 'latest'}, metrics: []},
-      ],
-    });
-  };
-
-  const handleRemoveScore = (index: number) => {
-    onUpdate({
-      ...column,
-      scores: column.scores.filter((_, i) => i !== index),
-    });
-  };
-
-  const handleUpdateScore = (
-    index: number,
-    updatedScore: (typeof column.scores)[0]
-  ) => {
-    onUpdate({
-      ...column,
-      scores: column.scores.map((score, i) =>
-        i === index ? updatedScore : score
-      ),
-    });
-  };
-
-  return (
-    <Box sx={{mb: 2, p: 2, border: '1px solid #ccc', borderRadius: 1}}>
-      <Stack
-        direction="row"
-        justifyContent="space-between"
-        alignItems="center"
-        sx={{mb: 2}}>
-        <Typography variant="subtitle1">Column</Typography>
-        <IconButton onClick={onRemove} size="small">
-          <DeleteIcon />
-        </IconButton>
-      </Stack>
-      <FormControl fullWidth sx={{mb: 2}}>
-        <InputLabel>Dataset</InputLabel>
-        <Select
-          value={column.dataset.name}
-          label="Dataset"
-          onChange={e =>
-            onUpdate({
-              ...column,
-              dataset: {...column.dataset, name: e.target.value},
-            })
-          }>
-          {datasetNames.map(name => (
-            <MenuItem key={name} value={name}>
-              {name}
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-      <FormControl fullWidth sx={{mb: 2}}>
-        <InputLabel>Dataset Version</InputLabel>
-        <Select
-          value={column.dataset.version}
-          label="Dataset Version"
-          onChange={e =>
-            onUpdate({
-              ...column,
-              dataset: {
-                ...column.dataset,
-                version: e.target.value as VersionSpec,
-              },
-            })
-          }>
-          <MenuItem value="latest">Latest</MenuItem>
-          <MenuItem value="all">All</MenuItem>
-          {datasetVersions.map(({version, versionIndex}) => (
-            <MenuItem key={version} value={version}>
-              Version {versionIndex}: {version}
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-      {column.scores.map((score, index) => (
-        <ScoreConfig
-          entity={entity}
-          project={project}
-          key={index}
-          score={score}
-          datasetName={column.dataset.name}
-          datasetVersion={column.dataset.version}
-          onUpdate={updatedScore => handleUpdateScore(index, updatedScore)}
-          onRemove={() => handleRemoveScore(index)}
-        />
-      ))}
-      <Button startIcon={<AddIcon />} onClick={handleAddScore} sx={{mt: 1}}>
-        Add Score
-      </Button>
-    </Box>
-  );
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
index 791400f3105..f1161f20477 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
@@ -5,31 +5,36 @@ export type LeaderboardConfigType = {
   version: number;
   config: {
     description: string;
-    datasets: Array<{
-      dataset: {
-        name: string;
-        version: VersionSpec;
-      };
-      scores: Array<{
-        scorer: {
-          name: string;
-          version: VersionSpec;
-        };
-        metrics: Array<{
-          displayName: string;
-          path: string[];
-          minTrials?: number;
-          maxTrials?: number;
-          sort?: {
-            precedence: number;
-            direction: Direction;
-          };
-        }>;
-      }>;
-    }>;
-    models: Array<{
-      name: string;
-      version: VersionSpec;
-    }>;
+    dataSelectionSpec: FilterAndGroupSpec;
   };
 };
+
+export type FilterAndGroupSpec = {
+  datasets?: FilterAndGroupDatasetSpec[]; // null is all
+  models?: FilterAndGroupModelSpec[]; // null is all
+};
+
+export type FilterAndGroupDatasetSpec = {
+  name: string; // "*" means all
+  version: string; // "*" means all
+  splitByVersion?: boolean;
+  scorers?: FilterAndGroupDatasetScorerSpec[]; // null is all
+};
+
+export type FilterAndGroupDatasetScorerSpec = {
+  name: string; // "*" means all
+  version: string; // "*" means all
+  splitByVersion?: boolean;
+  metrics?: FilterAndGroupDatasetScorerMetricSpec[]; // null is all
+};
+
+export type FilterAndGroupDatasetScorerMetricSpec = {
+  path: string; // "*" means all
+  shouldMinimize?: boolean;
+};
+
+export type FilterAndGroupModelSpec = {
+  name: string; // "*" means all
+  version: string; // "*" means all
+  splitByVersion?: boolean;
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index f366bc7f6d0..8717dd1a215 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -13,14 +13,16 @@ import React, {useCallback, useMemo, useState} from 'react';
 import {parseRefMaybe, SmallRef} from '../../../Browse2/SmallRef';
 import {StyledDataGrid} from '../../StyledDataGrid';
 import {PaginationButtons} from '../CallsPage/CallsTableButtons';
-// import {ObjectVersionLink} from '../common/Links';
 import {buildTree} from '../common/tabularListViews/buildTree';
-import {LeaderboardData} from './hooks';
+import {
+  GroupableLeaderboardValueRecord,
+  GroupedLeaderboardData,
+} from './leaderboardServerInterface';
 
 interface LeaderboardGridProps {
   entity: string;
   project: string;
-  data: LeaderboardData;
+  data: GroupedLeaderboardData;
   loading: boolean;
   onCellClick: (modelName: string, metricName: string, score: number) => void;
 }
@@ -28,7 +30,7 @@ interface LeaderboardGridProps {
 type RowData = {
   id: number;
   model: string;
-} & LeaderboardData['scores'][string];
+} & GroupableLeaderboardValueRecord;
 
 export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   entity,
@@ -37,264 +39,251 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   loading,
   onCellClick,
 }) => {
-  const [paginationModel, setPaginationModel] = useState<GridPaginationModel>({
-    pageSize: 50,
-    page: 0,
-  });
-
-  const orderedMetrics = useMemo(() => {
-    return Object.keys(data.metrics);
-    // return Object.keys(data.metrics).sort((a, b) => {
-    //   return a.localeCompare(b);
-    // });
-  }, [data.metrics]);
+  return <>Grid</>;
+  // const [paginationModel, setPaginationModel] = useState<GridPaginationModel>({
+  //   pageSize: 50,
+  //   page: 0,
+  // });
 
-  const metricRanges = useMemo(() => {
-    const ranges: {[key: string]: {min: number; max: number}} = {};
-    orderedMetrics.forEach(metric => {
-      const scores = data.models
-        .map(model => data.scores?.[model]?.[metric]?.value)
-        .filter(score => score !== undefined);
-      ranges[metric] = {
-        min: Math.min(...scores),
-        max: Math.max(...scores),
-      };
-    });
-    return ranges;
-  }, [data.models, data.scores, orderedMetrics]);
+  // const orderedMetrics = useMemo(() => {
+  //   return Object.keys(data.metrics);
+  //   // return Object.keys(data.metrics).sort((a, b) => {
+  //   //   return a.localeCompare(b);
+  //   // });
+  // }, [data.metrics]);
 
-  const getColorForScore = useCallback(
-    (metric: string, score: number | undefined) => {
-      if (score === undefined) {
-        return 'transparent';
-      }
-      const {min, max} = metricRanges[metric];
-      const normalizedScore = (score - min) / (max - min);
-      return `hsl(${120 * normalizedScore}, 70%, 85%)`;
-    },
-    [metricRanges]
-  );
+  // const metricRanges = useMemo(() => {
+  //   const ranges: {[key: string]: {min: number; max: number}} = {};
+  //   orderedMetrics.forEach(metric => {
+  //     const scores = data.models
+  //       .map(model => data.scores?.[model]?.[metric]?.value)
+  //       .filter(score => score !== undefined);
+  //     ranges[metric] = {
+  //       min: Math.min(...scores),
+  //       max: Math.max(...scores),
+  //     };
+  //   });
+  //   return ranges;
+  // }, [data.models, data.scores, orderedMetrics]);
 
-  const rows: RowData[] = useMemo(
-    () =>
-      data.models.map((model, index) => ({
-        id: index,
-        model,
-        ...data.scores[model],
-      })) as RowData[],
-    [data.models, data.scores]
-  );
-
-  const columns: Array<GridColDef<RowData>> = useMemo(
-    () => [
-      {
-        field: 'model',
-        headerName: 'Model',
-        minWidth: 100,
-        flex: 1,
-        renderCell: (params: GridRenderCellParams) => {
-          const modelRef = parseRefMaybe(params.value);
-          if (modelRef) {
-            return (
-              <div
-                style={{
-                  width: '100%',
-                  height: '100%',
-                  alignContent: 'center',
-                  display: 'flex',
-                  alignItems: 'center',
-                  justifyContent: 'center',
-                  lineHeight: '20px',
-                }}>
-                <SmallRef objRef={modelRef} />
-              </div>
-            );
-          }
-          return <div>{params.value}</div>;
-        },
-      },
-      ...orderedMetrics.map(metric => ({
-        field: metric,
-        headerName: data.metrics[metric].metricPath.split('.').pop(),
-        minWidth: 100,
-        flex: 1,
-        valueGetter: (value: RowData) => {
-          return value?.value;
-        },
-        getSortComparator: (dir: GridSortDirection) => (a: any, b: any) => {
-          const aValue = a;
-          const bValue = b;
-          if (aValue == null && bValue == null) {
-            return 0;
-          }
-          // Ignoring direction here allows nulls to always sort to the end
-          if (aValue == null) {
-            return 1;
-          }
-          if (bValue == null) {
-            return -1;
-          }
-          if (typeof aValue === 'number' && typeof bValue === 'number') {
-            if (dir === 'asc') {
-              return aValue - bValue;
-            } else {
-              return bValue - aValue;
-            }
-          }
-          return aValue.localeCompare(bValue);
-        },
-        renderCell: (params: GridRenderCellParams) => {
-          let inner = params.value;
-          if (typeof inner === 'number') {
-            if (inner < 1) {
-              inner = `${(inner * 100).toFixed(2)}%`;
-            } else {
-              inner = `${inner.toFixed(2)}`;
-            }
-          } else {
-            inner = JSON.stringify(params.value);
-          }
-          // const value =
-          // <CellValue value={params.value} />
-          return (
-            <div
-              style={{
-                width: '100%',
-                height: '100%',
-                display: 'flex',
-                alignItems: 'center',
-                justifyContent: 'center',
-                backgroundColor: getColorForScore(metric, params.value),
-              }}
-              onClick={() =>
-                onCellClick(
-                  params.row.model,
-                  params.field as string,
-                  params.value as number
-                )
-              }>
-              {' '}
-              {inner}
-            </div>
-          );
-        },
-      })),
-    ],
-    [data.metrics, getColorForScore, onCellClick, orderedMetrics]
-  );
+  // const getColorForScore = useCallback(
+  //   (metric: string, score: number | undefined) => {
+  //     if (score === undefined) {
+  //       return 'transparent';
+  //     }
+  //     const {min, max} = metricRanges[metric];
+  //     const normalizedScore = (score - min) / (max - min);
+  //     return `hsl(${120 * normalizedScore}, 70%, 85%)`;
+  //   },
+  //   [metricRanges]
+  // );
 
-  const tree = buildTree([...Object.keys(data.metrics)]);
-  let groupingModel: GridColumnGroupingModel = tree.children.filter(
-    c => 'groupId' in c
-  ) as GridColumnGroup[];
-  groupingModel = walkGroupingModel(groupingModel, node => {
-    if ('groupId' in node) {
-      if (node.children.length === 1) {
-        if (
-          'groupId' in node.children[0] &&
-          !node.headerName?.includes(':') &&
-          !node.children[0].headerName?.includes(':')
-        ) {
-          const currNode = node;
-          node = node.children[0];
-          node.headerName = currNode.headerName + '.' + node.headerName;
-        } else {
-          // pass
-          // node = node.children[0];
-        }
-      }
-    }
-    return node;
-  }) as GridColumnGroup[];
-  groupingModel = walkGroupingModel(groupingModel, node => {
-    if ('groupId' in node) {
-      node.renderHeaderGroup = params => {
-        const ref = parseRefMaybe(
-          `weave:///${entity}/${project}/object/${node.headerName}` ?? ''
-        );
-        // console.log(node.headerName, ref);
-        if (ref) {
-          return <SmallRef objRef={ref} />;
-        }
-        return <div>{node.headerName}</div>;
-      };
-    }
-    return node;
-  }) as GridColumnGroup[];
+  // const rows: RowData[] = useMemo(
+  //   () =>
+  //     data.models.map((model, index) => ({
+  //       id: index,
+  //       model,
+  //       ...data.scores[model],
+  //     })) as RowData[],
+  //   [data.models, data.scores]
+  // );
 
-  // const groupingModel: GridColumnGroupingModel = useMemo(
-  //   () => {
-  //     return [
-  //       {
-  //         groupId: 'metrics',
-  //         children: Object.keys(data.metrics).map(metric => ({
-  //           field: metric,
-  //         })),
+  // const columns: Array<GridColDef<RowData>> = useMemo(
+  //   () => [
+  //     {
+  //       field: 'model',
+  //       headerName: 'Model',
+  //       minWidth: 100,
+  //       flex: 1,
+  //       renderCell: (params: GridRenderCellParams) => {
+  //         const modelRef = parseRefMaybe(params.value);
+  //         if (modelRef) {
+  //           return (
+  //             <div
+  //               style={{
+  //                 width: '100%',
+  //                 height: '100%',
+  //                 alignContent: 'center',
+  //                 display: 'flex',
+  //                 alignItems: 'center',
+  //                 justifyContent: 'center',
+  //                 lineHeight: '20px',
+  //               }}>
+  //               <SmallRef objRef={modelRef} />
+  //             </div>
+  //           );
+  //         }
+  //         return <div>{params.value}</div>;
   //       },
-  //     ];
-  //   },
-  //   [data.metrics]
+  //     },
+  //     ...orderedMetrics.map(metric => ({
+  //       field: metric,
+  //       headerName: data.metrics[metric].metricPath.split('.').pop(),
+  //       minWidth: 100,
+  //       flex: 1,
+  //       valueGetter: (value: RowData) => {
+  //         return value?.value;
+  //       },
+  //       getSortComparator: (dir: GridSortDirection) => (a: any, b: any) => {
+  //         const aValue = a;
+  //         const bValue = b;
+  //         if (aValue == null && bValue == null) {
+  //           return 0;
+  //         }
+  //         // Ignoring direction here allows nulls to always sort to the end
+  //         if (aValue == null) {
+  //           return 1;
+  //         }
+  //         if (bValue == null) {
+  //           return -1;
+  //         }
+  //         if (typeof aValue === 'number' && typeof bValue === 'number') {
+  //           if (dir === 'asc') {
+  //             return aValue - bValue;
+  //           } else {
+  //             return bValue - aValue;
+  //           }
+  //         }
+  //         return aValue.localeCompare(bValue);
+  //       },
+  //       renderCell: (params: GridRenderCellParams) => {
+  //         let inner = params.value;
+  //         if (typeof inner === 'number') {
+  //           if (inner < 1) {
+  //             inner = `${(inner * 100).toFixed(2)}%`;
+  //           } else {
+  //             inner = `${inner.toFixed(2)}`;
+  //           }
+  //         } else {
+  //           inner = JSON.stringify(params.value);
+  //         }
+  //         // const value =
+  //         // <CellValue value={params.value} />
+  //         return (
+  //           <div
+  //             style={{
+  //               width: '100%',
+  //               height: '100%',
+  //               display: 'flex',
+  //               alignItems: 'center',
+  //               justifyContent: 'center',
+  //               backgroundColor: getColorForScore(metric, params.value),
+  //             }}
+  //             onClick={() =>
+  //               onCellClick(
+  //                 params.row.model,
+  //                 params.field as string,
+  //                 params.value as number
+  //               )
+  //             }>
+  //             {' '}
+  //             {inner}
+  //           </div>
+  //         );
+  //       },
+  //     })),
+  //   ],
+  //   [data.metrics, getColorForScore, onCellClick, orderedMetrics]
   // );
 
-  return (
-    <Box
-      sx={{
-        height: '100%',
-        width: '100%',
-        display: 'flex',
-        flexDirection: 'column',
-        overflow: 'hidden', // Prevent outer container from scrolling
-      }}>
-      <StyledDataGrid
-        rows={rows}
-        columns={columns}
-        columnGroupingModel={groupingModel}
-        pagination
-        paginationModel={paginationModel}
-        onPaginationModelChange={setPaginationModel}
-        pageSizeOptions={[50]}
-        disableRowSelectionOnClick
-        hideFooterSelectedRowCount
-        disableMultipleColumnsSorting={false}
-        columnHeaderHeight={40}
-        loading={loading}
-        sx={{
-          borderRadius: 0,
-          '& .MuiDataGrid-footerContainer': {
-            justifyContent: 'flex-start',
-          },
-          '& .MuiDataGrid-cell': {
-            cursor: 'pointer',
-          },
-          flexGrow: 1,
-          width: 'calc(100% + 1px)', // Add 1px to account for the right border
-          '& .MuiDataGrid-main': {
-            overflow: 'hidden',
-          },
-          '& .MuiDataGrid-virtualScroller': {
-            overflow: 'auto',
-          },
-          '& .MuiDataGrid-columnHeaders': {
-            overflow: 'hidden',
-          },
-        }}
-        slots={{
-          // columnMenu: CallsCustomColumnMenu,
-          pagination: PaginationButtons,
-        }}
-      />
-    </Box>
-  );
-};
+  // const tree = buildTree([...Object.keys(data.metrics)]);
+  // let groupingModel: GridColumnGroupingModel = tree.children.filter(
+  //   c => 'groupId' in c
+  // ) as GridColumnGroup[];
+  // groupingModel = walkGroupingModel(groupingModel, node => {
+  //   if ('groupId' in node) {
+  //     if (node.children.length === 1) {
+  //       if (
+  //         'groupId' in node.children[0] &&
+  //         !node.headerName?.includes(':') &&
+  //         !node.children[0].headerName?.includes(':')
+  //       ) {
+  //         const currNode = node;
+  //         node = node.children[0];
+  //         node.headerName = currNode.headerName + '.' + node.headerName;
+  //       } else {
+  //         // pass
+  //         // node = node.children[0];
+  //       }
+  //     }
+  //   }
+  //   return node;
+  // }) as GridColumnGroup[];
+  // groupingModel = walkGroupingModel(groupingModel, node => {
+  //   if ('groupId' in node) {
+  //     node.renderHeaderGroup = params => {
+  //       const ref = parseRefMaybe(
+  //         `weave:///${entity}/${project}/object/${node.headerName}` ?? ''
+  //       );
+  //       // console.log(node.headerName, ref);
+  //       if (ref) {
+  //         return <SmallRef objRef={ref} />;
+  //       }
+  //       return <div>{node.headerName}</div>;
+  //     };
+  //   }
+  //   return node;
+  // }) as GridColumnGroup[];
 
-const walkGroupingModel = (
-  nodes: GridColumnNode[],
-  fn: (node: GridColumnNode) => GridColumnNode
-) => {
-  return nodes.map(node => {
-    if ('children' in node) {
-      node.children = walkGroupingModel(node.children, fn);
-    }
-    return fn(node);
-  });
+  // return (
+  //   <Box
+  //     sx={{
+  //       height: '100%',
+  //       width: '100%',
+  //       display: 'flex',
+  //       flexDirection: 'column',
+  //       overflow: 'hidden', // Prevent outer container from scrolling
+  //     }}>
+  //     <StyledDataGrid
+  //       rows={rows}
+  //       columns={columns}
+  //       columnGroupingModel={groupingModel}
+  //       pagination
+  //       paginationModel={paginationModel}
+  //       onPaginationModelChange={setPaginationModel}
+  //       pageSizeOptions={[50]}
+  //       disableRowSelectionOnClick
+  //       hideFooterSelectedRowCount
+  //       disableMultipleColumnsSorting={false}
+  //       columnHeaderHeight={40}
+  //       loading={loading}
+  //       sx={{
+  //         borderRadius: 0,
+  //         '& .MuiDataGrid-footerContainer': {
+  //           justifyContent: 'flex-start',
+  //         },
+  //         '& .MuiDataGrid-cell': {
+  //           cursor: 'pointer',
+  //         },
+  //         flexGrow: 1,
+  //         width: 'calc(100% + 1px)', // Add 1px to account for the right border
+  //         '& .MuiDataGrid-main': {
+  //           overflow: 'hidden',
+  //         },
+  //         '& .MuiDataGrid-virtualScroller': {
+  //           overflow: 'auto',
+  //         },
+  //         '& .MuiDataGrid-columnHeaders': {
+  //           overflow: 'hidden',
+  //         },
+  //       }}
+  //       slots={{
+  //         // columnMenu: CallsCustomColumnMenu,
+  //         pagination: PaginationButtons,
+  //       }}
+  //     />
+  //   </Box>
+  // );
 };
+
+// const walkGroupingModel = (
+//   nodes: GridColumnNode[],
+//   fn: (node: GridColumnNode) => GridColumnNode
+// ) => {
+//   return nodes.map(node => {
+//     if ('children' in node) {
+//       node.children = walkGroupingModel(node.children, fn);
+//     }
+//     return fn(node);
+//   });
+// };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index a3a3996360d..dcc69a5ca1b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,16 +1,12 @@
 import {Alert, Box, Typography} from '@mui/material';
 import {useTraceUpdate} from '@wandb/weave/common/util/hooks';
 import {Button} from '@wandb/weave/components/Button';
-import React, {useCallback, useState} from 'react';
+import React, {useCallback, useMemo, useState} from 'react';
 import {useHistory} from 'react-router-dom';
 
 import {useWeaveflowRouteContext} from '../../context';
 import {EditableMarkdown} from './EditableMarkdown';
-import {useLeaderboardData, useLeaderboardData2} from './hooks';
-import {
-  persistLeaderboardConfig,
-  useCurrentLeaderboardConfig,
-} from './leaderboardConfigQuery';
+import {useLeaderboardData} from './hooks';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
 import {LeaderboardGrid} from './LeaderboardGrid';
 import {LeaderboardConfig} from './LeaderboardPageConfig';
@@ -31,13 +27,19 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
 const DEFAULT_DESCRIPTION = `# Leaderboard`;
 
 const usePersistedLeaderboardConfig = () => {
-  const initialConfig = useCurrentLeaderboardConfig();
+  const initialConfig: LeaderboardConfigType = useMemo(() => {
+    return {
+      version: 1,
+      config: {description: '', dataSelectionSpec: {}},
+    };
+  }, []);
+
   const [config, setConfigLocal] =
     useState<LeaderboardConfigType>(initialConfig);
 
   const persistConfig = useCallback(() => {
-    persistLeaderboardConfig(config);
-  }, [config]);
+    // persistLeaderboardConfig(config);
+  }, []);
 
   const cancelChanges = useCallback(() => {
     setConfigLocal(initialConfig);
@@ -71,8 +73,12 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     },
     [setConfigLocal, persistConfig]
   );
-  useLeaderboardData2(entity, project);
-  const {loading, data} = useLeaderboardData(entity, project, currentConfig);
+
+  const {loading, data} = useLeaderboardData(
+    entity,
+    project,
+    currentConfig.config.dataSelectionSpec
+  );
   useTraceUpdate('a', {entity, project, currentConfig});
 
   const handleCellClick = (
@@ -80,8 +86,8 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     metricName: string,
     score: number
   ) => {
-    const sourceCallId =
-      data.scores?.[modelName]?.[metricName]?.sourceEvalCallId;
+    const sourceCallId = false;
+    // data.scores?.[modelName]?.[metricName]?.sourceEvalCallId;
     if (sourceCallId) {
       let to: string;
       if (USE_COMPARE_EVALUATIONS_PAGE) {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 8d289b4c6f0..1f1f70b0b7f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -1,10 +1,7 @@
-import AddIcon from '@mui/icons-material/Add';
 import {Alert, Box, Button, Typography} from '@mui/material';
 import React, {useState} from 'react';
 
-import {ColumnConfig} from './ColumnConfig';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
-import {ModelConfig} from './ModelConfig';
 
 export const LeaderboardConfig: React.FC<{
   entity: string;
@@ -24,79 +21,6 @@ export const LeaderboardConfig: React.FC<{
     onCancel();
   };
 
-  const handleAddColumn = () => {
-    setConfig(prev => ({
-      ...prev,
-      config: {
-        ...prev.config,
-        datasets: [
-          ...prev.config.datasets,
-          {dataset: {name: '', version: 'latest'}, scores: []},
-        ],
-      },
-    }));
-  };
-
-  const handleRemoveColumn = (index: number) => {
-    setConfig(prev => ({
-      ...prev,
-      config: {
-        ...prev.config,
-        datasets: prev.config.datasets.filter((_, i) => i !== index),
-      },
-    }));
-  };
-
-  const handleUpdateColumn = (
-    index: number,
-    updatedColumn: LeaderboardConfigType['config']['datasets'][0]
-  ) => {
-    setConfig(prev => ({
-      ...prev,
-      config: {
-        ...prev.config,
-        datasets: prev.config.datasets.map((column, i) =>
-          i === index ? updatedColumn : column
-        ),
-      },
-    }));
-  };
-
-  const handleAddModel = () => {
-    setConfig(prev => ({
-      ...prev,
-      config: {
-        ...prev.config,
-        models: [...prev.config.models, {name: '', version: 'latest'}],
-      },
-    }));
-  };
-
-  const handleRemoveModel = (index: number) => {
-    setConfig(prev => ({
-      ...prev,
-      config: {
-        ...prev.config,
-        models: prev.config.models.filter((_, i) => i !== index),
-      },
-    }));
-  };
-
-  const handleUpdateModel = (
-    index: number,
-    updatedModel: LeaderboardConfigType['config']['models'][0]
-  ) => {
-    setConfig(prev => ({
-      ...prev,
-      config: {
-        ...prev.config,
-        models: prev.config.models.map((model, i) =>
-          i === index ? updatedModel : model
-        ),
-      },
-    }));
-  };
-
   const [showAlert, setShowAlert] = useState(true);
 
   return (
@@ -118,47 +42,6 @@ export const LeaderboardConfig: React.FC<{
         <Typography variant="h5" gutterBottom>
           Leaderboard Configuration
         </Typography>
-
-        <Box sx={{mb: 4}}>
-          <Typography variant="h6" gutterBottom>
-            Columns
-          </Typography>
-          {config.config.datasets.map((column, index) => (
-            <ColumnConfig
-              entity={entity}
-              project={project}
-              key={index}
-              column={column}
-              onUpdate={updatedColumn =>
-                handleUpdateColumn(index, updatedColumn)
-              }
-              onRemove={() => handleRemoveColumn(index)}
-            />
-          ))}
-          <Button
-            startIcon={<AddIcon />}
-            onClick={handleAddColumn}
-            sx={{mt: 2}}>
-            Add Column
-          </Button>
-        </Box>
-
-        <Box sx={{mb: 4}}>
-          <Typography variant="h6" gutterBottom>
-            Models
-          </Typography>
-          {config.config.models.map((model, index) => (
-            <ModelConfig
-              key={index}
-              model={model}
-              onUpdate={updatedModel => handleUpdateModel(index, updatedModel)}
-              onRemove={() => handleRemoveModel(index)}
-            />
-          ))}
-          <Button startIcon={<AddIcon />} onClick={handleAddModel} sx={{mt: 2}}>
-            Add Model
-          </Button>
-        </Box>
       </Box>
 
       <Box
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
deleted file mode 100644
index 53dceb058b8..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/MetricConfig.tsx
+++ /dev/null
@@ -1,148 +0,0 @@
-import DeleteIcon from '@mui/icons-material/Delete';
-import {
-  Box,
-  FormControl,
-  IconButton,
-  InputLabel,
-  MenuItem,
-  Select,
-  Stack,
-  TextField,
-  Typography,
-} from '@mui/material';
-import React from 'react';
-
-import {useMetricPathsForDatasetAndScorer} from './leaderboardConfigQuery';
-import {Direction, LeaderboardConfigType} from './LeaderboardConfigType';
-
-export const MetricConfig: React.FC<{
-  metric: LeaderboardConfigType['config']['columns'][0]['scores'][0]['metrics'][0];
-  entity: string;
-  project: string;
-  datasetName: string;
-  datasetVersion: string;
-  scorerName: string;
-  scorerVersion: string;
-  onUpdate: (
-    updatedMetric: LeaderboardConfigType['config']['columns'][0]['scores'][0]['metrics'][0]
-  ) => void;
-  onRemove: () => void;
-}> = ({
-  metric,
-  entity,
-  project,
-  datasetName,
-  datasetVersion,
-  scorerName,
-  scorerVersion,
-  onUpdate,
-  onRemove,
-}) => {
-  const metricPaths = useMetricPathsForDatasetAndScorer(
-    entity,
-    project,
-    datasetName,
-    datasetVersion,
-    scorerName,
-    scorerVersion
-  );
-
-  return (
-    <Box sx={{mb: 2, p: 2, border: '1px solid #f5f5f5', borderRadius: 1}}>
-      <Stack
-        direction="row"
-        justifyContent="space-between"
-        alignItems="center"
-        sx={{mb: 2}}>
-        <Typography variant="subtitle2">Metric</Typography>
-        <IconButton onClick={onRemove} size="small">
-          <DeleteIcon />
-        </IconButton>
-      </Stack>
-      {/* <TextField
-        fullWidth
-        label="Display Name"
-        value={metric.displayName}
-        onChange={e => onUpdate({...metric, displayName: e.target.value})}
-        sx={{mb: 2}}
-      /> */}
-      <FormControl fullWidth sx={{mb: 2}}>
-        <InputLabel>Metric Path</InputLabel>
-        <Select
-          value={metric.path.join('.')}
-          label="Metric Path"
-          onChange={e =>
-            onUpdate({...metric, path: e.target.value.split('.')})
-          }>
-          {metricPaths.map(path => (
-            <MenuItem key={path} value={path}>
-              {path}
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-      {/* <TextField
-        fullWidth
-        label="Min Trials"
-        type="number"
-        value={metric.minTrials || ''}
-        onChange={e =>
-          onUpdate({
-            ...metric,
-            minTrials: parseInt(e.target.value, 10) || undefined,
-          })
-        }
-        sx={{mb: 2}}
-      />
-      <TextField
-        fullWidth
-        label="Max Trials"
-        type="number"
-        value={metric.maxTrials || ''}
-        onChange={e =>
-          onUpdate({
-            ...metric,
-            maxTrials: parseInt(e.target.value, 10) || undefined,
-          })
-        }
-        sx={{mb: 2}}
-      /> */}
-      <FormControl fullWidth sx={{mb: 2}}>
-        <InputLabel>Sort Direction</InputLabel>
-        <Select
-          value={metric.sort?.direction || ''}
-          label="Sort Direction"
-          onChange={e => {
-            const direction = e.target.value as Direction | 'desc';
-            onUpdate({
-              ...metric,
-              sort: direction
-                ? {...(metric.sort || {precedence: 0}), direction}
-                : undefined,
-            });
-          }}>
-          <MenuItem value="">No sort</MenuItem>
-          <MenuItem value="asc">Ascending</MenuItem>
-          <MenuItem value="desc">Descending</MenuItem>
-        </Select>
-      </FormControl>
-      {metric.sort && (
-        <TextField
-          fullWidth
-          label="Sort Precedence"
-          type="number"
-          value={metric.sort.precedence === 0 ? '' : metric.sort.precedence}
-          onChange={e =>
-            onUpdate({
-              ...metric,
-              sort: {
-                ...metric.sort!,
-                precedence: parseInt(e.target.value, 10) || 0,
-              },
-            })
-          }
-        />
-      )}
-    </Box>
-  );
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
deleted file mode 100644
index eff5ea4c943..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ModelConfig.tsx
+++ /dev/null
@@ -1,74 +0,0 @@
-import DeleteIcon from '@mui/icons-material/Delete';
-import {
-  Box,
-  FormControl,
-  IconButton,
-  InputLabel,
-  MenuItem,
-  Select,
-  Stack,
-  Typography,
-} from '@mui/material';
-import React from 'react';
-
-import {
-  useModelNames,
-  useModelVersionsForModelName,
-} from './leaderboardConfigQuery';
-import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
-
-export const ModelConfig: React.FC<{
-  model: LeaderboardConfigType['config']['models'][0];
-  onUpdate: (
-    updatedModel: LeaderboardConfigType['config']['models'][0]
-  ) => void;
-  onRemove: () => void;
-}> = ({model, onUpdate, onRemove}) => {
-  const modelNames = useModelNames();
-  const modelVersions = useModelVersionsForModelName(model.name);
-
-  return (
-    <Box sx={{mb: 2, p: 2, border: '1px solid #ccc', borderRadius: 1}}>
-      <Stack
-        direction="row"
-        justifyContent="space-between"
-        alignItems="center"
-        sx={{mb: 2}}>
-        <Typography variant="subtitle1">Model</Typography>
-        <IconButton onClick={onRemove} size="small">
-          <DeleteIcon />
-        </IconButton>
-      </Stack>
-      <FormControl fullWidth sx={{mb: 2}}>
-        <InputLabel>Model Name</InputLabel>
-        <Select
-          value={model.name}
-          label="Model Name"
-          onChange={e => onUpdate({...model, name: e.target.value})}>
-          {modelNames.map(name => (
-            <MenuItem key={name} value={name}>
-              {name}
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-      <FormControl fullWidth>
-        <InputLabel>Model Version</InputLabel>
-        <Select
-          value={model.version}
-          label="Model Version"
-          onChange={e =>
-            onUpdate({...model, version: e.target.value as VersionSpec})
-          }>
-          <MenuItem value="latest">Latest</MenuItem>
-          <MenuItem value="all">All</MenuItem>
-          {modelVersions.map(({version, versionIndex}) => (
-            <MenuItem key={version} value={version}>
-              Version {versionIndex}
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-    </Box>
-  );
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
deleted file mode 100644
index d96e27c1c2f..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/ScoreConfig.tsx
+++ /dev/null
@@ -1,154 +0,0 @@
-import AddIcon from '@mui/icons-material/Add';
-import DeleteIcon from '@mui/icons-material/Delete';
-import {
-  Box,
-  Button,
-  FormControl,
-  IconButton,
-  InputLabel,
-  MenuItem,
-  Select,
-  Stack,
-  Typography,
-} from '@mui/material';
-import React from 'react';
-
-import {
-  useScorerNamesForDataset,
-  useScorerVersionsForDatasetAndScorer,
-} from './leaderboardConfigQuery';
-import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
-import {MetricConfig} from './MetricConfig';
-
-export const ScoreConfig: React.FC<{
-  entity: string;
-  project: string;
-  score: LeaderboardConfigType['config']['columns'][0]['scores'][0];
-  datasetName: string;
-  datasetVersion: string;
-  onUpdate: (
-    updatedScore: LeaderboardConfigType['config']['columns'][0]['scores'][0]
-  ) => void;
-  onRemove: () => void;
-}> = ({
-  entity,
-  project,
-  score,
-  datasetName,
-  datasetVersion,
-  onUpdate,
-  onRemove,
-}) => {
-  const scorerNames = useScorerNamesForDataset(
-    entity,
-    project,
-    datasetName,
-    datasetVersion
-  );
-  const scorerVersions = useScorerVersionsForDatasetAndScorer(
-    entity,
-    project,
-    datasetName,
-    datasetVersion,
-    score.scorer.name
-  );
-
-  const handleAddMetric = () => {
-    onUpdate({
-      ...score,
-      metrics: [
-        ...score.metrics,
-        {displayName: '', path: [], minTrials: undefined, maxTrials: undefined},
-      ],
-    });
-  };
-
-  const handleRemoveMetric = (index: number) => {
-    onUpdate({
-      ...score,
-      metrics: score.metrics.filter((_, i) => i !== index),
-    });
-  };
-
-  const handleUpdateMetric = (
-    index: number,
-    updatedMetric: (typeof score.metrics)[0]
-  ) => {
-    onUpdate({
-      ...score,
-      metrics: score.metrics.map((metric, i) =>
-        i === index ? updatedMetric : metric
-      ),
-    });
-  };
-
-  return (
-    <Box sx={{mb: 2, p: 2, border: '1px solid #eee', borderRadius: 1}}>
-      <Stack
-        direction="row"
-        justifyContent="space-between"
-        alignItems="center"
-        sx={{mb: 2}}>
-        <Typography variant="subtitle2">Score</Typography>
-        <IconButton onClick={onRemove} size="small">
-          <DeleteIcon />
-        </IconButton>
-      </Stack>
-      <FormControl fullWidth sx={{mb: 2}}>
-        <InputLabel>Scorer</InputLabel>
-        <Select
-          value={score.scorer.name}
-          label="Scorer"
-          onChange={e =>
-            onUpdate({
-              ...score,
-              scorer: {...score.scorer, name: e.target.value},
-            })
-          }>
-          {scorerNames.map(name => (
-            <MenuItem key={name} value={name}>
-              {name}
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-      <FormControl fullWidth sx={{mb: 2}}>
-        <InputLabel>Scorer Version</InputLabel>
-        <Select
-          value={score.scorer.version}
-          label="Scorer Version"
-          onChange={e =>
-            onUpdate({
-              ...score,
-              scorer: {...score.scorer, version: e.target.value as VersionSpec},
-            })
-          }>
-          <MenuItem value="latest">Latest</MenuItem>
-          <MenuItem value="all">All</MenuItem>
-          {scorerVersions.map(({version, versionIndex}) => (
-            <MenuItem key={version} value={version}>
-              Version {versionIndex}: {version}
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-      {score.metrics.map((metric, index) => (
-        <MetricConfig
-          key={index}
-          entity={entity}
-          project={project}
-          metric={metric}
-          datasetName={datasetName}
-          datasetVersion={datasetVersion}
-          scorerName={score.scorer.name}
-          scorerVersion={score.scorer.version}
-          onUpdate={updatedMetric => handleUpdateMetric(index, updatedMetric)}
-          onRemove={() => handleRemoveMetric(index)}
-        />
-      ))}
-      <Button startIcon={<AddIcon />} onClick={handleAddMetric} sx={{mt: 1}}>
-        Add Metric
-      </Button>
-    </Box>
-  );
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
index 87c51f5c0c1..7ff48cfcca8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
@@ -1,263 +1,38 @@
-import {ObjectRef} from '@wandb/weave/react';
-import {useEffect, useMemo, useState} from 'react';
+import {useEffect, useState} from 'react';
 
-import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
-import {parseRefMaybe} from '../../../Browse2/SmallRef';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
-import {useEvalCallsForConfig} from './leaderboardConfigQuery';
-import {LeaderboardConfigType} from './LeaderboardConfigType';
+import {FilterAndGroupSpec} from './LeaderboardConfigType';
 import {
   getLeaderboardData,
-  GroupedLeaderboardData2,
+  GroupedLeaderboardData,
 } from './leaderboardServerInterface';
 
-export type LeaderboardData = {
-  metrics: {
-    [metricId: string]: {
-      // evaluationName: string;
-      datasetName: string;
-      datasetVersion: string;
-      scorerName: string;
-      scorerVersion: string;
-      metricPath: string;
-    };
-  };
-  models: string[];
-  scores: {
-    [modelId: string]: {
-      [metricId: string]: {value: number; sourceEvalCallId: string};
-    };
-  };
+type LeaderboardDataState = {
+  loading: boolean;
+  data: GroupedLeaderboardData;
 };
-
 export const useLeaderboardData = (
   entity: string,
   project: string,
-  config: LeaderboardConfigType
-): {loading: boolean; data: LeaderboardData} => {
-  // console.log('Fetching leaderboard data', config);
-  // const {useRootObjectVersions, useCalls} = useWFHooks();
-
-  // // Get the last 100 (latest) evaluation versions
-  // const evaluationVersions = useRootObjectVersions(
-  //   entity,
-  //   project,
-  //   {
-  //     baseObjectClasses: ['Evaluation'],
-  //     // latestOnly: true,
-  //   },
-  //   100,
-  //   true
-  // );
-
-  // // Unfortunately, the eval framework does not build models automatically!!
-  // // const modelVersions = useRootObjectVersions(entity, project, {
-  // //     baseObjectClasses: ['Model'],
-  // //     latestOnly: true,
-  // // }, 100, true)
-
-  // // Get the runs for these evaluation versions.
-  // const evaluationVersionsResult = evaluationVersions?.result;
-  // const evaluationRuns = useCalls(
-  //   entity,
-  //   project,
-  //   {
-  //     opVersionRefs: [
-  //       opVersionKeyToRefUri({
-  //         entity,
-  //         project,
-  //         opId: EVALUATE_OP_NAME_POST_PYDANTIC,
-  //         versionHash: '*',
-  //       }),
-  //     ],
-  //     traceRootsOnly: true,
-  //     inputObjectVersionRefs: (evaluationVersionsResult ?? []).map(version =>
-  //       objectVersionKeyToRefUri(version)
-  //     ),
-  //   },
-  //   100,
-  //   undefined,
-  //   undefined,
-  //   undefined,
-  //   undefined,
-  //   undefined,
-  //   {skip: !evaluationVersionsResult}
-  // );
-  const {calls: evaluationRuns, evals: evaluationVersions} =
-    useEvalCallsForConfig(entity, project, config);
-
-  // Build the dataset
-
-  const results: {loading: boolean; data: LeaderboardData} = useMemo(() => {
-    const finalData: LeaderboardData = {
-      metrics: {},
-      models: [],
-      scores: {},
-    };
-    if (evaluationRuns.loading) {
-      return {
-        loading: true,
-        data: finalData,
-      };
-    } else if (!evaluationRuns.result) {
-      // || !evaluationVersionsResult) {
-      return {
-        loading: false,
-        data: finalData,
-      };
-    }
-
-    const runs = evaluationRuns.result;
-    // const versions = evaluationVersionsResult
-    runs.forEach(r => {
-      const modelName = r.traceCall?.inputs.model;
-      if (!modelName) {
-        return;
-      }
-      const evaluationVersion = r.traceCall?.inputs.self;
-      if (!evaluationVersion) {
-        return;
-      }
-      const evalVersion = parseRefMaybe(evaluationVersion)?.artifactVersion;
-      if (!evalVersion) {
-        return;
-      }
-      const evalObject = evaluationVersions.find(
-        e => e.versionHash === evalVersion
-      );
-      if (!evalObject) {
-        return;
-      }
-      const outputSummary = r.traceCall?.output;
-      if (!outputSummary) {
-        return;
-      }
-
-      const datasetRef = parseRefMaybe(evalObject.val.dataset);
-      if (!datasetRef) {
-        return;
-      }
-
-      const datasetName = datasetRef.artifactName;
-      const datasetVersion = datasetRef.artifactVersion;
-      const scorerNameMap = Object.fromEntries(
-        (evalObject.val.scorers ?? [])
-          .map(parseRefMaybe)
-          .filter(Boolean)
-          .map((s: ObjectRef) => [s.artifactName, s.artifactVersion])
-          .filter(([name, version]: [string, string]) => {
-            return (
-              config.config.datasets.length === 0 ||
-              config.config.datasets.some(dc => {
-                return (
-                  (dc.dataset.name === '' || dc.dataset.name === datasetName) &&
-                  (dc.dataset.version === 'all' ||
-                    dc.dataset.version === 'latest' ||
-                    dc.dataset.version === datasetVersion) &&
-                  (dc.scores.length === 0 ||
-                    dc.scores.some(s => {
-                      return (
-                        (s.scorer.name === '' || s.scorer.name === name) &&
-                        (s.scorer.version === 'all' ||
-                          s.scorer.version === 'latest' ||
-                          s.scorer.version === version)
-                      );
-                    }))
-                );
-              })
-            );
-          })
-      );
-
-      if (!finalData.models.includes(modelName)) {
-        finalData.models.push(modelName);
-        if (!finalData.scores[modelName]) {
-          finalData.scores[modelName] = {};
-        }
-      }
-
-      Object.entries(outputSummary ?? {}).forEach(
-        ([scorerName, scorerMetricsVal]) => {
-          const scorerVersion = scorerNameMap[scorerName];
-          if (!scorerVersion) {
-            return;
-          }
-          if (
-            scorerMetricsVal == null ||
-            typeof scorerMetricsVal !== 'object'
-          ) {
-            return;
-          }
-          const flattened = flattenObjectPreservingWeaveTypes(scorerMetricsVal);
-          Object.entries(flattened).forEach(([metricPath, metricValue]) => {
-            const metricId = `${datasetName}:${datasetVersion}.${scorerName}:${scorerVersion}.${metricPath}`;
-            if (!finalData.metrics[metricId]) {
-              finalData.metrics[metricId] = {
-                datasetName,
-                datasetVersion,
-                scorerName,
-                scorerVersion,
-                metricPath,
-              };
-            }
-            finalData.scores[modelName][metricId] = {
-              value: metricValue,
-              sourceEvalCallId: r.callId,
-            };
-          });
-        }
-      );
-    });
-
-    return {
-      loading: false,
-      data: finalData,
-    };
-  }, [
-    config.config.datasets,
-    evaluationRuns.loading,
-    evaluationRuns.result,
-    evaluationVersions,
-  ]);
-
-  return results;
-};
-
-export const useLeaderboardData2 = (entity: string, project: string) => {
+  spec: FilterAndGroupSpec
+): LeaderboardDataState => {
   const getTraceServerClient = useGetTraceServerClientContext();
-  const [state, setState] = useState<{
-    loading: boolean;
-    data: GroupedLeaderboardData2;
-  }>({loading: true, data: []});
+  const [state, setState] = useState<LeaderboardDataState>({
+    loading: true,
+    data: {modelGroups: {}},
+  });
   useEffect(() => {
     let mounted = true;
-    getLeaderboardData(getTraceServerClient(), entity, project, {
-      datasets: [
-        {
-          name: 'SWEBenchVerified-shuffle808-50',
-          version: '*',
-        },
-      ],
-      models: [
-        {
-          name: '*',
-          version: '*',
-          splitByVersion: false,
-        },
-        {
-          name: 'reproProblem',
-          version: '*',
-          splitByVersion: true,
-        },
-      ],
-    }).then(data => {
-      if (mounted) {
-        setState({loading: false, data});
+    getLeaderboardData(getTraceServerClient(), entity, project, spec).then(
+      data => {
+        if (mounted) {
+          setState({loading: false, data});
+        }
       }
-    });
+    );
     return () => {
       mounted = false;
     };
-  }, [entity, project, getTraceServerClient]);
+  }, [entity, project, getTraceServerClient, spec]);
   return state;
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
deleted file mode 100644
index bf918e31008..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigQuery.ts
+++ /dev/null
@@ -1,460 +0,0 @@
-import {useMemo} from 'react';
-
-import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
-import {parseRefMaybe} from '../../../Browse2/SmallRef';
-import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
-import {useWFHooks} from '../wfReactInterface/context';
-import {
-  objectVersionKeyToRefUri,
-  opVersionKeyToRefUri,
-} from '../wfReactInterface/utilities';
-import {
-  CallSchema,
-  ObjectVersionSchema,
-} from '../wfReactInterface/wfDataModelHooksInterface';
-import {LeaderboardConfigType, VersionSpec} from './LeaderboardConfigType';
-
-export const useCurrentLeaderboardConfig = (): LeaderboardConfigType => {
-  // TODO: Implement this
-  // console.log('Fetching current leaderboard config');
-  return useMemo(() => {
-    return {
-      version: 1,
-      config: {
-        description: '',
-        columns: [],
-        models: [],
-      },
-    };
-  }, []);
-};
-export const persistLeaderboardConfig = (config: LeaderboardConfigType) => {
-  // TODO: Implement this
-  // console.log('Persisting leaderboard config:', config);
-};
-
-export const useDatasetNames = (entity: string, project: string): string[] => {
-  const {useRootObjectVersions} = useWFHooks();
-  const evalQuery = useRootObjectVersions(
-    entity,
-    project,
-    {
-      baseObjectClasses: ['Evaluation'],
-    },
-    // This 100 is very limited
-    100
-  );
-
-  return useMemo(() => {
-    const datasets = (evalQuery.result ?? [])
-      .map(e => parseRefMaybe(e.val.dataset)?.artifactName)
-      .filter(name => !!name)
-      .sort()
-      .filter((name, index, self) => self.indexOf(name) === index) as string[];
-    return datasets;
-  }, [evalQuery.result]);
-};
-export const useDatasetVersionsForDatasetName = (
-  entity: string,
-  project: string,
-  datasetName: string
-): Array<{version: string; versionIndex: number}> => {
-  const {useRootObjectVersions} = useWFHooks();
-  const query = useRootObjectVersions(
-    entity,
-    project,
-    {
-      baseObjectClasses: ['Dataset'],
-      objectIds: [datasetName],
-    },
-    100,
-    true
-  );
-
-  const allVersions = useMemo(() => {
-    return (
-      (query.result ?? []).map(obj => ({
-        version: obj.versionHash,
-        versionIndex: obj.versionIndex,
-      })) ?? []
-    );
-  }, [query]);
-
-  const evalQuery = useRootObjectVersions(
-    entity,
-    project,
-    {
-      baseObjectClasses: ['Evaluation'],
-    },
-    // This 100 is very limited
-    100,
-    false,
-    {skip: allVersions.length === 0}
-  );
-
-  return useMemo(() => {
-    const datasets = (evalQuery.result ?? [])
-      .map(e => {
-        const ref = parseRefMaybe(e.val.dataset);
-        if (!ref) {
-          return null;
-        }
-        if (ref.artifactName !== datasetName) {
-          return null;
-        }
-        const match = allVersions.find(v => v.version === ref.artifactVersion);
-        return match;
-      })
-      .filter(version => !!version) as Array<{
-      version: string;
-      versionIndex: number;
-    }>;
-    return datasets;
-  }, [allVersions, datasetName, evalQuery.result]);
-};
-export const useScorerNamesForDataset = (
-  entity: string,
-  project: string,
-  datasetName: string,
-  datasetVersion: VersionSpec
-): string[] => {
-  // This one is a bit more involved:
-  // 1. Lookup all the evaluations that contain this dataset
-  // 2. Of each evaluation, get the scorer names
-
-  const {useRootObjectVersions} = useWFHooks();
-  const evalQuery = useRootObjectVersions(
-    entity,
-    project,
-    {
-      baseObjectClasses: ['Evaluation'],
-    },
-    // This 100 is very limited
-    100
-  );
-
-  return useMemo(() => {
-    const evalResults =
-      evalQuery.result?.filter(obj => {
-        const ref = parseRefMaybe(obj.val.dataset ?? '');
-        if (!ref) {
-          return false;
-        }
-        if (ref.artifactName !== datasetName) {
-          return false;
-        }
-        if (datasetVersion === 'latest' || datasetVersion === 'all') {
-          return true;
-        }
-        return ref.artifactVersion === datasetVersion;
-      }) ?? [];
-
-    const res = evalResults
-      .map(obj => obj.val.scorers ?? [])
-      .flat()
-      .map(scorer => parseRefMaybe(scorer)?.artifactName)
-      .filter(name => !!name)
-      .sort() as string[]; // .filter((name, index, self) => self.indexOf(name) === index) as string[];
-
-    return res;
-  }, [datasetName, datasetVersion, evalQuery.result]);
-};
-export const useScorerVersionsForDatasetAndScorer = (
-  entity: string,
-  project: string,
-  datasetName: string,
-  datasetVersion: string,
-  scorerName: string
-): Array<{version: string; versionIndex: number}> => {
-  const {useRootObjectVersions, useOpVersions} = useWFHooks();
-  const query1 = useRootObjectVersions(
-    entity,
-    project,
-    {
-      objectIds: [scorerName],
-    },
-    100,
-    true
-  );
-
-  const query2 = useOpVersions(
-    entity,
-    project,
-    {
-      opIds: [scorerName],
-    },
-    100,
-    true
-  );
-
-  const allVersions = useMemo(() => {
-    const versions1 =
-      (query1.result ?? []).map(obj => ({
-        version: obj.versionHash,
-        versionIndex: obj.versionIndex,
-      })) ?? [];
-    const versions2 =
-      (query2.result ?? []).map(obj => ({
-        version: obj.versionHash,
-        versionIndex: obj.versionIndex,
-      })) ?? [];
-    return versions1.concat(versions2);
-  }, [query1, query2]);
-
-  const evalQuery = useRootObjectVersions(
-    entity,
-    project,
-    {
-      baseObjectClasses: ['Evaluation'],
-    },
-    // This 100 is very limited
-    100,
-    false,
-    {skip: allVersions.length === 0}
-  );
-
-  // console.log(allVersions);
-
-  return useMemo(() => {
-    const scorers = (evalQuery.result ?? [])
-      .map(e => {
-        const ref = parseRefMaybe(e.val.dataset);
-        if (!ref) {
-          return [];
-        }
-        if (
-          ref.artifactName !== datasetName ||
-          ref.artifactVersion !== datasetVersion
-        ) {
-          return [];
-        }
-        return (e.val.scorers ?? []).map((scorer: string) => {
-          const sRef = parseRefMaybe(scorer);
-          if (!sRef) {
-            return null;
-          }
-          const match = allVersions.find(
-            v => v.version === sRef.artifactVersion
-          );
-          return match;
-        });
-      })
-      .flat()
-      .filter(version => !!version);
-    return scorers;
-  }, [allVersions, datasetName, datasetVersion, evalQuery.result]);
-};
-
-export const useMetricPathsForDatasetAndScorer = (
-  entity: string,
-  project: string,
-  datasetName: string,
-  datasetVersion: string,
-  scorerName: string,
-  scorerVersion: string
-): string[] => {
-  const {useRootObjectVersions, useCalls} = useWFHooks();
-  const evalQuery = useRootObjectVersions(
-    entity,
-    project,
-    {
-      baseObjectClasses: ['Evaluation'],
-    },
-    // This 100 is very limited
-    100
-  );
-
-  const evals = useMemo(() => {
-    // Find the matching evals:
-    // console.log(evalQuery.result, scorerName, scorerVersion);
-    return (evalQuery.result ?? []).filter(e => {
-      const match = (e.val.scorers ?? []).find((s: string) => {
-        const sRef = parseRefMaybe(s);
-        if (!sRef) {
-          return false;
-        }
-        // console.log(sRef);
-        return (
-          sRef.artifactName === scorerName &&
-          sRef.artifactVersion === scorerVersion
-        );
-      });
-      return !!match;
-    });
-  }, [evalQuery.result, scorerName, scorerVersion]);
-
-  const evalCalls = useCalls(
-    entity,
-    project,
-    {
-      inputObjectVersionRefs: evals.map(objectVersionKeyToRefUri),
-    },
-    10,
-    undefined,
-    undefined,
-    undefined,
-    undefined,
-    undefined,
-    {
-      skip: evals.length === 0,
-    }
-  );
-
-  return useMemo(() => {
-    const foundCall = (evalCalls.result ?? []).find(
-      (call: CallSchema) =>
-        call.traceCall?.exception == null &&
-        call.traceCall?.ended_at != null &&
-        (call?.traceCall?.output as any)?.[scorerName] != null
-    );
-    // console.log(foundCall, evalCalls);
-    const output = foundCall?.traceCall?.output;
-    if (!output) {
-      return [];
-    }
-    if (typeof output !== 'object') {
-      return [];
-    }
-    const res = Object.keys(
-      flattenObjectPreservingWeaveTypes(
-        (output as {[key: string]: any})[scorerName] ?? {}
-      )
-    );
-    console.log(foundCall, scorerName, res);
-    return res;
-  }, [evalCalls, scorerName]);
-};
-
-export const useModelNames = (): string[] => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return ['model-1', 'model-2', 'model-3'];
-  }, []);
-};
-
-export const useModelVersionsForModelName = (
-  modelName: string
-): Array<{version: string; versionIndex: number}> => {
-  // TODO: Implement this
-  return useMemo(() => {
-    return [
-      {version: 'mug657ioy8j1', versionIndex: 0},
-      {version: 'mnkjubyhvasd', versionIndex: 1},
-      {version: 'madsgf3f451d', versionIndex: 2},
-    ];
-  }, []);
-};
-
-export const useEvalObjsForConfig = (
-  entity: string,
-  project: string,
-  config: LeaderboardConfigType
-) => {
-  const {useRootObjectVersions} = useWFHooks();
-  // # TODO: handle latests
-  const evalQuery = useRootObjectVersions(
-    entity,
-    project,
-    {
-      baseObjectClasses: ['Evaluation'],
-    },
-    // This 100 is very limited
-    100
-  );
-  return useMemo(() => {
-    if (!evalQuery.result) {
-      return [];
-    }
-    const finalEvals: ObjectVersionSchema[] = [];
-    evalQuery.result.forEach(evaluation => {
-      const datasetRef = parseRefMaybe(evaluation.val.dataset);
-      if (!datasetRef) {
-        return;
-      }
-      if (config.config.datasets.length === 0) {
-        finalEvals.push(evaluation);
-        return;
-      }
-      const datasetName = datasetRef.artifactName;
-      const datasetVersion = datasetRef.artifactVersion;
-      // Determine this dataset matches any of the config's datasets
-      let matched = false;
-      for (const column of config.config.datasets) {
-        if (
-          (datasetName === column.dataset.name || column.dataset.name === '') &&
-          (column.dataset.version === datasetVersion ||
-            column.dataset.version === 'latest' ||
-            column.dataset.version === 'all')
-        ) {
-          if (column.scores.length === 0) {
-            finalEvals.push(evaluation);
-            break;
-          }
-          for (const scorer of evaluation.val.scorers ?? []) {
-            const sRef = parseRefMaybe(scorer);
-            if (!sRef) {
-              continue;
-            }
-            const scorerName = sRef.artifactName;
-            const scorerVersion = sRef.artifactVersion;
-            for (const score of column.scores) {
-              if (
-                (scorerName === score.scorer.name ||
-                  score.scorer.name === '') &&
-                (scorerVersion === score.scorer.version ||
-                  score.scorer.version === 'latest' ||
-                  score.scorer.version === 'all')
-              ) {
-                finalEvals.push(evaluation);
-                matched = true;
-                break;
-              }
-            }
-          }
-          if (matched) {
-            break;
-          }
-        }
-      }
-    });
-
-    return finalEvals;
-  }, [config.config.datasets, evalQuery.result]);
-};
-
-export const useEvalCallsForConfig = (
-  entity: string,
-  project: string,
-  config: LeaderboardConfigType
-) => {
-  const {useCalls} = useWFHooks();
-  // Step 1: Get the qualifying evaluation objects
-  const evals = useEvalObjsForConfig(entity, project, config);
-  // Step 2: Get the calls for each evaluation object
-  const calls = useCalls(
-    entity,
-    project,
-    {
-      opVersionRefs: [
-        opVersionKeyToRefUri({
-          entity,
-          project,
-          opId: EVALUATE_OP_NAME_POST_PYDANTIC,
-          versionHash: '*',
-        }),
-      ],
-      traceRootsOnly: true,
-      inputObjectVersionRefs: evals.map(objectVersionKeyToRefUri),
-    },
-    1000,
-    undefined,
-    undefined,
-    undefined,
-    undefined,
-    undefined,
-    {
-      skip: evals.length === 0,
-    }
-  );
-  return {calls, evals};
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 18fcd88a571..83f40b77eaa 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -11,6 +11,7 @@ import {
   projectIdFromParts,
 } from '../wfReactInterface/tsDataModelHooks';
 import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
+import {FilterAndGroupSpec} from './LeaderboardConfigType';
 
 type LeaderboardValueRecord = {
   datasetName: string;
@@ -34,38 +35,31 @@ type LeaderboardValueRecord = {
   sourceEvaluationObjectRef: string;
 };
 
-type GroupableLeaderboardValueRecord = {
+export type GroupableLeaderboardValueRecord = {
+  modelGroup: string;
   datasetGroup: string;
   scorerGroup: string;
   metricPathGroup: string;
-  modelGroup: string;
   sortKey: number;
   row: LeaderboardValueRecord;
 };
 
-export type GroupedLeaderboardData2 = GroupableLeaderboardValueRecord[];
-export type LeaderboardData2 = LeaderboardValueRecord[];
-
-export type FilterAndGroupSpec = {
-  datasets?: Array<{
-    name: string; // "*" means all
-    version: string; // "*" means all
-    splitByVersion?: boolean;
-    scorers?: Array<{
-      name: string; // "*" means all
-      version: string; // "*" means all
-      splitByVersion?: boolean;
-      metrics?: Array<{
-        path: string; // "*" means all
-        shouldMinimize?: boolean;
-      }>; // null is all
-    }>; // null is all
-  }>; // null is all
-  models?: Array<{
-    name: string; // "*" means all
-    version: string; // "*" means all
-    splitByVersion?: boolean;
-  }>; // null is all
+export type GroupedLeaderboardData = {
+  modelGroups: {
+    [modelGroup: string]: {
+      datasetGroups: {
+        [datasetGroup: string]: {
+          scorerGroups: {
+            [scorerGroup: string]: {
+              metricPathGroups: {
+                [metricPathGroup: string]: LeaderboardValueRecord;
+              };
+            };
+          };
+        };
+      };
+    };
+  };
 };
 
 export const getLeaderboardData = async (
@@ -73,7 +67,7 @@ export const getLeaderboardData = async (
   entity: string,
   project: string,
   spec: FilterAndGroupSpec = {}
-): Promise<GroupedLeaderboardData2> => {
+): Promise<GroupedLeaderboardData> => {
   // get all the evaluations
   const allEvaluationObjectsProm = client.objsQuery({
     project_id: projectIdFromParts({entity, project}),
@@ -119,7 +113,7 @@ export const getLeaderboardData = async (
   });
 
   const allEvaluationCallsRes = await allEvaluationCallsProm;
-  const data: LeaderboardData2 = [];
+  const data: LeaderboardValueRecord[] = [];
   allEvaluationCallsRes.calls.forEach(call => {
     const evalObjectRefUri = call.inputs.self;
     const evalObjectRef = parseRefMaybe(evalObjectRefUri ?? '');
@@ -343,30 +337,37 @@ export const getLeaderboardData = async (
     .filter(entry => entry.include)
     .map(entry => entry.groupableRow);
 
-  const finalData: GroupedLeaderboardData2 = [];
-  const groupedVisitor = (
-    recordList: GroupableLeaderboardValueRecord[],
-    fields: string[]
-  ): any => {
-    if (fields.length === 0) {
-      // Sort by created at descending and return the most recent record
-      // Would be better to use some form of latest.
-      const res = recordList.sort((a, b) => a.sortKey - b.sortKey)[0];
-      finalData.push(res);
-    }
-
-    const [currentField, ...remainingFields] = fields;
-    return _.values(_.groupBy(recordList, currentField)).forEach(groupedRecords =>
-      groupedVisitor(groupedRecords, remainingFields)
-    );
+  const finalData: GroupedLeaderboardData = {
+    modelGroups: _.mapValues(
+      _.groupBy(groupableData, 'modelGroup'),
+      modelGroup => {
+        return {
+          datasetGroups: _.mapValues(
+            _.groupBy(modelGroup, 'datasetGroup'),
+            datasetGroup => {
+              return {
+                scorerGroups: _.mapValues(
+                  _.groupBy(datasetGroup, 'scorerGroup'),
+                  scorerGroup => {
+                    return {
+                      metricPathGroups: _.mapValues(
+                        _.groupBy(scorerGroup, 'metricPathGroup'),
+                        metricPathGroup => {
+                          return metricPathGroup.sort(
+                            (a, b) => a.sortKey - b.sortKey
+                          )[0].row;
+                        }
+                      ),
+                    };
+                  }
+                ),
+              };
+            }
+          ),
+        };
+      }
+    ),
   };
 
-  groupedVisitor(groupableData, [
-    'datasetGroup',
-    'scorerGroup',
-    'metricPathGroup',
-    'modelGroup',
-  ]);
-
   return finalData;
 };

From 88690865ee6cee0f155f0fb4322aa951faaf12d0 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 01:42:18 -0700
Subject: [PATCH 039/169] Massive refactor step 2

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx | 700 ++++++++++++------
 .../leaderboardServerInterface.ts             |  30 +-
 2 files changed, 481 insertions(+), 249 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 8717dd1a215..2399da14570 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -4,19 +4,21 @@ import {
   GridColumnGroup,
   GridColumnGroupingModel,
   GridColumnNode,
-  GridPaginationModel,
+  GridLeafColumn,
   GridRenderCellParams,
   GridSortDirection,
 } from '@mui/x-data-grid-pro';
-import React, {useCallback, useMemo, useState} from 'react';
+import _ from 'lodash';
+import React, {useCallback, useMemo} from 'react';
 
+import {NotApplicable} from '../../../Browse2/NotApplicable';
 import {parseRefMaybe, SmallRef} from '../../../Browse2/SmallRef';
 import {StyledDataGrid} from '../../StyledDataGrid';
 import {PaginationButtons} from '../CallsPage/CallsTableButtons';
-import {buildTree} from '../common/tabularListViews/buildTree';
 import {
-  GroupableLeaderboardValueRecord,
   GroupedLeaderboardData,
+  GroupedLeaderboardModelGroup,
+  LeaderboardValueRecord,
 } from './leaderboardServerInterface';
 
 interface LeaderboardGridProps {
@@ -28,9 +30,10 @@ interface LeaderboardGridProps {
 }
 
 type RowData = {
-  id: number;
-  model: string;
-} & GroupableLeaderboardValueRecord;
+  id: string;
+  modelGroupName: string;
+  modelGroup: GroupedLeaderboardModelGroup;
+};
 
 export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   entity,
@@ -39,241 +42,297 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   loading,
   onCellClick,
 }) => {
-  return <>Grid</>;
-  // const [paginationModel, setPaginationModel] = useState<GridPaginationModel>({
-  //   pageSize: 50,
-  //   page: 0,
-  // });
+  const columnStats = useMemo(() => getColumnStats(data), [data]);
+
+  const getColorForScore = useCallback(
+    (datasetGroup, scorerGroup, metricPathGroup, score) => {
+      if (score == null) {
+        return 'transparent';
+      }
+      const {min, max} =
+        columnStats.datasetGroups[datasetGroup].scorerGroups[scorerGroup]
+          .metricPathGroups[metricPathGroup];
+      const normalizedScore = (score - min) / (max - min);
+      return `hsl(${120 * normalizedScore}, 70%, 85%)`;
+    },
+    [columnStats.datasetGroups]
+  );
+
+  const rows: RowData[] = useMemo(() => {
+    const rowData: RowData[] = [];
+    Object.entries(data.modelGroups).forEach(([modelGroupName, modelGroup]) => {
+      rowData.push({
+        id: modelGroupName,
+        modelGroupName,
+        modelGroup,
+      });
+    });
+    return rowData;
+  }, [data]);
 
-  // const orderedMetrics = useMemo(() => {
-  //   return Object.keys(data.metrics);
-  //   // return Object.keys(data.metrics).sort((a, b) => {
-  //   //   return a.localeCompare(b);
-  //   // });
-  // }, [data.metrics]);
+  const columns: Array<GridColDef<RowData>> = useMemo(
+    () => [
+      {
+        field: 'modelGroupName',
+        headerName: 'Model',
+        minWidth: 150,
+        flex: 1,
+        renderCell: (params: GridRenderCellParams) => {
+          const modelRef = parseRefMaybe(params.value);
+          if (modelRef) {
+            return (
+              <div
+                style={{
+                  width: '100%',
+                  height: '100%',
+                  alignContent: 'center',
+                  display: 'flex',
+                  alignItems: 'center',
+                  justifyContent: 'center',
+                  lineHeight: '20px',
+                }}>
+                <SmallRef objRef={modelRef} />
+              </div>
+            );
+          }
+          return <div>{params.value}</div>;
+        },
+      },
+      ...Object.entries(columnStats.datasetGroups).flatMap(
+        ([datasetGroupName, datasetGroup]) =>
+          Object.entries(datasetGroup.scorerGroups).flatMap(
+            ([scorerGroupName, scorerGroup]) => {
+              return Object.entries(scorerGroup.metricPathGroups).map(
+                ([metricPathGroupName, metricPathGroup]) => {
+                  return {
+                    field: `${datasetGroupName}--${scorerGroupName}--${metricPathGroupName}`,
+                    headerName: `${metricPathGroupName.split('.').pop()}`,
+                    minWidth: 100,
+                    flex: 1,
+                    valueGetter: (value: any, row: RowData) => {
+                      return valueFromRowData(
+                        row,
+                        datasetGroupName,
+                        scorerGroupName,
+                        metricPathGroupName
+                      );
+                    },
+                    getSortComparator: defaultGetSortComparator,
 
-  // const metricRanges = useMemo(() => {
-  //   const ranges: {[key: string]: {min: number; max: number}} = {};
-  //   orderedMetrics.forEach(metric => {
-  //     const scores = data.models
-  //       .map(model => data.scores?.[model]?.[metric]?.value)
-  //       .filter(score => score !== undefined);
-  //     ranges[metric] = {
-  //       min: Math.min(...scores),
-  //       max: Math.max(...scores),
-  //     };
-  //   });
-  //   return ranges;
-  // }, [data.models, data.scores, orderedMetrics]);
+                    renderCell: (params: GridRenderCellParams) => {
+                      const value = valueFromRowData(
+                        params.row,
+                        datasetGroupName,
+                        scorerGroupName,
+                        metricPathGroupName
+                      );
+                      let inner: React.ReactNode = value;
+                      if (inner == null) {
+                        inner = <NotApplicable />;
+                      } else if (typeof inner === 'number') {
+                        if (inner < 1) {
+                          inner = `${(inner * 100).toFixed(2)}%`;
+                        } else {
+                          inner = `${inner.toFixed(2)}`;
+                        }
+                      } else {
+                        inner = JSON.stringify(params.value);
+                      }
+                      return (
+                        <div
+                          style={{
+                            width: '100%',
+                            height: '100%',
+                            display: 'flex',
+                            alignItems: 'center',
+                            justifyContent: 'center',
+                            backgroundColor: getColorForScore(
+                              datasetGroupName,
+                              scorerGroupName,
+                              metricPathGroupName,
+                              value
+                            ),
+                          }}
+                          onClick={() =>
+                            onCellClick(
+                              params.row.model,
+                              params.field as string,
+                              params.value as number
+                            )
+                          }>
+                          {' '}
+                          {inner}
+                        </div>
+                      );
+                    },
+                  };
+                }
+              );
+            }
+          )
+      ),
+    ],
+    [columnStats.datasetGroups, getColorForScore, onCellClick]
+  );
 
-  // const getColorForScore = useCallback(
-  //   (metric: string, score: number | undefined) => {
-  //     if (score === undefined) {
-  //       return 'transparent';
-  //     }
-  //     const {min, max} = metricRanges[metric];
-  //     const normalizedScore = (score - min) / (max - min);
-  //     return `hsl(${120 * normalizedScore}, 70%, 85%)`;
-  //   },
-  //   [metricRanges]
-  // );
+  const groupingModel: GridColumnGroup[] = useMemo(() => {
+    const datasetGroups: GridColumnGroup[] = [];
+    Object.entries(columnStats.datasetGroups).forEach(
+      ([datasetGroupName, datasetGroup]) => {
+        const datasetColGroup: GridColumnGroup = {
+          groupId: datasetGroupName,
+          headerName: datasetGroupName,
+          children: [],
+          renderHeaderGroup: params => {
+            const ref = parseRefMaybe(
+              `weave:///${entity}/${project}/object/${datasetGroupName}` ?? ''
+            );
+            if (ref) {
+              return <SmallRef objRef={ref} />;
+            }
+            return <div>{datasetGroupName}</div>;
+          },
+        };
+        datasetGroups.push(datasetColGroup);
+        Object.entries(datasetGroup.scorerGroups).forEach(
+          ([scorerGroupName, scorerGroup]) => {
+            const scorerColGroup: GridColumnGroup = {
+              groupId: `${datasetGroupName}--${scorerGroupName}`,
+              headerName: scorerGroupName,
+              children: [],
+              renderHeaderGroup: params => {
+                const ref = parseRefMaybe(
+                  `weave:///${entity}/${project}/object/${scorerGroupName}` ??
+                    ''
+                );
+                // console.log(node.headerName, ref);
+                if (ref) {
+                  return <SmallRef objRef={ref} />;
+                }
+                return <div>{scorerGroupName}</div>;
+              },
+            };
+            datasetColGroup.children.push(scorerColGroup);
+            Object.entries(scorerGroup.metricPathGroups).forEach(
+              ([metricPathGroupName, metricPathGroup]) => {
+                // const prefix = `${datasetGroupName}--${scorerGroupName}--`
+                const metricPathParts = metricPathGroupName.split('.');
+                let targetParentGroup = scorerColGroup;
+                for (let i = 0; i < metricPathParts.length - 2; i++) {
+                  const part = metricPathParts[i];
+                  let existingChild = targetParentGroup.children.find(
+                    child => 'groupId' in child && child.groupId === part
+                  );
+                  if (!existingChild) {
+                    existingChild = {
+                      groupId: part,
+                      headerName: part,
+                      children: [],
+                    };
+                    targetParentGroup.children.push(existingChild);
+                  }
+                  targetParentGroup = existingChild as GridColumnGroup;
+                }
+                // const finalPart = metricPathParts[metricPathParts.length - 1];
+                const metricPathColGroup: GridLeafColumn = {
+                  field: `${datasetGroupName}--${scorerGroupName}--${metricPathGroupName}`,
+                };
+                targetParentGroup.children.push(metricPathColGroup);
+              }
+            );
 
-  // const rows: RowData[] = useMemo(
-  //   () =>
-  //     data.models.map((model, index) => ({
-  //       id: index,
-  //       model,
-  //       ...data.scores[model],
-  //     })) as RowData[],
-  //   [data.models, data.scores]
-  // );
+            // scorerColGroup.children = splitCommonRootLeafColumns(scorerColGroup.children as GridLeafColumn[], scorerColGroup.groupId);
+          }
+        );
+      }
+    );
 
-  // const columns: Array<GridColDef<RowData>> = useMemo(
-  //   () => [
-  //     {
-  //       field: 'model',
-  //       headerName: 'Model',
-  //       minWidth: 100,
-  //       flex: 1,
-  //       renderCell: (params: GridRenderCellParams) => {
-  //         const modelRef = parseRefMaybe(params.value);
-  //         if (modelRef) {
-  //           return (
-  //             <div
-  //               style={{
-  //                 width: '100%',
-  //                 height: '100%',
-  //                 alignContent: 'center',
-  //                 display: 'flex',
-  //                 alignItems: 'center',
-  //                 justifyContent: 'center',
-  //                 lineHeight: '20px',
-  //               }}>
-  //               <SmallRef objRef={modelRef} />
-  //             </div>
-  //           );
-  //         }
-  //         return <div>{params.value}</div>;
-  //       },
-  //     },
-  //     ...orderedMetrics.map(metric => ({
-  //       field: metric,
-  //       headerName: data.metrics[metric].metricPath.split('.').pop(),
-  //       minWidth: 100,
-  //       flex: 1,
-  //       valueGetter: (value: RowData) => {
-  //         return value?.value;
-  //       },
-  //       getSortComparator: (dir: GridSortDirection) => (a: any, b: any) => {
-  //         const aValue = a;
-  //         const bValue = b;
-  //         if (aValue == null && bValue == null) {
-  //           return 0;
-  //         }
-  //         // Ignoring direction here allows nulls to always sort to the end
-  //         if (aValue == null) {
-  //           return 1;
-  //         }
-  //         if (bValue == null) {
-  //           return -1;
-  //         }
-  //         if (typeof aValue === 'number' && typeof bValue === 'number') {
-  //           if (dir === 'asc') {
-  //             return aValue - bValue;
-  //           } else {
-  //             return bValue - aValue;
-  //           }
-  //         }
-  //         return aValue.localeCompare(bValue);
-  //       },
-  //       renderCell: (params: GridRenderCellParams) => {
-  //         let inner = params.value;
-  //         if (typeof inner === 'number') {
-  //           if (inner < 1) {
-  //             inner = `${(inner * 100).toFixed(2)}%`;
-  //           } else {
-  //             inner = `${inner.toFixed(2)}`;
-  //           }
-  //         } else {
-  //           inner = JSON.stringify(params.value);
-  //         }
-  //         // const value =
-  //         // <CellValue value={params.value} />
-  //         return (
-  //           <div
-  //             style={{
-  //               width: '100%',
-  //               height: '100%',
-  //               display: 'flex',
-  //               alignItems: 'center',
-  //               justifyContent: 'center',
-  //               backgroundColor: getColorForScore(metric, params.value),
-  //             }}
-  //             onClick={() =>
-  //               onCellClick(
-  //                 params.row.model,
-  //                 params.field as string,
-  //                 params.value as number
-  //               )
-  //             }>
-  //             {' '}
-  //             {inner}
-  //           </div>
-  //         );
-  //       },
-  //     })),
-  //   ],
-  //   [data.metrics, getColorForScore, onCellClick, orderedMetrics]
-  // );
+    let finalGroupingModel: GridColumnGroupingModel = datasetGroups.filter(
+      c => 'groupId' in c
+    ) as GridColumnGroup[];
+    finalGroupingModel = walkGroupingModel(finalGroupingModel, node => {
+      if ('groupId' in node) {
+        if (node.children.length === 1) {
+          if (
+            'groupId' in node.children[0] &&
+            !node.headerName?.includes(':') &&
+            !node.children[0].headerName?.includes(':')
+          ) {
+            const currNode = node;
+            node = node.children[0];
+            node.headerName = currNode.headerName + '.' + node.headerName;
+          } else {
+            // pass
+            // node = node.children[0];
+          }
+        }
+      }
+      return node;
+    }) as GridColumnGroup[];
+    finalGroupingModel = walkGroupingModel(finalGroupingModel, node => {
+      if ('groupId' in node) {
+        node.renderHeaderGroup = params => {
+          const ref = parseRefMaybe(
+            `weave:///${entity}/${project}/object/${node.headerName}` ?? ''
+          );
+          // console.log(node.headerName, ref);
+          if (ref) {
+            return <SmallRef objRef={ref} />;
+          }
+          return <div>{node.headerName}</div>;
+        };
+      }
+      return node;
+    }) as GridColumnGroup[];
 
-  // const tree = buildTree([...Object.keys(data.metrics)]);
-  // let groupingModel: GridColumnGroupingModel = tree.children.filter(
-  //   c => 'groupId' in c
-  // ) as GridColumnGroup[];
-  // groupingModel = walkGroupingModel(groupingModel, node => {
-  //   if ('groupId' in node) {
-  //     if (node.children.length === 1) {
-  //       if (
-  //         'groupId' in node.children[0] &&
-  //         !node.headerName?.includes(':') &&
-  //         !node.children[0].headerName?.includes(':')
-  //       ) {
-  //         const currNode = node;
-  //         node = node.children[0];
-  //         node.headerName = currNode.headerName + '.' + node.headerName;
-  //       } else {
-  //         // pass
-  //         // node = node.children[0];
-  //       }
-  //     }
-  //   }
-  //   return node;
-  // }) as GridColumnGroup[];
-  // groupingModel = walkGroupingModel(groupingModel, node => {
-  //   if ('groupId' in node) {
-  //     node.renderHeaderGroup = params => {
-  //       const ref = parseRefMaybe(
-  //         `weave:///${entity}/${project}/object/${node.headerName}` ?? ''
-  //       );
-  //       // console.log(node.headerName, ref);
-  //       if (ref) {
-  //         return <SmallRef objRef={ref} />;
-  //       }
-  //       return <div>{node.headerName}</div>;
-  //     };
-  //   }
-  //   return node;
-  // }) as GridColumnGroup[];
+    return finalGroupingModel;
+  }, [columnStats.datasetGroups, entity, project]);
 
-  // return (
-  //   <Box
-  //     sx={{
-  //       height: '100%',
-  //       width: '100%',
-  //       display: 'flex',
-  //       flexDirection: 'column',
-  //       overflow: 'hidden', // Prevent outer container from scrolling
-  //     }}>
-  //     <StyledDataGrid
-  //       rows={rows}
-  //       columns={columns}
-  //       columnGroupingModel={groupingModel}
-  //       pagination
-  //       paginationModel={paginationModel}
-  //       onPaginationModelChange={setPaginationModel}
-  //       pageSizeOptions={[50]}
-  //       disableRowSelectionOnClick
-  //       hideFooterSelectedRowCount
-  //       disableMultipleColumnsSorting={false}
-  //       columnHeaderHeight={40}
-  //       loading={loading}
-  //       sx={{
-  //         borderRadius: 0,
-  //         '& .MuiDataGrid-footerContainer': {
-  //           justifyContent: 'flex-start',
-  //         },
-  //         '& .MuiDataGrid-cell': {
-  //           cursor: 'pointer',
-  //         },
-  //         flexGrow: 1,
-  //         width: 'calc(100% + 1px)', // Add 1px to account for the right border
-  //         '& .MuiDataGrid-main': {
-  //           overflow: 'hidden',
-  //         },
-  //         '& .MuiDataGrid-virtualScroller': {
-  //           overflow: 'auto',
-  //         },
-  //         '& .MuiDataGrid-columnHeaders': {
-  //           overflow: 'hidden',
-  //         },
-  //       }}
-  //       slots={{
-  //         // columnMenu: CallsCustomColumnMenu,
-  //         pagination: PaginationButtons,
-  //       }}
-  //     />
-  //   </Box>
-  // );
+  return (
+    <Box
+      sx={{
+        height: '100%',
+        width: '100%',
+        display: 'flex',
+        flexDirection: 'column',
+        overflow: 'hidden', // Prevent outer container from scrolling
+      }}>
+      <StyledDataGrid
+        rows={rows}
+        columns={columns}
+        columnGroupingModel={groupingModel}
+        disableRowSelectionOnClick
+        hideFooterSelectedRowCount
+        disableMultipleColumnsSorting={false}
+        columnHeaderHeight={40}
+        loading={loading}
+        sx={{
+          borderRadius: 0,
+          '& .MuiDataGrid-footerContainer': {
+            justifyContent: 'flex-start',
+          },
+          '& .MuiDataGrid-cell': {
+            cursor: 'pointer',
+          },
+          flexGrow: 1,
+          width: 'calc(100% + 1px)', // Add 1px to account for the right border
+          '& .MuiDataGrid-main': {
+            overflow: 'hidden',
+          },
+          '& .MuiDataGrid-virtualScroller': {
+            overflow: 'auto',
+          },
+          '& .MuiDataGrid-columnHeaders': {
+            overflow: 'hidden',
+          },
+        }}
+        slots={{
+          // columnMenu: CallsCustomColumnMenu,
+          pagination: PaginationButtons,
+        }}
+      />
+    </Box>
+  );
 };
 
 // const walkGroupingModel = (
@@ -287,3 +346,174 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
 //     return fn(node);
 //   });
 // };
+
+type ColumnStats = {
+  datasetGroups: {
+    [datasetGroup: string]: {
+      scorerGroups: {
+        [scorerGroup: string]: {
+          metricPathGroups: {
+            [metricPathGroup: string]: {
+              min: number;
+              max: number;
+            };
+          };
+        };
+      };
+    };
+  };
+};
+const getColumnStats = (data: GroupedLeaderboardData): ColumnStats => {
+  const stats: ColumnStats = {
+    datasetGroups: {},
+  };
+
+  Object.values(data.modelGroups).forEach(modelGroup => {
+    Object.entries(modelGroup.datasetGroups).forEach(
+      ([datasetGroupName, datasetGroup]) => {
+        if (stats.datasetGroups[datasetGroupName] == null) {
+          stats.datasetGroups[datasetGroupName] = {
+            scorerGroups: {},
+          };
+        }
+        const currDatasetGroup = stats.datasetGroups[datasetGroupName];
+
+        Object.entries(datasetGroup.scorerGroups).forEach(
+          ([scorerGroupName, scorerGroup]) => {
+            if (currDatasetGroup.scorerGroups[scorerGroupName] == null) {
+              currDatasetGroup.scorerGroups[scorerGroupName] = {
+                metricPathGroups: {},
+              };
+            }
+            const currScorerGroup =
+              currDatasetGroup.scorerGroups[scorerGroupName];
+            Object.entries(scorerGroup.metricPathGroups).forEach(
+              ([metricPathGroupName, metricPathGroup]) => {
+                const groupMin = _.min(
+                  metricPathGroup.map(g => g.metricValue)
+                ) as number;
+                const groupMax = _.max(
+                  metricPathGroup.map(g => g.metricValue)
+                ) as number;
+                if (
+                  currScorerGroup.metricPathGroups[metricPathGroupName] == null
+                ) {
+                  currScorerGroup.metricPathGroups[metricPathGroupName] = {
+                    min: groupMin,
+                    max: groupMax,
+                  };
+                } else {
+                  currScorerGroup.metricPathGroups[metricPathGroupName].min =
+                    Math.min(
+                      currScorerGroup.metricPathGroups[metricPathGroupName].min,
+                      groupMin
+                    );
+                  currScorerGroup.metricPathGroups[metricPathGroupName].max =
+                    Math.max(
+                      currScorerGroup.metricPathGroups[metricPathGroupName].max,
+                      groupMax
+                    );
+                }
+              }
+            );
+          }
+        );
+      }
+    );
+  });
+
+  return stats;
+};
+
+const valueFromRowData = (
+  rowData: RowData,
+  datasetGroupName: string,
+  scorerGroupName: string,
+  metricPathGroupName: string
+): number | string | boolean | null => {
+  return getAggregatedResults(
+    rowData.modelGroup.datasetGroups[datasetGroupName]?.scorerGroups[
+      scorerGroupName
+    ]?.metricPathGroups[metricPathGroupName] ?? []
+  );
+};
+const getAggregatedResults = (
+  data: null | LeaderboardValueRecord[]
+): number | string | boolean | null => {
+  if (data == null || data.length === 0) {
+    return null;
+  }
+  if (data.length === 1) {
+    return data[0].metricValue;
+  }
+  return data.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())[0]
+    .metricValue;
+};
+
+const defaultGetSortComparator =
+  (dir: GridSortDirection) => (a: any, b: any) => {
+    const aValue = a;
+    const bValue = b;
+    if (aValue == null && bValue == null) {
+      return 0;
+    }
+    // Ignoring direction here allows nulls to always sort to the end
+    if (aValue == null) {
+      return 1;
+    }
+    if (bValue == null) {
+      return -1;
+    }
+    if (typeof aValue === 'number' && typeof bValue === 'number') {
+      if (dir === 'asc') {
+        return aValue - bValue;
+      } else {
+        return bValue - aValue;
+      }
+    }
+    return aValue.localeCompare(bValue);
+  };
+
+const walkGroupingModel = (
+  nodes: GridColumnNode[],
+  fn: (node: GridColumnNode) => GridColumnNode
+) => {
+  return nodes.map(node => {
+    if ('children' in node) {
+      node.children = walkGroupingModel(node.children, fn);
+    }
+    return fn(node);
+  });
+};
+
+// // TODO: refactor to accumulate common root columns in a single pass
+//   const splitCommonRootLeafColumns = (columns: GridLeafColumn[], groupIdPrefix: string, depth: number = 0): GridColumnNode[] => {
+//     if (columns.length < 2) {
+//       return columns;
+//     }
+//     const groups: {[key: string]: GridLeafColumn[]} = {};
+//     columns.forEach(col => {
+//       const key = col.field.split('.')[depth];
+//       if (groups[key] == null) {
+//         groups[key] = [];
+//       }
+//       groups[key].push(col);
+//     });
+
+//     return Object.entries(groups).map(([key, group]) => {
+//       if (group.length === 1) {
+//         return group[0];
+//       }
+//       const newGroupId = `${groupIdPrefix}.${key}`
+
+//       const groupCol: GridColumnGroup = {
+//         groupId: newGroupId,
+//         headerName: key,
+//         renderHeaderGroup: params => {
+//           return <div>{key}</div>
+//         },
+//         children: splitCommonRootLeafColumns(group, newGroupId, depth + 1),
+//       };
+//       return groupCol;
+//     });
+//   }
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 83f40b77eaa..73ed0cc85bf 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -13,7 +13,7 @@ import {
 import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
 import {FilterAndGroupSpec} from './LeaderboardConfigType';
 
-type LeaderboardValueRecord = {
+export type LeaderboardValueRecord = {
   datasetName: string;
   datasetVersion: string;
   metricType:
@@ -46,22 +46,23 @@ export type GroupableLeaderboardValueRecord = {
 
 export type GroupedLeaderboardData = {
   modelGroups: {
-    [modelGroup: string]: {
-      datasetGroups: {
-        [datasetGroup: string]: {
-          scorerGroups: {
-            [scorerGroup: string]: {
-              metricPathGroups: {
-                [metricPathGroup: string]: LeaderboardValueRecord;
-              };
-            };
+    [modelGroup: string]: GroupedLeaderboardModelGroup;
+  };
+};
+
+export type GroupedLeaderboardModelGroup = {
+  datasetGroups: {
+    [datasetGroup: string]: {
+      scorerGroups: {
+        [scorerGroup: string]: {
+          metricPathGroups: {
+            [metricPathGroup: string]: LeaderboardValueRecord[];
           };
         };
       };
     };
   };
 };
-
 export const getLeaderboardData = async (
   client: TraceServerClient,
   entity: string,
@@ -353,9 +354,10 @@ export const getLeaderboardData = async (
                       metricPathGroups: _.mapValues(
                         _.groupBy(scorerGroup, 'metricPathGroup'),
                         metricPathGroup => {
-                          return metricPathGroup.sort(
-                            (a, b) => a.sortKey - b.sortKey
-                          )[0].row;
+                          return metricPathGroup.map(row => row.row);
+                          //   .sort(
+                          //     (a, b) => a.sortKey - b.sortKey
+                          //   )[0].row;
                         }
                       ),
                     };

From 805451e4fb7b6a68de8a10f0da99e926b03f750e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 01:52:21 -0700
Subject: [PATCH 040/169] Massive refactor step 3

---
 .../pages/LeaderboardPage/LeaderboardConfigType.tsx   |  2 +-
 .../Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx |  6 ++++--
 .../LeaderboardPage/leaderboardServerInterface.ts     | 11 ++++++-----
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
index f1161f20477..5d421790b47 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
@@ -36,5 +36,5 @@ export type FilterAndGroupDatasetScorerMetricSpec = {
 export type FilterAndGroupModelSpec = {
   name: string; // "*" means all
   version: string; // "*" means all
-  splitByVersion?: boolean;
+  groupByVersion?: boolean;
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 2399da14570..b8b563b31a2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -78,7 +78,9 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         minWidth: 150,
         flex: 1,
         renderCell: (params: GridRenderCellParams) => {
-          const modelRef = parseRefMaybe(params.value);
+          const modelRef = parseRefMaybe(
+            `weave:///${entity}/${project}/object/${params.value}` ?? ''
+          );
           if (modelRef) {
             return (
               <div
@@ -172,7 +174,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
           )
       ),
     ],
-    [columnStats.datasetGroups, getColorForScore, onCellClick]
+    [columnStats.datasetGroups, entity, getColorForScore, onCellClick, project]
   );
 
   const groupingModel: GridColumnGroup[] = useMemo(() => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 73ed0cc85bf..1cf0fb95d8b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -63,6 +63,7 @@ export type GroupedLeaderboardModelGroup = {
     };
   };
 };
+
 export const getLeaderboardData = async (
   client: TraceServerClient,
   entity: string,
@@ -230,7 +231,7 @@ export const getLeaderboardData = async (
     const groupableRow: GroupableLeaderboardValueRecord = {
       datasetGroup: row.datasetName,
       scorerGroup: row.scorerName,
-      modelGroup: row.modelName,
+      modelGroup: `${row.modelName}:${row.modelVersion}`,
       metricPathGroup: row.metricPath,
       sortKey: -row.createdAt.getTime(),
       row,
@@ -269,7 +270,7 @@ export const getLeaderboardData = async (
       return {include: false, groupableRow};
     }
     if (datasetSpec.splitByVersion) {
-      groupableRow.datasetGroup += `${row.datasetName}:${row.datasetVersion}`;
+      groupableRow.datasetGroup = `${row.datasetName}:${row.datasetVersion}`;
     }
     if (datasetSpec.scorers) {
       let scorerSpec = datasetSpec.scorers.find(
@@ -294,7 +295,7 @@ export const getLeaderboardData = async (
         return {include: false, groupableRow};
       }
       if (scorerSpec.splitByVersion) {
-        groupableRow.scorerGroup += `${row.scorerName}:${row.scorerVersion}`;
+        groupableRow.scorerGroup = `${row.scorerName}:${row.scorerVersion}`;
       }
       if (scorerSpec.metrics) {
         const metricSpec = scorerSpec.metrics.find(
@@ -327,8 +328,8 @@ export const getLeaderboardData = async (
       if (!modelSpec) {
         return {include: false, groupableRow};
       }
-      if (modelSpec.splitByVersion) {
-        groupableRow.modelGroup += `${row.modelName}:${row.modelVersion}`;
+      if (modelSpec.groupByVersion) {
+        groupableRow.modelGroup = `${row.modelName}`;
       }
     }
     return {include: true, groupableRow};

From 7029a9934331ab7c85b4ceba3d6573583f5a02f7 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 02:08:49 -0700
Subject: [PATCH 041/169] Massive refactor step 4

---
 .../LeaderboardPage/LeaderboardConfigType.tsx |  6 +--
 .../pages/LeaderboardPage/LeaderboardGrid.tsx | 13 ++---
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 10 ++--
 .../leaderboardServerInterface.ts             | 53 ++++++++++---------
 4 files changed, 38 insertions(+), 44 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
index 5d421790b47..5dd6a5651ef 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
@@ -17,14 +17,14 @@ export type FilterAndGroupSpec = {
 export type FilterAndGroupDatasetSpec = {
   name: string; // "*" means all
   version: string; // "*" means all
-  splitByVersion?: boolean;
+  groupAllVersions?: boolean;
   scorers?: FilterAndGroupDatasetScorerSpec[]; // null is all
 };
 
 export type FilterAndGroupDatasetScorerSpec = {
   name: string; // "*" means all
   version: string; // "*" means all
-  splitByVersion?: boolean;
+  groupAllVersions?: boolean;
   metrics?: FilterAndGroupDatasetScorerMetricSpec[]; // null is all
 };
 
@@ -36,5 +36,5 @@ export type FilterAndGroupDatasetScorerMetricSpec = {
 export type FilterAndGroupModelSpec = {
   name: string; // "*" means all
   version: string; // "*" means all
-  groupByVersion?: boolean;
+  groupAllVersions?: boolean;
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index b8b563b31a2..9101e1f0285 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -26,7 +26,7 @@ interface LeaderboardGridProps {
   project: string;
   data: GroupedLeaderboardData;
   loading: boolean;
-  onCellClick: (modelName: string, metricName: string, score: number) => void;
+  onCellClick: (record: LeaderboardValueRecord) => void;
 }
 
 type RowData = {
@@ -155,13 +155,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
                               value
                             ),
                           }}
-                          onClick={() =>
-                            onCellClick(
-                              params.row.model,
-                              params.field as string,
-                              params.value as number
-                            )
-                          }>
+                          onClick={() => onCellClick(params.row)}>
                           {' '}
                           {inner}
                         </div>
@@ -204,8 +198,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
               children: [],
               renderHeaderGroup: params => {
                 const ref = parseRefMaybe(
-                  `weave:///${entity}/${project}/object/${scorerGroupName}` ??
-                    ''
+                  `weave:///${entity}/${project}/op/${scorerGroupName}` ?? ''
                 );
                 // console.log(node.headerName, ref);
                 if (ref) {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index dcc69a5ca1b..ffa8df9b091 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -10,6 +10,7 @@ import {useLeaderboardData} from './hooks';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
 import {LeaderboardGrid} from './LeaderboardGrid';
 import {LeaderboardConfig} from './LeaderboardPageConfig';
+import {LeaderboardValueRecord} from './leaderboardServerInterface';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
@@ -81,13 +82,8 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   );
   useTraceUpdate('a', {entity, project, currentConfig});
 
-  const handleCellClick = (
-    modelName: string,
-    metricName: string,
-    score: number
-  ) => {
-    const sourceCallId = false;
-    // data.scores?.[modelName]?.[metricName]?.sourceEvalCallId;
+  const handleCellClick = (record: LeaderboardValueRecord) => {
+    const sourceCallId = record.sourceEvaluationCallId;
     if (sourceCallId) {
       let to: string;
       if (USE_COMPARE_EVALUATIONS_PAGE) {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 1cf0fb95d8b..feda52db12b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -77,9 +77,11 @@ export const getLeaderboardData = async (
       base_object_classes: ['Evaluation'],
       is_op: false,
     },
+    sort_by: [{field: 'created_at', direction: 'desc'}],
   });
   const allEvaluationCallsProm = client.callsStreamQuery({
     project_id: projectIdFromParts({entity, project}),
+    sort_by: [{field: 'ended_at', direction: 'desc'}],
     filter: {
       op_names: [
         opVersionKeyToRefUri({
@@ -175,23 +177,6 @@ export const getLeaderboardData = async (
       sourceEvaluationObjectRef: evalObjectRefUri,
     };
 
-    const modelLatency = (call.output as any)?.model_latency?.mean;
-    if (modelLatency == null) {
-      console.warn('Skipping model latency', call);
-    } else {
-      const modelLatencyRecord: LeaderboardValueRecord = {
-        ...recordPartial,
-        metricType: 'modelLatency',
-        scorerName: 'modelLatency',
-        scorerVersion: 'modelLatency',
-        metricPath: 'model_latency.mean',
-        metricValue: modelLatency,
-      };
-      data.push(modelLatencyRecord);
-    }
-
-    // TODO: add modelCost, modelTokens, modelErrors
-
     const scorerRefUris = (evalObject.val.scorers ?? []) as string[];
     scorerRefUris.forEach(scorerRefUri => {
       const scorerRef = parseRefMaybe(scorerRefUri);
@@ -225,12 +210,32 @@ export const getLeaderboardData = async (
         data.push(scoreRecord);
       });
     });
+
+    const modelLatency = (call.output as any)?.model_latency?.mean;
+    if (modelLatency == null) {
+      console.warn('Skipping model latency', call);
+    } else {
+      const modelLatencyRecord: LeaderboardValueRecord = {
+        ...recordPartial,
+        metricType: 'modelLatency',
+        scorerName: 'modelLatency',
+        scorerVersion: 'modelLatency',
+        metricPath: 'model_latency.mean',
+        metricValue: modelLatency,
+      };
+      data.push(modelLatencyRecord);
+    }
+
+    // TODO: add modelCost, modelTokens, modelErrors
   });
 
   const filterableGroupableData = data.map(row => {
     const groupableRow: GroupableLeaderboardValueRecord = {
-      datasetGroup: row.datasetName,
-      scorerGroup: row.scorerName,
+      datasetGroup: `${row.datasetName}:${row.datasetVersion}`,
+      scorerGroup:
+        row.metricType === 'scorerMetric'
+          ? `${row.scorerName}:${row.scorerVersion}`
+          : row.scorerName,
       modelGroup: `${row.modelName}:${row.modelVersion}`,
       metricPathGroup: row.metricPath,
       sortKey: -row.createdAt.getTime(),
@@ -269,8 +274,8 @@ export const getLeaderboardData = async (
     if (!datasetSpec) {
       return {include: false, groupableRow};
     }
-    if (datasetSpec.splitByVersion) {
-      groupableRow.datasetGroup = `${row.datasetName}:${row.datasetVersion}`;
+    if (datasetSpec.groupAllVersions) {
+      groupableRow.datasetGroup = `${row.datasetName}`;
     }
     if (datasetSpec.scorers) {
       let scorerSpec = datasetSpec.scorers.find(
@@ -294,8 +299,8 @@ export const getLeaderboardData = async (
       if (!scorerSpec) {
         return {include: false, groupableRow};
       }
-      if (scorerSpec.splitByVersion) {
-        groupableRow.scorerGroup = `${row.scorerName}:${row.scorerVersion}`;
+      if (scorerSpec.groupAllVersions) {
+        groupableRow.scorerGroup = `${row.scorerName}`;
       }
       if (scorerSpec.metrics) {
         const metricSpec = scorerSpec.metrics.find(
@@ -328,7 +333,7 @@ export const getLeaderboardData = async (
       if (!modelSpec) {
         return {include: false, groupableRow};
       }
-      if (modelSpec.groupByVersion) {
+      if (modelSpec.groupAllVersions) {
         groupableRow.modelGroup = `${row.modelName}`;
       }
     }

From e1fc438692b6a731b4cc94a33c9fbd2a9d0159df Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 02:13:00 -0700
Subject: [PATCH 042/169] Fixed clicking

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx      | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 9101e1f0285..76118ff213f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -140,6 +140,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
                       } else {
                         inner = JSON.stringify(params.value);
                       }
+                      const record = recordFromRowData(params.row, datasetGroupName, scorerGroupName, metricPathGroupName);
                       return (
                         <div
                           style={{
@@ -155,8 +156,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
                               value
                             ),
                           }}
-                          onClick={() => onCellClick(params.row)}>
-                          {' '}
+                          onClick={() => record && onCellClick(record)}>
                           {inner}
                         </div>
                       );
@@ -432,6 +432,16 @@ const valueFromRowData = (
     ]?.metricPathGroups[metricPathGroupName] ?? []
   );
 };
+const recordFromRowData = (
+  rowData: RowData,
+  datasetGroupName: string,
+  scorerGroupName: string,
+  metricPathGroupName: string
+): LeaderboardValueRecord | null => {
+  return rowData.modelGroup.datasetGroups[datasetGroupName]?.scorerGroups[
+    scorerGroupName
+  ]?.metricPathGroups[metricPathGroupName]?.[0]
+};
 const getAggregatedResults = (
   data: null | LeaderboardValueRecord[]
 ): number | string | boolean | null => {

From 1e26f1ac350ee1df59cd11533912480cd3e78305 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 03:01:56 -0700
Subject: [PATCH 043/169] Fixed clicking

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx |  37 ++--
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  19 +-
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 170 +++++++++++++++++-
 .../leaderboardServerInterface.ts             |  54 +++---
 4 files changed, 224 insertions(+), 56 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 76118ff213f..5de6bf4ab5a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -7,9 +7,10 @@ import {
   GridLeafColumn,
   GridRenderCellParams,
   GridSortDirection,
+  GridSortItem,
 } from '@mui/x-data-grid-pro';
 import _ from 'lodash';
-import React, {useCallback, useMemo} from 'react';
+import React, {useCallback, useEffect, useMemo, useState} from 'react';
 
 import {NotApplicable} from '../../../Browse2/NotApplicable';
 import {parseRefMaybe, SmallRef} from '../../../Browse2/SmallRef';
@@ -140,7 +141,12 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
                       } else {
                         inner = JSON.stringify(params.value);
                       }
-                      const record = recordFromRowData(params.row, datasetGroupName, scorerGroupName, metricPathGroupName);
+                      const record = recordFromRowData(
+                        params.row,
+                        datasetGroupName,
+                        scorerGroupName,
+                        metricPathGroupName
+                      );
                       return (
                         <div
                           style={{
@@ -283,6 +289,14 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     return finalGroupingModel;
   }, [columnStats.datasetGroups, entity, project]);
 
+  const [sortModel, setSortModel] = useState<GridSortItem[]>([]);
+
+  useEffect(() => {
+    if (sortModel.length === 0 && columns.length > 1) {
+      setSortModel([{field: columns[1].field, sort: 'desc'}]);
+    }
+  }, [columns, sortModel]);
+
   return (
     <Box
       sx={{
@@ -290,7 +304,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         width: '100%',
         display: 'flex',
         flexDirection: 'column',
-        overflow: 'hidden', // Prevent outer container from scrolling
+        overflow: 'hidden',
       }}>
       <StyledDataGrid
         rows={rows}
@@ -301,6 +315,8 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         disableMultipleColumnsSorting={false}
         columnHeaderHeight={40}
         loading={loading}
+        sortModel={sortModel}
+        onSortModelChange={setSortModel}
         sx={{
           borderRadius: 0,
           '& .MuiDataGrid-footerContainer': {
@@ -322,7 +338,6 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
           },
         }}
         slots={{
-          // columnMenu: CallsCustomColumnMenu,
           pagination: PaginationButtons,
         }}
       />
@@ -330,18 +345,6 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   );
 };
 
-// const walkGroupingModel = (
-//   nodes: GridColumnNode[],
-//   fn: (node: GridColumnNode) => GridColumnNode
-// ) => {
-//   return nodes.map(node => {
-//     if ('children' in node) {
-//       node.children = walkGroupingModel(node.children, fn);
-//     }
-//     return fn(node);
-//   });
-// };
-
 type ColumnStats = {
   datasetGroups: {
     [datasetGroup: string]: {
@@ -440,7 +443,7 @@ const recordFromRowData = (
 ): LeaderboardValueRecord | null => {
   return rowData.modelGroup.datasetGroups[datasetGroupName]?.scorerGroups[
     scorerGroupName
-  ]?.metricPathGroups[metricPathGroupName]?.[0]
+  ]?.metricPathGroups[metricPathGroupName]?.[0];
 };
 const getAggregatedResults = (
   data: null | LeaderboardValueRecord[]
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index ffa8df9b091..dc06ca0d599 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,5 +1,4 @@
 import {Alert, Box, Typography} from '@mui/material';
-import {useTraceUpdate} from '@wandb/weave/common/util/hooks';
 import {Button} from '@wandb/weave/components/Button';
 import React, {useCallback, useMemo, useState} from 'react';
 import {useHistory} from 'react-router-dom';
@@ -28,23 +27,24 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
 const DEFAULT_DESCRIPTION = `# Leaderboard`;
 
 const usePersistedLeaderboardConfig = () => {
-  const initialConfig: LeaderboardConfigType = useMemo(() => {
-    return {
+  const [configPersisted, setConfigPersisted] = useState<LeaderboardConfigType>(
+    {
       version: 1,
       config: {description: '', dataSelectionSpec: {}},
-    };
-  }, []);
+    }
+  );
 
   const [config, setConfigLocal] =
-    useState<LeaderboardConfigType>(initialConfig);
+    useState<LeaderboardConfigType>(configPersisted);
 
   const persistConfig = useCallback(() => {
+    setConfigPersisted(config);
     // persistLeaderboardConfig(config);
-  }, []);
+  }, [config]);
 
   const cancelChanges = useCallback(() => {
-    setConfigLocal(initialConfig);
-  }, [initialConfig]);
+    setConfigLocal(configPersisted);
+  }, [configPersisted]);
 
   return {config, setConfigLocal, persistConfig, cancelChanges};
 };
@@ -80,7 +80,6 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     project,
     currentConfig.config.dataSelectionSpec
   );
-  useTraceUpdate('a', {entity, project, currentConfig});
 
   const handleCellClick = (record: LeaderboardValueRecord) => {
     const sourceCallId = record.sourceEvaluationCallId;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 1f1f70b0b7f..d932c689a64 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -1,6 +1,18 @@
-import {Alert, Box, Button, Typography} from '@mui/material';
-import React, {useState} from 'react';
+import {
+  Alert,
+  Box,
+  Button,
+  FormControl,
+  InputLabel,
+  MenuItem,
+  Select,
+  Typography,
+} from '@mui/material';
+import {ObjectRef} from '@wandb/weave/react';
+import React, {useCallback, useMemo, useState} from 'react';
 
+import {parseRefMaybe} from '../../../Browse2/SmallRef';
+import {useWFHooks} from '../wfReactInterface/context';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
 
 export const LeaderboardConfig: React.FC<{
@@ -23,6 +35,109 @@ export const LeaderboardConfig: React.FC<{
 
   const [showAlert, setShowAlert] = useState(true);
 
+  const {useRootObjectVersions} = useWFHooks();
+
+  const evalObjects = useRootObjectVersions(entity, project, {
+    baseObjectClasses: ['Evaluation'],
+  });
+
+  const [selectedEvalObject, setSelectedEvalObject] = useState<string | null>(
+    null
+  );
+
+  const evalObjectsMap = useMemo(() => {
+    return new Map(
+      (evalObjects.result ?? []).map(obj => [
+        `${obj.objectId}:${obj.versionHash}`,
+        obj,
+      ])
+    );
+  }, [evalObjects]);
+
+  const onEvalObjectChange = useCallback(
+    (newEvalObject: string) => {
+      const evalObject = evalObjectsMap.get(newEvalObject);
+      if (!evalObject) {
+        console.warn('Invalid eval object selected', newEvalObject);
+        return;
+      }
+      const datasetRef = parseRefMaybe(evalObject.val.dataset ?? '');
+      const scorers = (evalObject.val.scorers ?? [])
+        .map((scorer: string) => parseRefMaybe(scorer ?? ''))
+        .filter((scorer: ObjectRef | null) => scorer !== null) as ObjectRef[];
+      if (!datasetRef) {
+        console.warn('Invalid dataset ref', evalObject.val.dataset);
+        return;
+      }
+      setConfig(old => ({
+        ...old,
+        config: {
+          ...old.config,
+          dataSelectionSpec: {
+            datasets: [
+              ...(old.config.dataSelectionSpec.datasets ?? []),
+              {
+                name: datasetRef.artifactName,
+                version: datasetRef.artifactVersion,
+                scorers: [
+                  ...scorers.map(scorer => ({
+                    name: scorer.artifactName,
+                    version: scorer.artifactVersion,
+                  })),
+                  {
+                    name: 'modelLatency',
+                    version: 'modelLatency',
+                  },
+                ],
+              },
+            ],
+          },
+        },
+      }));
+      setSelectedEvalObject(newEvalObject);
+    },
+    [evalObjectsMap, setConfig]
+  );
+
+  const evalOptions = useMemo(() => {
+    return (evalObjects.result ?? []).map(obj => ({
+      label: `${obj.objectId}:v${obj.versionIndex} (${obj.versionHash.slice(
+        0,
+        6
+      )})`,
+      value: `${obj.objectId}:${obj.versionHash}`,
+    }));
+  }, [evalObjects]);
+
+  const toggleModelsGrouped = (shouldGroup: boolean) => {
+    setConfig(old => {
+      let newModels = [];
+      const currModels = old.config.dataSelectionSpec.models ?? [];
+      if (currModels.length === 0) {
+        newModels.push({
+          name: '*',
+          version: '*',
+          groupAllVersions: shouldGroup,
+        });
+      } else {
+        newModels = currModels.map(model => ({
+          ...model,
+          groupAllVersions: shouldGroup,
+        }));
+      }
+      return {
+        ...old,
+        config: {
+          ...old.config,
+          dataSelectionSpec: {
+            ...old.config.dataSelectionSpec,
+            models: newModels,
+          },
+        },
+      };
+    });
+  };
+
   return (
     <Box
       sx={{
@@ -34,7 +149,7 @@ export const LeaderboardConfig: React.FC<{
       }}>
       <Box
         sx={{
-          flexGrow: 1,
+          // flexGrow: 1,
           overflowY: 'auto',
           p: 2,
         }}>
@@ -42,6 +157,55 @@ export const LeaderboardConfig: React.FC<{
         <Typography variant="h5" gutterBottom>
           Leaderboard Configuration
         </Typography>
+        <FormControl fullWidth sx={{mt: 2, mb: 2}}>
+          <InputLabel id="eval-select-label">
+            Add 'Preset' Evaluation
+          </InputLabel>
+          <Select
+            labelId="eval-select-label"
+            id="eval-select"
+            value={selectedEvalObject || ''}
+            onChange={event => {
+              const selectedValue = event.target.value;
+              onEvalObjectChange(selectedValue);
+            }}
+            label="Add 'Preset' Evaluation">
+            {evalOptions.map(option => (
+              <MenuItem key={option.value} value={option.value}>
+                {option.label}
+              </MenuItem>
+            ))}
+          </Select>
+        </FormControl>
+        <Button
+          variant="outlined"
+          onClick={() => toggleModelsGrouped(true)}
+          sx={{mt: 2, mb: 2}}>
+          Mark Model Versions as Grouped
+        </Button>
+        <Button
+          variant="outlined"
+          onClick={() => toggleModelsGrouped(false)}
+          sx={{mt: 2, mb: 2}}>
+          Mark Model Versions as Individual
+        </Button>
+      </Box>
+
+      <Box sx={{mt: 2, mb: 2, flex: 1, overflowY: 'auto'}}>
+        <Typography variant="h6" gutterBottom>
+          Configuration Preview
+        </Typography>
+        <pre
+          style={{
+            backgroundColor: '#f5f5f5',
+            padding: '10px',
+            borderRadius: '4px',
+            overflowX: 'auto',
+            whiteSpace: 'pre-wrap',
+            wordWrap: 'break-word',
+          }}>
+          {JSON.stringify(config, null, 2)}
+        </pre>
       </Box>
 
       <Box
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index feda52db12b..94ac3c4fcb3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -242,6 +242,33 @@ export const getLeaderboardData = async (
       row,
     };
 
+    if (spec.models) {
+      let modelSpec = spec.models.find(
+        model =>
+          model.name === row.modelName && model.version === row.modelVersion
+      );
+      modelSpec =
+        modelSpec ||
+        spec.models.find(
+          model =>
+            model.name === row.modelName &&
+            (model.version === '*' || model.version === row.modelVersion)
+        );
+      modelSpec =
+        modelSpec ||
+        spec.models.find(
+          model =>
+            (model.name === '*' || model.name === row.modelName) &&
+            (model.version === '*' || model.version === row.modelVersion)
+        );
+      if (!modelSpec) {
+        return {include: false, groupableRow};
+      }
+      if (modelSpec.groupAllVersions) {
+        groupableRow.modelGroup = `${row.modelName}`;
+      }
+    }
+
     if (!spec.datasets) {
       return {include: true, groupableRow};
     }
@@ -311,32 +338,7 @@ export const getLeaderboardData = async (
         }
       }
     }
-    if (spec.models) {
-      let modelSpec = spec.models.find(
-        model =>
-          model.name === row.modelName && model.version === row.modelVersion
-      );
-      modelSpec =
-        modelSpec ||
-        spec.models.find(
-          model =>
-            model.name === row.modelName &&
-            (model.version === '*' || model.version === row.modelVersion)
-        );
-      modelSpec =
-        modelSpec ||
-        spec.models.find(
-          model =>
-            (model.name === '*' || model.name === row.modelName) &&
-            (model.version === '*' || model.version === row.modelVersion)
-        );
-      if (!modelSpec) {
-        return {include: false, groupableRow};
-      }
-      if (modelSpec.groupAllVersions) {
-        groupableRow.modelGroup = `${row.modelName}`;
-      }
-    }
+
     return {include: true, groupableRow};
   });
 

From 4c0772069769a43bddd98552754438833f3e4afa Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 08:59:24 -0700
Subject: [PATCH 044/169] Fixed clicking

---
 .../Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index dc06ca0d599..7133f19f7db 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,6 +1,6 @@
 import {Alert, Box, Typography} from '@mui/material';
 import {Button} from '@wandb/weave/components/Button';
-import React, {useCallback, useMemo, useState} from 'react';
+import React, {useCallback, useState} from 'react';
 import {useHistory} from 'react-router-dom';
 
 import {useWeaveflowRouteContext} from '../../context';

From 14ed5b21cc507e38935fe9007f0d3bd4a2279137 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 10:20:30 -0700
Subject: [PATCH 045/169] Added eval sources

---
 .../LeaderboardPage/LeaderboardConfigType.tsx |  6 ++
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  6 ++
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 69 +++++++++----------
 .../leaderboardServerInterface.ts             | 29 +++++++-
 4 files changed, 71 insertions(+), 39 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
index 5dd6a5651ef..f1eebbd6526 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
@@ -10,10 +10,16 @@ export type LeaderboardConfigType = {
 };
 
 export type FilterAndGroupSpec = {
+  sourceEvaluations?: FilterAndGroupSourceEvaluationSpec[]; // null is all
   datasets?: FilterAndGroupDatasetSpec[]; // null is all
   models?: FilterAndGroupModelSpec[]; // null is all
 };
 
+export type FilterAndGroupSourceEvaluationSpec = {
+  name: string;
+  version: string; // "*" means all
+};
+
 export type FilterAndGroupDatasetSpec = {
   name: string; // "*" means all
   version: string; // "*" means all
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 7133f19f7db..c39dcb11ed3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -185,3 +185,9 @@ const UnlistedAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
     </Alert>
   );
 };
+
+
+// TODO:
+// * [ ] Make the editable context (i think we should have an object and version and isEditing, etc... Think more like reports)
+// * [ ] Edit panel revisions
+// * [ ] UX Cleanup
\ No newline at end of file
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index d932c689a64..506c35d2a61 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -39,64 +39,57 @@ export const LeaderboardConfig: React.FC<{
 
   const evalObjects = useRootObjectVersions(entity, project, {
     baseObjectClasses: ['Evaluation'],
-  });
+    latestOnly: true,
+  }, undefined, true);
 
   const [selectedEvalObject, setSelectedEvalObject] = useState<string | null>(
     null
   );
 
-  const evalObjectsMap = useMemo(() => {
-    return new Map(
-      (evalObjects.result ?? []).map(obj => [
-        `${obj.objectId}:${obj.versionHash}`,
-        obj,
-      ])
-    );
-  }, [evalObjects]);
+  // const evalObjectsMap = useMemo(() => {
+  //   return new Map(
+  //     (evalObjects.result ?? []).map(obj => [
+  //       `${obj.objectId}:${obj.versionHash}`,
+  //       obj,
+  //     ])
+  //   );
+  // }, [evalObjects]);
 
   const onEvalObjectChange = useCallback(
     (newEvalObject: string) => {
-      const evalObject = evalObjectsMap.get(newEvalObject);
-      if (!evalObject) {
-        console.warn('Invalid eval object selected', newEvalObject);
-        return;
-      }
-      const datasetRef = parseRefMaybe(evalObject.val.dataset ?? '');
-      const scorers = (evalObject.val.scorers ?? [])
-        .map((scorer: string) => parseRefMaybe(scorer ?? ''))
-        .filter((scorer: ObjectRef | null) => scorer !== null) as ObjectRef[];
-      if (!datasetRef) {
-        console.warn('Invalid dataset ref', evalObject.val.dataset);
-        return;
-      }
+      // const evalObject = evalObjectsMap.get(newEvalObject);
+      // if (!evalObject) {
+      //   console.warn('Invalid eval object selected', newEvalObject);
+      //   return;
+      // }
+      const [name, version] = newEvalObject.split(':');
+      // const datasetRef = parseRefMaybe(evalObject.val.dataset ?? '');
+      // const scorers = (evalObject.val.scorers ?? [])
+      //   .map((scorer: string) => parseRefMaybe(scorer ?? ''))
+      //   .filter((scorer: ObjectRef | null) => scorer !== null) as ObjectRef[];
+      // if (!datasetRef) {
+      //   console.warn('Invalid dataset ref', evalObject.val.dataset);
+      //   return;
+      // }
       setConfig(old => ({
         ...old,
         config: {
           ...old.config,
           dataSelectionSpec: {
-            datasets: [
-              ...(old.config.dataSelectionSpec.datasets ?? []),
+            ...old.config.dataSelectionSpec,
+            sourceEvaluations: [
+              ...(old.config.dataSelectionSpec.sourceEvaluations ?? []),
               {
-                name: datasetRef.artifactName,
-                version: datasetRef.artifactVersion,
-                scorers: [
-                  ...scorers.map(scorer => ({
-                    name: scorer.artifactName,
-                    version: scorer.artifactVersion,
-                  })),
-                  {
-                    name: 'modelLatency',
-                    version: 'modelLatency',
-                  },
-                ],
-              },
+                name: name,
+                version: version,
+              }
             ],
           },
         },
       }));
       setSelectedEvalObject(newEvalObject);
     },
-    [evalObjectsMap, setConfig]
+    [setConfig]
   );
 
   const evalOptions = useMemo(() => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 94ac3c4fcb3..122cc738b26 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -10,7 +10,7 @@ import {
   convertISOToDate,
   projectIdFromParts,
 } from '../wfReactInterface/tsDataModelHooks';
-import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
+import {objectVersionKeyToRefUri, opVersionKeyToRefUri} from '../wfReactInterface/utilities';
 import {FilterAndGroupSpec} from './LeaderboardConfigType';
 
 export type LeaderboardValueRecord = {
@@ -70,12 +70,27 @@ export const getLeaderboardData = async (
   project: string,
   spec: FilterAndGroupSpec = {}
 ): Promise<GroupedLeaderboardData> => {
+  const sourceEvals = (spec.sourceEvaluations ?? [])
+  const evalNames  =sourceEvals.map(sourceEvaluation => sourceEvaluation.name)
+  const fullyQualifiedEvalRefs =sourceEvals.map(sourceEvaluation => {
+    return objectVersionKeyToRefUri({
+        scheme: 'weave',
+        weaveKind: 'object',
+        entity,
+        project,
+        objectId: sourceEvaluation.name,
+        versionHash: sourceEvaluation.version,
+        path: '',
+    })
+  });
   // get all the evaluations
   const allEvaluationObjectsProm = client.objsQuery({
     project_id: projectIdFromParts({entity, project}),
     filter: {
       base_object_classes: ['Evaluation'],
       is_op: false,
+      // Sad :( we can't actually filter by version here!
+      object_ids: evalNames,
     },
     sort_by: [{field: 'created_at', direction: 'desc'}],
   });
@@ -91,10 +106,22 @@ export const getLeaderboardData = async (
           versionHash: '*',
         }),
       ],
+      input_refs: fullyQualifiedEvalRefs
     },
   });
 
+
   const allEvaluationObjectsRes = await allEvaluationObjectsProm;
+
+  // This hack to get around the fact that we can't filter by version in the query
+  if (sourceEvals.length > 0) {
+    allEvaluationObjectsRes.objs = allEvaluationObjectsRes.objs.filter(obj => {
+        return sourceEvals.some(sourceEval => {
+            return obj.object_id === sourceEval.name && (obj.digest === sourceEval.version || sourceEval.version === '*');
+        });
+    });
+  }
+
   const evaluationObjectDigestMap = new Map<
     string,
     {versions: Map<string, TraceObjSchema>; versionOrder: string[]}

From aad123219ae601fd2cd7c4659fac53160e43f4e7 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 10:20:39 -0700
Subject: [PATCH 046/169] Added eval sources - lint

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  3 +-
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 22 ++++++-----
 .../leaderboardServerInterface.ts             | 39 +++++++++++--------
 3 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index c39dcb11ed3..6afbd03ced4 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -186,8 +186,7 @@ const UnlistedAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
   );
 };
 
-
 // TODO:
 // * [ ] Make the editable context (i think we should have an object and version and isEditing, etc... Think more like reports)
 // * [ ] Edit panel revisions
-// * [ ] UX Cleanup
\ No newline at end of file
+// * [ ] UX Cleanup
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 506c35d2a61..6b3eb4cb219 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -8,10 +8,8 @@ import {
   Select,
   Typography,
 } from '@mui/material';
-import {ObjectRef} from '@wandb/weave/react';
 import React, {useCallback, useMemo, useState} from 'react';
 
-import {parseRefMaybe} from '../../../Browse2/SmallRef';
 import {useWFHooks} from '../wfReactInterface/context';
 import {LeaderboardConfigType} from './LeaderboardConfigType';
 
@@ -37,10 +35,16 @@ export const LeaderboardConfig: React.FC<{
 
   const {useRootObjectVersions} = useWFHooks();
 
-  const evalObjects = useRootObjectVersions(entity, project, {
-    baseObjectClasses: ['Evaluation'],
-    latestOnly: true,
-  }, undefined, true);
+  const evalObjects = useRootObjectVersions(
+    entity,
+    project,
+    {
+      baseObjectClasses: ['Evaluation'],
+      latestOnly: true,
+    },
+    undefined,
+    true
+  );
 
   const [selectedEvalObject, setSelectedEvalObject] = useState<string | null>(
     null
@@ -80,9 +84,9 @@ export const LeaderboardConfig: React.FC<{
             sourceEvaluations: [
               ...(old.config.dataSelectionSpec.sourceEvaluations ?? []),
               {
-                name: name,
-                version: version,
-              }
+                name,
+                version,
+              },
             ],
           },
         },
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 122cc738b26..2dd8eae5fa6 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -10,7 +10,10 @@ import {
   convertISOToDate,
   projectIdFromParts,
 } from '../wfReactInterface/tsDataModelHooks';
-import {objectVersionKeyToRefUri, opVersionKeyToRefUri} from '../wfReactInterface/utilities';
+import {
+  objectVersionKeyToRefUri,
+  opVersionKeyToRefUri,
+} from '../wfReactInterface/utilities';
 import {FilterAndGroupSpec} from './LeaderboardConfigType';
 
 export type LeaderboardValueRecord = {
@@ -70,18 +73,18 @@ export const getLeaderboardData = async (
   project: string,
   spec: FilterAndGroupSpec = {}
 ): Promise<GroupedLeaderboardData> => {
-  const sourceEvals = (spec.sourceEvaluations ?? [])
-  const evalNames  =sourceEvals.map(sourceEvaluation => sourceEvaluation.name)
-  const fullyQualifiedEvalRefs =sourceEvals.map(sourceEvaluation => {
+  const sourceEvals = spec.sourceEvaluations ?? [];
+  const evalNames = sourceEvals.map(sourceEvaluation => sourceEvaluation.name);
+  const fullyQualifiedEvalRefs = sourceEvals.map(sourceEvaluation => {
     return objectVersionKeyToRefUri({
-        scheme: 'weave',
-        weaveKind: 'object',
-        entity,
-        project,
-        objectId: sourceEvaluation.name,
-        versionHash: sourceEvaluation.version,
-        path: '',
-    })
+      scheme: 'weave',
+      weaveKind: 'object',
+      entity,
+      project,
+      objectId: sourceEvaluation.name,
+      versionHash: sourceEvaluation.version,
+      path: '',
+    });
   });
   // get all the evaluations
   const allEvaluationObjectsProm = client.objsQuery({
@@ -106,19 +109,21 @@ export const getLeaderboardData = async (
           versionHash: '*',
         }),
       ],
-      input_refs: fullyQualifiedEvalRefs
+      input_refs: fullyQualifiedEvalRefs,
     },
   });
 
-
   const allEvaluationObjectsRes = await allEvaluationObjectsProm;
 
   // This hack to get around the fact that we can't filter by version in the query
   if (sourceEvals.length > 0) {
     allEvaluationObjectsRes.objs = allEvaluationObjectsRes.objs.filter(obj => {
-        return sourceEvals.some(sourceEval => {
-            return obj.object_id === sourceEval.name && (obj.digest === sourceEval.version || sourceEval.version === '*');
-        });
+      return sourceEvals.some(sourceEval => {
+        return (
+          obj.object_id === sourceEval.name &&
+          (obj.digest === sourceEval.version || sourceEval.version === '*')
+        );
+      });
     });
   }
 

From 3e1b0a9b053107087dd05d3c81c4fd8c104b3b9a Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 10:31:21 -0700
Subject: [PATCH 047/169] Added eval * support

---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx |  4 +-
 .../LeaderboardPage/LeaderboardPageConfig.tsx |  9 ++-
 .../leaderboardServerInterface.ts             | 67 ++++++++++++-------
 3 files changed, 49 insertions(+), 31 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 5de6bf4ab5a..362f021b8e2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -292,10 +292,10 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   const [sortModel, setSortModel] = useState<GridSortItem[]>([]);
 
   useEffect(() => {
-    if (sortModel.length === 0 && columns.length > 1) {
+    if (sortModel.length === 0 && columns.length > 1 && !loading) {
       setSortModel([{field: columns[1].field, sort: 'desc'}]);
     }
-  }, [columns, sortModel]);
+  }, [columns, loading, sortModel]);
 
   return (
     <Box
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 6b3eb4cb219..62506ba3612 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -97,13 +97,16 @@ export const LeaderboardConfig: React.FC<{
   );
 
   const evalOptions = useMemo(() => {
-    return (evalObjects.result ?? []).map(obj => ({
-      label: `${obj.objectId}:v${obj.versionIndex} (${obj.versionHash.slice(
+    return (evalObjects.result ?? []).flatMap(obj => ([{
+      label: `${obj.objectId}:v${obj.versionIndex} (latest) (${obj.versionHash.slice(
         0,
         6
       )})`,
       value: `${obj.objectId}:${obj.versionHash}`,
-    }));
+    }, {
+      label: `${obj.objectId}:* (all)`,
+      value: `${obj.objectId}:*`,
+    }]));
   }, [evalObjects]);
 
   const toggleModelsGrouped = (shouldGroup: boolean) => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
index 2dd8eae5fa6..e3501d13d85 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
@@ -75,17 +75,17 @@ export const getLeaderboardData = async (
 ): Promise<GroupedLeaderboardData> => {
   const sourceEvals = spec.sourceEvaluations ?? [];
   const evalNames = sourceEvals.map(sourceEvaluation => sourceEvaluation.name);
-  const fullyQualifiedEvalRefs = sourceEvals.map(sourceEvaluation => {
-    return objectVersionKeyToRefUri({
-      scheme: 'weave',
-      weaveKind: 'object',
-      entity,
-      project,
-      objectId: sourceEvaluation.name,
-      versionHash: sourceEvaluation.version,
-      path: '',
-    });
-  });
+  // const fullyQualifiedEvalRefs = sourceEvals.map(sourceEvaluation => {
+  //   return objectVersionKeyToRefUri({
+  //     scheme: 'weave',
+  //     weaveKind: 'object',
+  //     entity,
+  //     project,
+  //     objectId: sourceEvaluation.name,
+  //     versionHash: sourceEvaluation.version,
+  //     path: '',
+  //   });
+  // });
   // get all the evaluations
   const allEvaluationObjectsProm = client.objsQuery({
     project_id: projectIdFromParts({entity, project}),
@@ -97,21 +97,6 @@ export const getLeaderboardData = async (
     },
     sort_by: [{field: 'created_at', direction: 'desc'}],
   });
-  const allEvaluationCallsProm = client.callsStreamQuery({
-    project_id: projectIdFromParts({entity, project}),
-    sort_by: [{field: 'ended_at', direction: 'desc'}],
-    filter: {
-      op_names: [
-        opVersionKeyToRefUri({
-          entity,
-          project,
-          opId: EVALUATE_OP_NAME_POST_PYDANTIC,
-          versionHash: '*',
-        }),
-      ],
-      input_refs: fullyQualifiedEvalRefs,
-    },
-  });
 
   const allEvaluationObjectsRes = await allEvaluationObjectsProm;
 
@@ -148,7 +133,37 @@ export const getLeaderboardData = async (
       .map(entry => entry[0]);
   });
 
+  const fullyQualifiedEvalRefs = allEvaluationObjectsRes.objs.map(obj => {
+    return objectVersionKeyToRefUri({
+      scheme: 'weave',
+      weaveKind: 'object',
+      entity,
+      project,
+      objectId: obj.object_id,
+      versionHash: obj.digest,
+      path: '',
+    });
+  });
+
+  // SAD: if `input_refs` supported `*` wildcard, then we could do the queries in parallel
+  const allEvaluationCallsProm = client.callsStreamQuery({
+    project_id: projectIdFromParts({entity, project}),
+    sort_by: [{field: 'ended_at', direction: 'desc'}],
+    filter: {
+      op_names: [
+        opVersionKeyToRefUri({
+          entity,
+          project,
+          opId: EVALUATE_OP_NAME_POST_PYDANTIC,
+          versionHash: '*',
+        }),
+      ],
+      input_refs: fullyQualifiedEvalRefs,
+    },
+  });
+
   const allEvaluationCallsRes = await allEvaluationCallsProm;
+
   const data: LeaderboardValueRecord[] = [];
   allEvaluationCallsRes.calls.forEach(call => {
     const evalObjectRefUri = call.inputs.self;

From 5f2bd3e45c9a1b96f67a169eed96141ec6806b4e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 10:31:44 -0700
Subject: [PATCH 048/169] Added eval * support - lint

---
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 62506ba3612..bf01b0d63eb 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -97,16 +97,18 @@ export const LeaderboardConfig: React.FC<{
   );
 
   const evalOptions = useMemo(() => {
-    return (evalObjects.result ?? []).flatMap(obj => ([{
-      label: `${obj.objectId}:v${obj.versionIndex} (latest) (${obj.versionHash.slice(
-        0,
-        6
-      )})`,
-      value: `${obj.objectId}:${obj.versionHash}`,
-    }, {
-      label: `${obj.objectId}:* (all)`,
-      value: `${obj.objectId}:*`,
-    }]));
+    return (evalObjects.result ?? []).flatMap(obj => [
+      {
+        label: `${obj.objectId}:v${
+          obj.versionIndex
+        } (latest) (${obj.versionHash.slice(0, 6)})`,
+        value: `${obj.objectId}:${obj.versionHash}`,
+      },
+      {
+        label: `${obj.objectId}:* (all)`,
+        value: `${obj.objectId}:*`,
+      },
+    ]);
   }, [evalObjects]);
 
   const toggleModelsGrouped = (shouldGroup: boolean) => {

From bdded0f2a96fec7a2895a04a3dabc250746bcd69 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 11:40:20 -0700
Subject: [PATCH 049/169] Human: Make room for the AI

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx |   2 +-
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 190 +++++-------------
 2 files changed, 52 insertions(+), 140 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 6afbd03ced4..2ff1ed41d99 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -180,7 +180,7 @@ const UnlistedAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
     <Alert severity="info" onClose={onClose}>
       <Typography variant="body1">
         You have found an internal, unlisted beta page! Please expect bugs and
-        incomplete features. Permilinks are not yet supported.
+        incomplete features. Permilinks & saving state are not yet supported
       </Typography>
     </Alert>
   );
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index bf01b0d63eb..fa126ad2be8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -11,7 +11,7 @@ import {
 import React, {useCallback, useMemo, useState} from 'react';
 
 import {useWFHooks} from '../wfReactInterface/context';
-import {LeaderboardConfigType} from './LeaderboardConfigType';
+import {FilterAndGroupDatasetSpec, FilterAndGroupSourceEvaluationSpec, FilterAndGroupSpec, LeaderboardConfigType} from './LeaderboardConfigType';
 
 export const LeaderboardConfig: React.FC<{
   entity: string;
@@ -33,112 +33,6 @@ export const LeaderboardConfig: React.FC<{
 
   const [showAlert, setShowAlert] = useState(true);
 
-  const {useRootObjectVersions} = useWFHooks();
-
-  const evalObjects = useRootObjectVersions(
-    entity,
-    project,
-    {
-      baseObjectClasses: ['Evaluation'],
-      latestOnly: true,
-    },
-    undefined,
-    true
-  );
-
-  const [selectedEvalObject, setSelectedEvalObject] = useState<string | null>(
-    null
-  );
-
-  // const evalObjectsMap = useMemo(() => {
-  //   return new Map(
-  //     (evalObjects.result ?? []).map(obj => [
-  //       `${obj.objectId}:${obj.versionHash}`,
-  //       obj,
-  //     ])
-  //   );
-  // }, [evalObjects]);
-
-  const onEvalObjectChange = useCallback(
-    (newEvalObject: string) => {
-      // const evalObject = evalObjectsMap.get(newEvalObject);
-      // if (!evalObject) {
-      //   console.warn('Invalid eval object selected', newEvalObject);
-      //   return;
-      // }
-      const [name, version] = newEvalObject.split(':');
-      // const datasetRef = parseRefMaybe(evalObject.val.dataset ?? '');
-      // const scorers = (evalObject.val.scorers ?? [])
-      //   .map((scorer: string) => parseRefMaybe(scorer ?? ''))
-      //   .filter((scorer: ObjectRef | null) => scorer !== null) as ObjectRef[];
-      // if (!datasetRef) {
-      //   console.warn('Invalid dataset ref', evalObject.val.dataset);
-      //   return;
-      // }
-      setConfig(old => ({
-        ...old,
-        config: {
-          ...old.config,
-          dataSelectionSpec: {
-            ...old.config.dataSelectionSpec,
-            sourceEvaluations: [
-              ...(old.config.dataSelectionSpec.sourceEvaluations ?? []),
-              {
-                name,
-                version,
-              },
-            ],
-          },
-        },
-      }));
-      setSelectedEvalObject(newEvalObject);
-    },
-    [setConfig]
-  );
-
-  const evalOptions = useMemo(() => {
-    return (evalObjects.result ?? []).flatMap(obj => [
-      {
-        label: `${obj.objectId}:v${
-          obj.versionIndex
-        } (latest) (${obj.versionHash.slice(0, 6)})`,
-        value: `${obj.objectId}:${obj.versionHash}`,
-      },
-      {
-        label: `${obj.objectId}:* (all)`,
-        value: `${obj.objectId}:*`,
-      },
-    ]);
-  }, [evalObjects]);
-
-  const toggleModelsGrouped = (shouldGroup: boolean) => {
-    setConfig(old => {
-      let newModels = [];
-      const currModels = old.config.dataSelectionSpec.models ?? [];
-      if (currModels.length === 0) {
-        newModels.push({
-          name: '*',
-          version: '*',
-          groupAllVersions: shouldGroup,
-        });
-      } else {
-        newModels = currModels.map(model => ({
-          ...model,
-          groupAllVersions: shouldGroup,
-        }));
-      }
-      return {
-        ...old,
-        config: {
-          ...old.config,
-          dataSelectionSpec: {
-            ...old.config.dataSelectionSpec,
-            models: newModels,
-          },
-        },
-      };
-    });
-  };
 
   return (
     <Box
@@ -159,38 +53,6 @@ export const LeaderboardConfig: React.FC<{
         <Typography variant="h5" gutterBottom>
           Leaderboard Configuration
         </Typography>
-        <FormControl fullWidth sx={{mt: 2, mb: 2}}>
-          <InputLabel id="eval-select-label">
-            Add 'Preset' Evaluation
-          </InputLabel>
-          <Select
-            labelId="eval-select-label"
-            id="eval-select"
-            value={selectedEvalObject || ''}
-            onChange={event => {
-              const selectedValue = event.target.value;
-              onEvalObjectChange(selectedValue);
-            }}
-            label="Add 'Preset' Evaluation">
-            {evalOptions.map(option => (
-              <MenuItem key={option.value} value={option.value}>
-                {option.label}
-              </MenuItem>
-            ))}
-          </Select>
-        </FormControl>
-        <Button
-          variant="outlined"
-          onClick={() => toggleModelsGrouped(true)}
-          sx={{mt: 2, mb: 2}}>
-          Mark Model Versions as Grouped
-        </Button>
-        <Button
-          variant="outlined"
-          onClick={() => toggleModelsGrouped(false)}
-          sx={{mt: 2, mb: 2}}>
-          Mark Model Versions as Individual
-        </Button>
       </Box>
 
       <Box sx={{mt: 2, mb: 2, flex: 1, overflowY: 'auto'}}>
@@ -210,6 +72,8 @@ export const LeaderboardConfig: React.FC<{
         </pre>
       </Box>
 
+      {/* Add config builder / editor here */}
+
       <Box
         sx={{
           display: 'flex',
@@ -239,3 +103,51 @@ const TempAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
     </Alert>
   );
 };
+
+// These functions are placeholders which I will implement.
+
+const fetchEvaluationNames = async (): Promise<string[]> => {
+  // TODO
+  return Promise.resolve([])
+}
+
+const fetchEvaluationVersionsForName = async (name: string): Promise<string[]> => {
+  // TODO
+  return Promise.resolve([])
+}
+
+const fetchDatasetNamesForSpec = async (spec: FilterAndGroupSpec): Promise<string[]> => {
+  // TODO
+  return Promise.resolve([])
+}
+
+const fetchDatasetVersionsForSpecAndName = async (spec: FilterAndGroupSpec, name: string): Promise<string[]> => {
+  // TODO
+  return Promise.resolve([])
+}
+
+const fetchModelNamesForSpec = async (spec: FilterAndGroupSpec): Promise<string[]> => {
+  // TODO
+  return Promise.resolve([])
+}
+
+const fetchModelVersionsForSpecndName = async (spec: FilterAndGroupSpec, name: string): Promise<string[]> => {
+  // TODO
+  return Promise.resolve([])
+}
+
+const fetchScorerNamesForSpec = async (spec: FilterAndGroupSpec): Promise<string[]> => {
+  // TODO
+  return Promise.resolve([])
+}
+
+const fetchScorerVersionsForSpecAndName = async (spec: FilterAndGroupSpec, name: string): Promise<string[]> => {
+  // TODO
+  return Promise.resolve([])
+}
+
+const fetchMetricPathsForSpec = async (spec: FilterAndGroupSpec): Promise<string[]> => {
+    // TODO
+    return Promise.resolve([])
+}
+

From ba9b46867dd55dfe908b188c1a6255dd8c251740 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 11:40:37 -0700
Subject: [PATCH 050/169] AI: Make room for the human

---
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 385 +++++++++++++++++-
 1 file changed, 375 insertions(+), 10 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index fa126ad2be8..36705bf86f2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -2,13 +2,17 @@ import {
   Alert,
   Box,
   Button,
+  Checkbox,
   FormControl,
+  FormControlLabel,
   InputLabel,
   MenuItem,
   Select,
+  SelectChangeEvent,
+  TextField,
   Typography,
 } from '@mui/material';
-import React, {useCallback, useMemo, useState} from 'react';
+import React, {useCallback, useEffect, useState} from 'react';
 
 import {useWFHooks} from '../wfReactInterface/context';
 import {FilterAndGroupDatasetSpec, FilterAndGroupSourceEvaluationSpec, FilterAndGroupSpec, LeaderboardConfigType} from './LeaderboardConfigType';
@@ -23,6 +27,8 @@ export const LeaderboardConfig: React.FC<{
     updater: (config: LeaderboardConfigType) => LeaderboardConfigType
   ) => void;
 }> = ({entity, project, config, setConfig, onPersist, onCancel}) => {
+  const [showAlert, setShowAlert] = useState(true);
+
   const handleSave = () => {
     onPersist();
   };
@@ -31,8 +37,15 @@ export const LeaderboardConfig: React.FC<{
     onCancel();
   };
 
-  const [showAlert, setShowAlert] = useState(true);
-
+  const updateConfig = (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => {
+    setConfig((prevConfig) => ({
+      ...prevConfig,
+      config: {
+        ...prevConfig.config,
+        dataSelectionSpec: updater(prevConfig.config.dataSelectionSpec),
+      },
+    }));
+  };
 
   return (
     <Box
@@ -45,7 +58,6 @@ export const LeaderboardConfig: React.FC<{
       }}>
       <Box
         sx={{
-          // flexGrow: 1,
           overflowY: 'auto',
           p: 2,
         }}>
@@ -55,7 +67,22 @@ export const LeaderboardConfig: React.FC<{
         </Typography>
       </Box>
 
-      <Box sx={{mt: 2, mb: 2, flex: 1, overflowY: 'auto'}}>
+      <Box sx={{mt: 2, mb: 2, flex: 1, overflowY: 'auto', px: 2}}>
+        <SourceEvaluationsConfig
+          sourceEvaluations={config.config.dataSelectionSpec.sourceEvaluations}
+          updateConfig={updateConfig}
+        />
+        <DatasetsConfig
+          datasets={config.config.dataSelectionSpec.datasets}
+          updateConfig={updateConfig}
+        />
+        <ModelsConfig
+          models={config.config.dataSelectionSpec.models}
+          updateConfig={updateConfig}
+        />
+      </Box>
+
+      <Box sx={{mt: 2, mb: 2, px: 2}}>
         <Typography variant="h6" gutterBottom>
           Configuration Preview
         </Typography>
@@ -65,6 +92,7 @@ export const LeaderboardConfig: React.FC<{
             padding: '10px',
             borderRadius: '4px',
             overflowX: 'auto',
+            maxHeight: '400px',
             whiteSpace: 'pre-wrap',
             wordWrap: 'break-word',
           }}>
@@ -72,8 +100,6 @@ export const LeaderboardConfig: React.FC<{
         </pre>
       </Box>
 
-      {/* Add config builder / editor here */}
-
       <Box
         sx={{
           display: 'flex',
@@ -104,7 +130,347 @@ const TempAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
   );
 };
 
-// These functions are placeholders which I will implement.
+const SourceEvaluationsConfig: React.FC<{
+  sourceEvaluations: FilterAndGroupSourceEvaluationSpec[] | undefined;
+  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+}> = ({sourceEvaluations, updateConfig}) => {
+  const [evaluationNames, setEvaluationNames] = useState<string[]>([]);
+
+  useEffect(() => {
+    fetchEvaluationNames().then(setEvaluationNames);
+  }, []);
+
+  const handleAddSourceEvaluation = () => {
+    updateConfig((spec) => ({
+      ...spec,
+      sourceEvaluations: [...(spec.sourceEvaluations || []), {name: '', version: '*'}],
+    }));
+  };
+
+  return (
+    <Box sx={{mb: 3}}>
+      <Typography variant="h6" gutterBottom>
+        Source Evaluations
+      </Typography>
+      {sourceEvaluations?.map((evaluation, index) => (
+        <SourceEvaluationItem
+          key={index}
+          evaluation={evaluation}
+          evaluationNames={evaluationNames}
+          updateConfig={updateConfig}
+          index={index}
+        />
+      ))}
+      <Button variant="outlined" onClick={handleAddSourceEvaluation}>
+        Add Source Evaluation
+      </Button>
+    </Box>
+  );
+};
+
+const SourceEvaluationItem: React.FC<{
+  evaluation: FilterAndGroupSourceEvaluationSpec;
+  evaluationNames: string[];
+  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+  index: number;
+}> = ({evaluation, evaluationNames, updateConfig, index}) => {
+  const [versions, setVersions] = useState<string[]>([]);
+
+  useEffect(() => {
+    if (evaluation.name) {
+      fetchEvaluationVersionsForName(evaluation.name).then(setVersions);
+    }
+  }, [evaluation.name]);
+
+  const handleNameChange = (event: SelectChangeEvent<string>) => {
+    const newName = event.target.value as string;
+    updateConfig((spec) => ({
+      ...spec,
+      sourceEvaluations: spec.sourceEvaluations?.map((e, i) =>
+        i === index ? {...e, name: newName} : e
+      ),
+    }));
+  };
+
+  const handleVersionChange = (event: SelectChangeEvent<string>) => {
+    const newVersion = event.target.value as string;
+    updateConfig((spec) => ({
+      ...spec,
+      sourceEvaluations: spec.sourceEvaluations?.map((e, i) =>
+        i === index ? {...e, version: newVersion} : e
+      ),
+    }));
+  };
+
+  return (
+    <Box sx={{display: 'flex', gap: 2, mb: 2}}>
+      <FormControl fullWidth>
+        <InputLabel>Name</InputLabel>
+        <Select value={evaluation.name} onChange={handleNameChange}>
+          <MenuItem value="*">All</MenuItem>
+          {evaluationNames.map((name) => (
+            <MenuItem key={name} value={name}>
+              {name}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+      <FormControl fullWidth>
+        <InputLabel>Version</InputLabel>
+        <Select value={evaluation.version} onChange={handleVersionChange}>
+          <MenuItem value="*">All</MenuItem>
+          {versions.map((version) => (
+            <MenuItem key={version} value={version}>
+              {version}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+    </Box>
+  );
+};
+
+const DatasetsConfig: React.FC<{
+  datasets: FilterAndGroupDatasetSpec[] | undefined;
+  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+}> = ({datasets, updateConfig}) => {
+  const [datasetNames, setDatasetNames] = useState<string[]>([]);
+
+  useEffect(() => {
+    fetchDatasetNamesForSpec({}).then(setDatasetNames);
+  }, []);
+
+  const handleAddDataset = () => {
+    updateConfig((spec) => ({
+      ...spec,
+      datasets: [...(spec.datasets || []), {name: '', version: '*', groupAllVersions: false}],
+    }));
+  };
+
+  return (
+    <Box sx={{mb: 3}}>
+      <Typography variant="h6" gutterBottom>
+        Datasets
+      </Typography>
+      {datasets?.map((dataset, index) => (
+        <DatasetItem
+          key={index}
+          dataset={dataset}
+          datasetNames={datasetNames}
+          updateConfig={updateConfig}
+          index={index}
+        />
+      ))}
+      <Button variant="outlined" onClick={handleAddDataset}>
+        Add Dataset
+      </Button>
+    </Box>
+  );
+};
+
+const DatasetItem: React.FC<{
+  dataset: FilterAndGroupDatasetSpec;
+  datasetNames: string[];
+  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+  index: number;
+}> = ({dataset, datasetNames, updateConfig, index}) => {
+  const [versions, setVersions] = useState<string[]>([]);
+
+  useEffect(() => {
+    if (dataset.name) {
+      fetchDatasetVersionsForSpecAndName({}, dataset.name).then(setVersions);
+    }
+  }, [dataset.name]);
+
+    const handleNameChange = (event: SelectChangeEvent<string>) => {
+    const newName = event.target.value as string;
+    updateConfig((spec) => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === index ? {...d, name: newName} : d
+      ),
+    }));
+  };
+
+  const handleVersionChange = (event: SelectChangeEvent<string>) => {
+    const newVersion = event.target.value as string;
+    updateConfig((spec) => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === index ? {...d, version: newVersion} : d
+      ),
+    }));
+  };
+
+  const handleGroupAllVersionsChange = (event: SelectChangeEvent<HTMLInputElement>) => {
+    const newGroupAllVersions = event.target.checked;
+    updateConfig((spec) => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === index ? {...d, groupAllVersions: newGroupAllVersions} : d
+      ),
+    }));
+  };
+
+  return (
+    <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, mb: 2}}>
+      <Box sx={{display: 'flex', gap: 2}}>
+        <FormControl fullWidth>
+          <InputLabel>Name</InputLabel>
+          <Select value={dataset.name} onChange={handleNameChange}>
+            <MenuItem value="*">All</MenuItem>
+            {datasetNames.map((name) => (
+              <MenuItem key={name} value={name}>
+                {name}
+              </MenuItem>
+            ))}
+          </Select>
+        </FormControl>
+        <FormControl fullWidth>
+          <InputLabel>Version</InputLabel>
+          <Select value={dataset.version} onChange={handleVersionChange}>
+            <MenuItem value="*">All</MenuItem>
+            {versions.map((version) => (
+              <MenuItem key={version} value={version}>
+                {version}
+              </MenuItem>
+            ))}
+          </Select>
+        </FormControl>
+      </Box>
+      <FormControlLabel
+        control={
+          <Checkbox
+            checked={dataset.groupAllVersions}
+            onChange={handleGroupAllVersionsChange}
+          />
+        }
+        label="Group All Versions"
+      />
+    </Box>
+  );
+};
+
+const ModelsConfig: React.FC<{
+  models: FilterAndGroupDatasetSpec[] | undefined;
+  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+}> = ({models, updateConfig}) => {
+  const [modelNames, setModelNames] = useState<string[]>([]);
+
+  useEffect(() => {
+    fetchModelNamesForSpec({}).then(setModelNames);
+  }, []);
+
+  const handleAddModel = () => {
+    updateConfig((spec) => ({
+      ...spec,
+      models: [...(spec.models || []), {name: '', version: '*', groupAllVersions: false}],
+    }));
+  };
+
+  return (
+    <Box sx={{mb: 3}}>
+      <Typography variant="h6" gutterBottom>
+        Models
+      </Typography>
+      {models?.map((model, index) => (
+        <ModelItem
+          key={index}
+          model={model}
+          modelNames={modelNames}
+          updateConfig={updateConfig}
+          index={index}
+        />
+      ))}
+      <Button variant="outlined" onClick={handleAddModel}>
+        Add Model
+      </Button>
+    </Box>
+  );
+};
+
+const ModelItem: React.FC<{
+  model: FilterAndGroupDatasetSpec;
+  modelNames: string[];
+  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+  index: number;
+}> = ({model, modelNames, updateConfig, index}) => {
+  const [versions, setVersions] = useState<string[]>([]);
+
+  useEffect(() => {
+    if (model.name) {
+      fetchModelVersionsForSpecndName({}, model.name).then(setVersions);
+    }
+  }, [model.name]);
+
+  const handleNameChange = (event: SelectChangeEvent<string>) => {
+    const newName = event.target.value as string;
+    updateConfig((spec) => ({
+      ...spec,
+      models: spec.models?.map((m, i) =>
+        i === index ? {...m, name: newName} : m
+      ),
+    }));
+  };
+
+  const handleVersionChange = (event: SelectChangeEvent<string>) => {
+    const newVersion = event.target.value as string;
+    updateConfig((spec) => ({
+      ...spec,
+      models: spec.models?.map((m, i) =>
+        i === index ? {...m, version: newVersion} : m
+      ),
+    }));
+  };
+
+  const handleGroupAllVersionsChange = (event: SelectChangeEvent<HTMLInputElement>) => {
+    const newGroupAllVersions = event.target.checked;
+    updateConfig((spec) => ({
+      ...spec,
+      models: spec.models?.map((m, i) =>
+        i === index ? {...m, groupAllVersions: newGroupAllVersions} : m
+      ),
+    }));
+  };
+
+  return (
+    <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, mb: 2}}>
+      <Box sx={{display: 'flex', gap: 2}}>
+        <FormControl fullWidth>
+          <InputLabel>Name</InputLabel>
+          <Select value={model.name} onChange={handleNameChange}>
+            <MenuItem value="*">All</MenuItem>
+            {modelNames.map((name) => (
+              <MenuItem key={name} value={name}>
+                {name}
+              </MenuItem>
+            ))}
+          </Select>
+        </FormControl>
+        <FormControl fullWidth>
+          <InputLabel>Version</InputLabel>
+          <Select value={model.version} onChange={handleVersionChange}>
+            <MenuItem value="*">All</MenuItem>
+            {versions.map((version) => (
+              <MenuItem key={version} value={version}>
+                {version}
+              </MenuItem>
+            ))}
+          </Select>
+        </FormControl>
+      </Box>
+      <FormControlLabel
+        control={
+          <Checkbox
+            checked={model.groupAllVersions}
+            onChange={handleGroupAllVersionsChange}
+          />
+        }
+        label="Group All Versions"
+      />
+    </Box>
+  );
+};
 
 const fetchEvaluationNames = async (): Promise<string[]> => {
   // TODO
@@ -149,5 +515,4 @@ const fetchScorerVersionsForSpecAndName = async (spec: FilterAndGroupSpec, name:
 const fetchMetricPathsForSpec = async (spec: FilterAndGroupSpec): Promise<string[]> => {
     // TODO
     return Promise.resolve([])
-}
-
+}
\ No newline at end of file

From 927253d30daf4d640980097f4e97b1a8f674e8ff Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 11:46:36 -0700
Subject: [PATCH 051/169] AI: Doing the hard work

---
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 450 +++++++++++++++---
 1 file changed, 388 insertions(+), 62 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 36705bf86f2..f9f230e0b16 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -9,13 +9,16 @@ import {
   MenuItem,
   Select,
   SelectChangeEvent,
-  TextField,
   Typography,
 } from '@mui/material';
-import React, {useCallback, useEffect, useState} from 'react';
+import React, {useEffect, useState} from 'react';
 
-import {useWFHooks} from '../wfReactInterface/context';
-import {FilterAndGroupDatasetSpec, FilterAndGroupSourceEvaluationSpec, FilterAndGroupSpec, LeaderboardConfigType} from './LeaderboardConfigType';
+import {
+  FilterAndGroupDatasetSpec,
+  FilterAndGroupSourceEvaluationSpec,
+  FilterAndGroupSpec,
+  LeaderboardConfigType,
+} from './LeaderboardConfigType';
 
 export const LeaderboardConfig: React.FC<{
   entity: string;
@@ -37,8 +40,10 @@ export const LeaderboardConfig: React.FC<{
     onCancel();
   };
 
-  const updateConfig = (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => {
-    setConfig((prevConfig) => ({
+  const updateConfig = (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => {
+    setConfig(prevConfig => ({
       ...prevConfig,
       config: {
         ...prevConfig.config,
@@ -132,7 +137,9 @@ const TempAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
 
 const SourceEvaluationsConfig: React.FC<{
   sourceEvaluations: FilterAndGroupSourceEvaluationSpec[] | undefined;
-  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+  updateConfig: (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => void;
 }> = ({sourceEvaluations, updateConfig}) => {
   const [evaluationNames, setEvaluationNames] = useState<string[]>([]);
 
@@ -141,9 +148,12 @@ const SourceEvaluationsConfig: React.FC<{
   }, []);
 
   const handleAddSourceEvaluation = () => {
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
-      sourceEvaluations: [...(spec.sourceEvaluations || []), {name: '', version: '*'}],
+      sourceEvaluations: [
+        ...(spec.sourceEvaluations || []),
+        {name: '', version: '*'},
+      ],
     }));
   };
 
@@ -171,7 +181,9 @@ const SourceEvaluationsConfig: React.FC<{
 const SourceEvaluationItem: React.FC<{
   evaluation: FilterAndGroupSourceEvaluationSpec;
   evaluationNames: string[];
-  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+  updateConfig: (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => void;
   index: number;
 }> = ({evaluation, evaluationNames, updateConfig, index}) => {
   const [versions, setVersions] = useState<string[]>([]);
@@ -184,7 +196,7 @@ const SourceEvaluationItem: React.FC<{
 
   const handleNameChange = (event: SelectChangeEvent<string>) => {
     const newName = event.target.value as string;
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
       sourceEvaluations: spec.sourceEvaluations?.map((e, i) =>
         i === index ? {...e, name: newName} : e
@@ -194,7 +206,7 @@ const SourceEvaluationItem: React.FC<{
 
   const handleVersionChange = (event: SelectChangeEvent<string>) => {
     const newVersion = event.target.value as string;
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
       sourceEvaluations: spec.sourceEvaluations?.map((e, i) =>
         i === index ? {...e, version: newVersion} : e
@@ -208,7 +220,7 @@ const SourceEvaluationItem: React.FC<{
         <InputLabel>Name</InputLabel>
         <Select value={evaluation.name} onChange={handleNameChange}>
           <MenuItem value="*">All</MenuItem>
-          {evaluationNames.map((name) => (
+          {evaluationNames.map(name => (
             <MenuItem key={name} value={name}>
               {name}
             </MenuItem>
@@ -219,7 +231,7 @@ const SourceEvaluationItem: React.FC<{
         <InputLabel>Version</InputLabel>
         <Select value={evaluation.version} onChange={handleVersionChange}>
           <MenuItem value="*">All</MenuItem>
-          {versions.map((version) => (
+          {versions.map(version => (
             <MenuItem key={version} value={version}>
               {version}
             </MenuItem>
@@ -232,7 +244,9 @@ const SourceEvaluationItem: React.FC<{
 
 const DatasetsConfig: React.FC<{
   datasets: FilterAndGroupDatasetSpec[] | undefined;
-  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+  updateConfig: (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => void;
 }> = ({datasets, updateConfig}) => {
   const [datasetNames, setDatasetNames] = useState<string[]>([]);
 
@@ -241,9 +255,12 @@ const DatasetsConfig: React.FC<{
   }, []);
 
   const handleAddDataset = () => {
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
-      datasets: [...(spec.datasets || []), {name: '', version: '*', groupAllVersions: false}],
+      datasets: [
+        ...(spec.datasets || []),
+        {name: '', version: '*', groupAllVersions: false},
+      ],
     }));
   };
 
@@ -271,20 +288,24 @@ const DatasetsConfig: React.FC<{
 const DatasetItem: React.FC<{
   dataset: FilterAndGroupDatasetSpec;
   datasetNames: string[];
-  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+  updateConfig: (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => void;
   index: number;
 }> = ({dataset, datasetNames, updateConfig, index}) => {
   const [versions, setVersions] = useState<string[]>([]);
+  const [scorerNames, setScorerNames] = useState<string[]>([]);
 
   useEffect(() => {
     if (dataset.name) {
       fetchDatasetVersionsForSpecAndName({}, dataset.name).then(setVersions);
+      fetchScorerNamesForSpec({}).then(setScorerNames);
     }
   }, [dataset.name]);
 
-    const handleNameChange = (event: SelectChangeEvent<string>) => {
+  const handleNameChange = (event: SelectChangeEvent<string>) => {
     const newName = event.target.value as string;
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
       datasets: spec.datasets?.map((d, i) =>
         i === index ? {...d, name: newName} : d
@@ -294,7 +315,7 @@ const DatasetItem: React.FC<{
 
   const handleVersionChange = (event: SelectChangeEvent<string>) => {
     const newVersion = event.target.value as string;
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
       datasets: spec.datasets?.map((d, i) =>
         i === index ? {...d, version: newVersion} : d
@@ -302,9 +323,11 @@ const DatasetItem: React.FC<{
     }));
   };
 
-  const handleGroupAllVersionsChange = (event: SelectChangeEvent<HTMLInputElement>) => {
+  const handleGroupAllVersionsChange = (
+    event: React.ChangeEvent<HTMLInputElement>
+  ) => {
     const newGroupAllVersions = event.target.checked;
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
       datasets: spec.datasets?.map((d, i) =>
         i === index ? {...d, groupAllVersions: newGroupAllVersions} : d
@@ -312,6 +335,23 @@ const DatasetItem: React.FC<{
     }));
   };
 
+  const handleAddScorer = () => {
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === index
+          ? {
+              ...d,
+              scorers: [
+                ...(d.scorers || []),
+                {name: '', version: '*', groupAllVersions: false, metrics: []},
+              ],
+            }
+          : d
+      ),
+    }));
+  };
+
   return (
     <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, mb: 2}}>
       <Box sx={{display: 'flex', gap: 2}}>
@@ -319,7 +359,7 @@ const DatasetItem: React.FC<{
           <InputLabel>Name</InputLabel>
           <Select value={dataset.name} onChange={handleNameChange}>
             <MenuItem value="*">All</MenuItem>
-            {datasetNames.map((name) => (
+            {datasetNames.map(name => (
               <MenuItem key={name} value={name}>
                 {name}
               </MenuItem>
@@ -330,7 +370,7 @@ const DatasetItem: React.FC<{
           <InputLabel>Version</InputLabel>
           <Select value={dataset.version} onChange={handleVersionChange}>
             <MenuItem value="*">All</MenuItem>
-            {versions.map((version) => (
+            {versions.map(version => (
               <MenuItem key={version} value={version}>
                 {version}
               </MenuItem>
@@ -347,13 +387,273 @@ const DatasetItem: React.FC<{
         }
         label="Group All Versions"
       />
+      <Typography variant="subtitle1">Scorers</Typography>
+      {dataset.scorers?.map((scorer, scorerIndex) => (
+        <ScorerItem
+          key={scorerIndex}
+          scorer={scorer}
+          scorerNames={scorerNames}
+          updateConfig={updateConfig}
+          datasetIndex={index}
+          scorerIndex={scorerIndex}
+        />
+      ))}
+      <Button variant="outlined" onClick={handleAddScorer}>
+        Add Scorer
+      </Button>
+    </Box>
+  );
+};
+
+const ScorerItem: React.FC<{
+  scorer: FilterAndGroupDatasetSpec['scorers'][0];
+  scorerNames: string[];
+  updateConfig: (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => void;
+  datasetIndex: number;
+  scorerIndex: number;
+}> = ({scorer, scorerNames, updateConfig, datasetIndex, scorerIndex}) => {
+  const [versions, setVersions] = useState<string[]>([]);
+  const [metricPaths, setMetricPaths] = useState<string[]>([]);
+
+  useEffect(() => {
+    if (scorer.name) {
+      fetchScorerVersionsForSpecAndName({}, scorer.name).then(setVersions);
+      fetchMetricPathsForSpec({}).then(setMetricPaths);
+    }
+  }, [scorer.name]);
+
+  const handleNameChange = (event: SelectChangeEvent<string>) => {
+    const newName = event.target.value as string;
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === datasetIndex
+          ? {
+              ...d,
+              scorers: d.scorers?.map((s, j) =>
+                j === scorerIndex ? {...s, name: newName} : s
+              ),
+            }
+          : d
+      ),
+    }));
+  };
+
+  const handleVersionChange = (event: SelectChangeEvent<string>) => {
+    const newVersion = event.target.value as string;
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === datasetIndex
+          ? {
+              ...d,
+              scorers: d.scorers?.map((s, j) =>
+                j === scorerIndex ? {...s, version: newVersion} : s
+              ),
+            }
+          : d
+      ),
+    }));
+  };
+
+  const handleGroupAllVersionsChange = (
+    event: React.ChangeEvent<HTMLInputElement>
+  ) => {
+    const newGroupAllVersions = event.target.checked;
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === datasetIndex
+          ? {
+              ...d,
+              scorers: d.scorers?.map((s, j) =>
+                j === scorerIndex
+                  ? {...s, groupAllVersions: newGroupAllVersions}
+                  : s
+              ),
+            }
+          : d
+      ),
+    }));
+  };
+
+  const handleAddMetric = () => {
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === datasetIndex
+          ? {
+              ...d,
+              scorers: d.scorers?.map((s, j) =>
+                j === scorerIndex
+                  ? {
+                      ...s,
+                      metrics: [...(s.metrics || []), {path: '', shouldMinimize: false}],
+                    }
+                  : s
+              ),
+            }
+          : d
+      ),
+    }));
+  };
+
+  return (
+    <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, ml: 2, mb: 2}}>
+      <Box sx={{display: 'flex', gap: 2}}>
+        <FormControl fullWidth>
+          <InputLabel>Name</InputLabel>
+          <Select value={scorer.name} onChange={handleNameChange}>
+            <MenuItem value="*">All</MenuItem>
+            {scorerNames.map(name => (
+              <MenuItem key={name} value={name}>
+                {name}
+              </MenuItem>
+            ))}
+          </Select>
+        </FormControl>
+        <FormControl fullWidth>
+          <InputLabel>Version</InputLabel>
+          <Select value={scorer.version} onChange={handleVersionChange}>
+            <MenuItem value="*">All</MenuItem>
+            {versions.map(version => (
+              <MenuItem key={version} value={version}>
+                {version}
+              </MenuItem>
+            ))}
+          </Select>
+        </FormControl>
+      </Box>
+      <FormControlLabel
+        control={
+          <Checkbox
+            checked={scorer.groupAllVersions}
+            onChange={handleGroupAllVersionsChange}
+          />
+        }
+        label="Group All Versions"
+      />
+      <Typography variant="subtitle2">Metrics</Typography>
+      {scorer.metrics?.map((metric, metricIndex) => (
+        <MetricItem
+          key={metricIndex}
+          metric={metric}
+          metricPaths={metricPaths}
+          updateConfig={updateConfig}
+          datasetIndex={datasetIndex}
+          scorerIndex={scorerIndex}
+          metricIndex={metricIndex}
+        />
+      ))}
+      <Button variant="outlined" onClick={handleAddMetric}>
+        Add Metric
+      </Button>
+    </Box>
+  );
+};
+
+const MetricItem: React.FC<{
+  metric: FilterAndGroupDatasetSpec['scorers'][0]['metrics'][0];
+  metricPaths: string[];
+  updateConfig: (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => void;
+  datasetIndex: number;
+  scorerIndex: number;
+  metricIndex: number;
+}> = ({
+  metric,
+  metricPaths,
+  updateConfig,
+  datasetIndex,
+  scorerIndex,
+  metricIndex,
+}) => {
+  const handlePathChange = (event: SelectChangeEvent<string>) => {
+    const newPath = event.target.value as string;
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === datasetIndex
+          ? {
+              ...d,
+              scorers: d.scorers?.map((s, j) =>
+                j === scorerIndex
+                  ? {
+                      ...s,
+                      metrics: s.metrics?.map((m, k) =>
+                        k === metricIndex ? {...m, path: newPath} : m
+                      ),
+                    }
+                  : s
+              ),
+            }
+          : d
+      ),
+    }));
+  };
+
+  const handleShouldMinimizeChange = (
+    event: React.ChangeEvent<HTMLInputElement>
+  ) => {
+    const newShouldMinimize = event.target.checked;
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === datasetIndex
+          ? {
+              ...d,
+              scorers: d.scorers?.map((s, j) =>
+                j === scorerIndex
+                  ? {
+                      ...s,
+                      metrics: s.metrics?.map((m, k) =>
+                        k === metricIndex
+                          ? {...m, shouldMinimize: newShouldMinimize}
+                          : m
+                      ),
+                    }
+                  : s
+              ),
+            }
+          : d
+      ),
+    }));
+  };
+
+  return (
+    <Box sx={{display: 'flex', gap: 2, ml: 2, mb: 2}}>
+      <FormControl fullWidth>
+        <InputLabel>Metric Path</InputLabel>
+        <Select value={metric.path} onChange={handlePathChange}>
+          <MenuItem value="*">All</MenuItem>
+          {metricPaths.map(path => (
+            <MenuItem key={path} value={path}>
+              {path}
+            </MenuItem>
+          ))}
+        </Select>
+      </FormControl>
+      <FormControlLabel
+        control={
+          <Checkbox
+            checked={metric.shouldMinimize}
+            onChange={handleShouldMinimizeChange}
+          />
+        }
+        label="Should Minimize"
+      />
     </Box>
   );
 };
 
 const ModelsConfig: React.FC<{
   models: FilterAndGroupDatasetSpec[] | undefined;
-  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+  updateConfig: (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => void;
 }> = ({models, updateConfig}) => {
   const [modelNames, setModelNames] = useState<string[]>([]);
 
@@ -362,9 +662,12 @@ const ModelsConfig: React.FC<{
   }, []);
 
   const handleAddModel = () => {
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
-      models: [...(spec.models || []), {name: '', version: '*', groupAllVersions: false}],
+      models: [
+        ...(spec.models || []),
+        {name: '', version: '*', groupAllVersions: false},
+      ],
     }));
   };
 
@@ -392,7 +695,9 @@ const ModelsConfig: React.FC<{
 const ModelItem: React.FC<{
   model: FilterAndGroupDatasetSpec;
   modelNames: string[];
-  updateConfig: (updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec) => void;
+  updateConfig: (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => void;
   index: number;
 }> = ({model, modelNames, updateConfig, index}) => {
   const [versions, setVersions] = useState<string[]>([]);
@@ -405,7 +710,7 @@ const ModelItem: React.FC<{
 
   const handleNameChange = (event: SelectChangeEvent<string>) => {
     const newName = event.target.value as string;
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
       models: spec.models?.map((m, i) =>
         i === index ? {...m, name: newName} : m
@@ -415,7 +720,7 @@ const ModelItem: React.FC<{
 
   const handleVersionChange = (event: SelectChangeEvent<string>) => {
     const newVersion = event.target.value as string;
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
       models: spec.models?.map((m, i) =>
         i === index ? {...m, version: newVersion} : m
@@ -423,9 +728,11 @@ const ModelItem: React.FC<{
     }));
   };
 
-  const handleGroupAllVersionsChange = (event: SelectChangeEvent<HTMLInputElement>) => {
+  const handleGroupAllVersionsChange = (
+    event: SelectChangeEvent<HTMLInputElement>
+  ) => {
     const newGroupAllVersions = event.target.checked;
-    updateConfig((spec) => ({
+    updateConfig(spec => ({
       ...spec,
       models: spec.models?.map((m, i) =>
         i === index ? {...m, groupAllVersions: newGroupAllVersions} : m
@@ -440,7 +747,7 @@ const ModelItem: React.FC<{
           <InputLabel>Name</InputLabel>
           <Select value={model.name} onChange={handleNameChange}>
             <MenuItem value="*">All</MenuItem>
-            {modelNames.map((name) => (
+            {modelNames.map(name => (
               <MenuItem key={name} value={name}>
                 {name}
               </MenuItem>
@@ -451,7 +758,7 @@ const ModelItem: React.FC<{
           <InputLabel>Version</InputLabel>
           <Select value={model.version} onChange={handleVersionChange}>
             <MenuItem value="*">All</MenuItem>
-            {versions.map((version) => (
+            {versions.map(version => (
               <MenuItem key={version} value={version}>
                 {version}
               </MenuItem>
@@ -474,45 +781,64 @@ const ModelItem: React.FC<{
 
 const fetchEvaluationNames = async (): Promise<string[]> => {
   // TODO
-  return Promise.resolve([])
-}
+  return Promise.resolve(["E1", "E2", "E3"]);
+};
 
-const fetchEvaluationVersionsForName = async (name: string): Promise<string[]> => {
+const fetchEvaluationVersionsForName = async (
+  name: string
+): Promise<string[]> => {
   // TODO
-  return Promise.resolve([])
-}
+  return Promise.resolve(["EV1", "EV2", "EV3"]);
+};
 
-const fetchDatasetNamesForSpec = async (spec: FilterAndGroupSpec): Promise<string[]> => {
+const fetchDatasetNamesForSpec = async (
+  spec: FilterAndGroupSpec
+): Promise<string[]> => {
   // TODO
-  return Promise.resolve([])
-}
+  return Promise.resolve(["D1", "D2", "D3"]);
+};
 
-const fetchDatasetVersionsForSpecAndName = async (spec: FilterAndGroupSpec, name: string): Promise<string[]> => {
+const fetchDatasetVersionsForSpecAndName = async (
+  spec: FilterAndGroupSpec,
+  name: string
+): Promise<string[]> => {
   // TODO
-  return Promise.resolve([])
-}
+  return Promise.resolve(["DV1", "DV2", "DV3"]);
+};
 
-const fetchModelNamesForSpec = async (spec: FilterAndGroupSpec): Promise<string[]> => {
+const fetchModelNamesForSpec = async (
+  spec: FilterAndGroupSpec
+): Promise<string[]> => {
   // TODO
-  return Promise.resolve([])
-}
+  return Promise.resolve(["M1", "M2", "M3"]);
+};
 
-const fetchModelVersionsForSpecndName = async (spec: FilterAndGroupSpec, name: string): Promise<string[]> => {
+const fetchModelVersionsForSpecndName = async (
+  spec: FilterAndGroupSpec,
+  name: string
+): Promise<string[]> => {
   // TODO
-  return Promise.resolve([])
-}
+  return Promise.resolve(["MV1", "MV2", "MV3"]);
+};
 
-const fetchScorerNamesForSpec = async (spec: FilterAndGroupSpec): Promise<string[]> => {
+const fetchScorerNamesForSpec = async (
+  spec: FilterAndGroupSpec
+): Promise<string[]> => {
   // TODO
-  return Promise.resolve([])
-}
+  return Promise.resolve(["S1", "S2", "S3"]);
+};
 
-const fetchScorerVersionsForSpecAndName = async (spec: FilterAndGroupSpec, name: string): Promise<string[]> => {
+const fetchScorerVersionsForSpecAndName = async (
+  spec: FilterAndGroupSpec,
+  name: string
+): Promise<string[]> => {
   // TODO
-  return Promise.resolve([])
-}
+  return Promise.resolve(["SV1", "SV2", "SV3"]);
+};
 
-const fetchMetricPathsForSpec = async (spec: FilterAndGroupSpec): Promise<string[]> => {
-    // TODO
-    return Promise.resolve([])
-}
\ No newline at end of file
+const fetchMetricPathsForSpec = async (
+  spec: FilterAndGroupSpec
+): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(["MP1", "MP2", "MP3"]);
+};

From 953067161f89518beebecddd0e500a39184dfd76 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 11:51:53 -0700
Subject: [PATCH 052/169] AI: Doing the hard work

---
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 64 ++++++++++++-------
 1 file changed, 40 insertions(+), 24 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index f9f230e0b16..75e73c5fa5d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -189,7 +189,7 @@ const SourceEvaluationItem: React.FC<{
   const [versions, setVersions] = useState<string[]>([]);
 
   useEffect(() => {
-    if (evaluation.name) {
+    if (evaluation.name && evaluation.name !== '*') {
       fetchEvaluationVersionsForName(evaluation.name).then(setVersions);
     }
   }, [evaluation.name]);
@@ -199,7 +199,7 @@ const SourceEvaluationItem: React.FC<{
     updateConfig(spec => ({
       ...spec,
       sourceEvaluations: spec.sourceEvaluations?.map((e, i) =>
-        i === index ? {...e, name: newName} : e
+        i === index ? {...e, name: newName, version: '*'} : e
       ),
     }));
   };
@@ -218,7 +218,7 @@ const SourceEvaluationItem: React.FC<{
     <Box sx={{display: 'flex', gap: 2, mb: 2}}>
       <FormControl fullWidth>
         <InputLabel>Name</InputLabel>
-        <Select value={evaluation.name} onChange={handleNameChange}>
+        <Select value={evaluation.name || '*'} onChange={handleNameChange}>
           <MenuItem value="*">All</MenuItem>
           {evaluationNames.map(name => (
             <MenuItem key={name} value={name}>
@@ -227,9 +227,13 @@ const SourceEvaluationItem: React.FC<{
           ))}
         </Select>
       </FormControl>
-      <FormControl fullWidth>
+      <FormControl fullWidth disabled={evaluation.name === '*'}>
         <InputLabel>Version</InputLabel>
-        <Select value={evaluation.version} onChange={handleVersionChange}>
+        <Select
+          value={evaluation.version}
+          onChange={handleVersionChange}
+          disabled={evaluation.name === '*'}
+        >
           <MenuItem value="*">All</MenuItem>
           {versions.map(version => (
             <MenuItem key={version} value={version}>
@@ -297,10 +301,10 @@ const DatasetItem: React.FC<{
   const [scorerNames, setScorerNames] = useState<string[]>([]);
 
   useEffect(() => {
-    if (dataset.name) {
+    if (dataset.name && dataset.name !== '*') {
       fetchDatasetVersionsForSpecAndName({}, dataset.name).then(setVersions);
-      fetchScorerNamesForSpec({}).then(setScorerNames);
     }
+    fetchScorerNamesForSpec({}).then(setScorerNames);
   }, [dataset.name]);
 
   const handleNameChange = (event: SelectChangeEvent<string>) => {
@@ -308,7 +312,7 @@ const DatasetItem: React.FC<{
     updateConfig(spec => ({
       ...spec,
       datasets: spec.datasets?.map((d, i) =>
-        i === index ? {...d, name: newName} : d
+        i === index ? {...d, name: newName, version: '*'} : d
       ),
     }));
   };
@@ -357,7 +361,7 @@ const DatasetItem: React.FC<{
       <Box sx={{display: 'flex', gap: 2}}>
         <FormControl fullWidth>
           <InputLabel>Name</InputLabel>
-          <Select value={dataset.name} onChange={handleNameChange}>
+          <Select value={dataset.name || '*'} onChange={handleNameChange}>
             <MenuItem value="*">All</MenuItem>
             {datasetNames.map(name => (
               <MenuItem key={name} value={name}>
@@ -366,9 +370,13 @@ const DatasetItem: React.FC<{
             ))}
           </Select>
         </FormControl>
-        <FormControl fullWidth>
+        <FormControl fullWidth disabled={dataset.name === '*'}>
           <InputLabel>Version</InputLabel>
-          <Select value={dataset.version} onChange={handleVersionChange}>
+          <Select
+            value={dataset.version}
+            onChange={handleVersionChange}
+            disabled={dataset.name === '*'}
+          >
             <MenuItem value="*">All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version} value={version}>
@@ -418,10 +426,10 @@ const ScorerItem: React.FC<{
   const [metricPaths, setMetricPaths] = useState<string[]>([]);
 
   useEffect(() => {
-    if (scorer.name) {
+    if (scorer.name && scorer.name !== '*') {
       fetchScorerVersionsForSpecAndName({}, scorer.name).then(setVersions);
-      fetchMetricPathsForSpec({}).then(setMetricPaths);
     }
+    fetchMetricPathsForSpec({}).then(setMetricPaths);
   }, [scorer.name]);
 
   const handleNameChange = (event: SelectChangeEvent<string>) => {
@@ -433,7 +441,7 @@ const ScorerItem: React.FC<{
           ? {
               ...d,
               scorers: d.scorers?.map((s, j) =>
-                j === scorerIndex ? {...s, name: newName} : s
+                j === scorerIndex ? {...s, name: newName, version: '*'} : s
               ),
             }
           : d
@@ -505,7 +513,7 @@ const ScorerItem: React.FC<{
       <Box sx={{display: 'flex', gap: 2}}>
         <FormControl fullWidth>
           <InputLabel>Name</InputLabel>
-          <Select value={scorer.name} onChange={handleNameChange}>
+          <Select value={scorer.name || '*'} onChange={handleNameChange}>
             <MenuItem value="*">All</MenuItem>
             {scorerNames.map(name => (
               <MenuItem key={name} value={name}>
@@ -514,9 +522,13 @@ const ScorerItem: React.FC<{
             ))}
           </Select>
         </FormControl>
-        <FormControl fullWidth>
+        <FormControl fullWidth disabled={scorer.name === '*'}>
           <InputLabel>Version</InputLabel>
-          <Select value={scorer.version} onChange={handleVersionChange}>
+          <Select
+            value={scorer.version}
+            onChange={handleVersionChange}
+            disabled={scorer.name === '*'}
+          >
             <MenuItem value="*">All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version} value={version}>
@@ -703,7 +715,7 @@ const ModelItem: React.FC<{
   const [versions, setVersions] = useState<string[]>([]);
 
   useEffect(() => {
-    if (model.name) {
+    if (model.name && model.name !== '*') {
       fetchModelVersionsForSpecndName({}, model.name).then(setVersions);
     }
   }, [model.name]);
@@ -713,7 +725,7 @@ const ModelItem: React.FC<{
     updateConfig(spec => ({
       ...spec,
       models: spec.models?.map((m, i) =>
-        i === index ? {...m, name: newName} : m
+        i === index ? {...m, name: newName, version: '*'} : m
       ),
     }));
   };
@@ -729,7 +741,7 @@ const ModelItem: React.FC<{
   };
 
   const handleGroupAllVersionsChange = (
-    event: SelectChangeEvent<HTMLInputElement>
+    event: React.ChangeEvent<HTMLInputElement>
   ) => {
     const newGroupAllVersions = event.target.checked;
     updateConfig(spec => ({
@@ -745,7 +757,7 @@ const ModelItem: React.FC<{
       <Box sx={{display: 'flex', gap: 2}}>
         <FormControl fullWidth>
           <InputLabel>Name</InputLabel>
-          <Select value={model.name} onChange={handleNameChange}>
+          <Select value={model.name || '*'} onChange={handleNameChange}>
             <MenuItem value="*">All</MenuItem>
             {modelNames.map(name => (
               <MenuItem key={name} value={name}>
@@ -754,9 +766,13 @@ const ModelItem: React.FC<{
             ))}
           </Select>
         </FormControl>
-        <FormControl fullWidth>
+        <FormControl fullWidth disabled={model.name === '*'}>
           <InputLabel>Version</InputLabel>
-          <Select value={model.version} onChange={handleVersionChange}>
+          <Select
+            value={model.version}
+            onChange={handleVersionChange}
+            disabled={model.name === '*'}
+          >
             <MenuItem value="*">All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version} value={version}>
@@ -841,4 +857,4 @@ const fetchMetricPathsForSpec = async (
 ): Promise<string[]> => {
   // TODO
   return Promise.resolve(["MP1", "MP2", "MP3"]);
-};
+};
\ No newline at end of file

From 1d848709906f849993cbdc751488d1f966f7d096 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 11:58:56 -0700
Subject: [PATCH 053/169] AI: Doing the hard work 3

---
 .../LeaderboardPage/LeaderboardPageConfig.tsx | 114 ++++++++++--------
 1 file changed, 63 insertions(+), 51 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
index 75e73c5fa5d..3371d8b47d1 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
@@ -10,6 +10,8 @@ import {
   Select,
   SelectChangeEvent,
   Typography,
+  Tabs,
+  Tab,
 } from '@mui/material';
 import React, {useEffect, useState} from 'react';
 
@@ -30,7 +32,7 @@ export const LeaderboardConfig: React.FC<{
     updater: (config: LeaderboardConfigType) => LeaderboardConfigType
   ) => void;
 }> = ({entity, project, config, setConfig, onPersist, onCancel}) => {
-  const [showAlert, setShowAlert] = useState(true);
+  const [activeTab, setActiveTab] = useState(0);
 
   const handleSave = () => {
     onPersist();
@@ -52,6 +54,10 @@ export const LeaderboardConfig: React.FC<{
     }));
   };
 
+  const handleTabChange = (event: React.SyntheticEvent, newValue: number) => {
+    setActiveTab(newValue);
+  };
+
   return (
     <Box
       sx={{
@@ -61,62 +67,34 @@ export const LeaderboardConfig: React.FC<{
         flexDirection: 'column',
         border: '1px solid #e0e0e0',
       }}>
-      <Box
-        sx={{
-          overflowY: 'auto',
-          p: 2,
-        }}>
-        {showAlert && <TempAlert onClose={() => setShowAlert(false)} />}
-        <Typography variant="h5" gutterBottom>
-          Leaderboard Configuration
-        </Typography>
+      <Box sx={{borderBottom: 1, borderColor: 'divider'}}>
+        <Tabs value={activeTab} onChange={handleTabChange}>
+          <Tab label="Config Editor" />
+          <Tab label="Config Preview" />
+        </Tabs>
       </Box>
 
-      <Box sx={{mt: 2, mb: 2, flex: 1, overflowY: 'auto', px: 2}}>
-        <SourceEvaluationsConfig
-          sourceEvaluations={config.config.dataSelectionSpec.sourceEvaluations}
-          updateConfig={updateConfig}
-        />
-        <DatasetsConfig
-          datasets={config.config.dataSelectionSpec.datasets}
-          updateConfig={updateConfig}
-        />
-        <ModelsConfig
-          models={config.config.dataSelectionSpec.models}
-          updateConfig={updateConfig}
-        />
-      </Box>
-
-      <Box sx={{mt: 2, mb: 2, px: 2}}>
-        <Typography variant="h6" gutterBottom>
-          Configuration Preview
-        </Typography>
-        <pre
-          style={{
-            backgroundColor: '#f5f5f5',
-            padding: '10px',
-            borderRadius: '4px',
-            overflowX: 'auto',
-            maxHeight: '400px',
-            whiteSpace: 'pre-wrap',
-            wordWrap: 'break-word',
-          }}>
-          {JSON.stringify(config, null, 2)}
-        </pre>
+      <Box sx={{flex: 1, overflowY: 'auto', p: 2}}>
+        {activeTab === 0 && (
+          <ConfigEditor
+            config={config.config.dataSelectionSpec}
+            updateConfig={updateConfig}
+          />
+        )}
+        {activeTab === 1 && <ConfigPreview config={config} />}
       </Box>
 
       <Box
         sx={{
           display: 'flex',
           justifyContent: 'flex-end',
-          height: '51px',
-          p: 1,
+          p: 2,
           borderTop: '1px solid #e0e0e0',
         }}>
         <Button variant="outlined" onClick={handleCancel} sx={{mr: 2}}>
           Cancel
         </Button>
-        <Button variant="contained" onClick={handleSave} sx={{mr: 2}}>
+        <Button variant="contained" onClick={handleSave}>
           Save
         </Button>
       </Box>
@@ -124,14 +102,48 @@ export const LeaderboardConfig: React.FC<{
   );
 };
 
-const TempAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
+const ConfigEditor: React.FC<{
+  config: FilterAndGroupSpec;
+  updateConfig: (
+    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
+  ) => void;
+}> = ({config, updateConfig}) => {
   return (
-    <Alert severity="info" onClose={onClose}>
-      <Typography variant="body1">
-        Configuration edtior purely for internal exploration, not for production
-        use.
-      </Typography>
-    </Alert>
+    <>
+      <SourceEvaluationsConfig
+        sourceEvaluations={config.sourceEvaluations}
+        updateConfig={updateConfig}
+      />
+      <DatasetsConfig
+        datasets={config.datasets}
+        updateConfig={updateConfig}
+      />
+      <ModelsConfig
+        models={config.models}
+        updateConfig={updateConfig}
+      />
+    </>
+  );
+};
+
+const ConfigPreview: React.FC<{
+  config: LeaderboardConfigType;
+}> = ({config}) => {
+  return (
+    <Box>
+      <pre
+        style={{
+          backgroundColor: '#f5f5f5',
+          padding: '10px',
+          borderRadius: '4px',
+          overflowX: 'auto',
+          maxHeight: 'calc(100vh - 200px)',
+          whiteSpace: 'pre-wrap',
+          wordWrap: 'break-word',
+        }}>
+        {JSON.stringify(config, null, 2)}
+      </pre>
+    </Box>
   );
 };
 

From 344876390075e62826476bbaa26890b59207d9b9 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 13:25:25 -0700
Subject: [PATCH 054/169] Human: Name refactorrs

---
 ...Config.tsx => LeaderboardConfigEditor.tsx} | 111 ++++--------------
 .../pages/LeaderboardPage/LeaderboardGrid.tsx |   2 +-
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  12 +-
 .../query/configEditorQuery.ts                |  57 +++++++++
 .../{hooks.ts => query/hookAdapters.ts}       |   9 +-
 .../leaderboardQuery.ts}                      |  16 +--
 .../leaderboardConfigType.ts}                 |   0
 7 files changed, 99 insertions(+), 108 deletions(-)
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/{LeaderboardPageConfig.tsx => LeaderboardConfigEditor.tsx} (91%)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/{hooks.ts => query/hookAdapters.ts} (75%)
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/{leaderboardServerInterface.ts => query/leaderboardQuery.ts} (96%)
 rename weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/{LeaderboardConfigType.tsx => types/leaderboardConfigType.ts} (100%)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
similarity index 91%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 3371d8b47d1..078dea90054 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPageConfig.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -1,5 +1,4 @@
 import {
-  Alert,
   Box,
   Button,
   Checkbox,
@@ -9,20 +8,31 @@ import {
   MenuItem,
   Select,
   SelectChangeEvent,
-  Typography,
-  Tabs,
   Tab,
+  Tabs,
+  Typography,
 } from '@mui/material';
 import React, {useEffect, useState} from 'react';
 
+import {
+  fetchDatasetNamesForSpec,
+  fetchDatasetVersionsForSpecAndName,
+  fetchEvaluationNames,
+  fetchEvaluationVersionsForName,
+  fetchMetricPathsForSpec,
+  fetchModelNamesForSpec,
+  fetchModelVersionsForSpecndName,
+  fetchScorerNamesForSpec,
+  fetchScorerVersionsForSpecAndName,
+} from './query/configEditorQuery';
 import {
   FilterAndGroupDatasetSpec,
   FilterAndGroupSourceEvaluationSpec,
   FilterAndGroupSpec,
   LeaderboardConfigType,
-} from './LeaderboardConfigType';
+} from './types/leaderboardConfigType';
 
-export const LeaderboardConfig: React.FC<{
+export const LeaderboardConfigEditor: React.FC<{
   entity: string;
   project: string;
   config: LeaderboardConfigType;
@@ -114,14 +124,8 @@ const ConfigEditor: React.FC<{
         sourceEvaluations={config.sourceEvaluations}
         updateConfig={updateConfig}
       />
-      <DatasetsConfig
-        datasets={config.datasets}
-        updateConfig={updateConfig}
-      />
-      <ModelsConfig
-        models={config.models}
-        updateConfig={updateConfig}
-      />
+      <DatasetsConfig datasets={config.datasets} updateConfig={updateConfig} />
+      <ModelsConfig models={config.models} updateConfig={updateConfig} />
     </>
   );
 };
@@ -244,8 +248,7 @@ const SourceEvaluationItem: React.FC<{
         <Select
           value={evaluation.version}
           onChange={handleVersionChange}
-          disabled={evaluation.name === '*'}
-        >
+          disabled={evaluation.name === '*'}>
           <MenuItem value="*">All</MenuItem>
           {versions.map(version => (
             <MenuItem key={version} value={version}>
@@ -387,8 +390,7 @@ const DatasetItem: React.FC<{
           <Select
             value={dataset.version}
             onChange={handleVersionChange}
-            disabled={dataset.name === '*'}
-          >
+            disabled={dataset.name === '*'}>
             <MenuItem value="*">All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version} value={version}>
@@ -510,7 +512,10 @@ const ScorerItem: React.FC<{
                 j === scorerIndex
                   ? {
                       ...s,
-                      metrics: [...(s.metrics || []), {path: '', shouldMinimize: false}],
+                      metrics: [
+                        ...(s.metrics || []),
+                        {path: '', shouldMinimize: false},
+                      ],
                     }
                   : s
               ),
@@ -539,8 +544,7 @@ const ScorerItem: React.FC<{
           <Select
             value={scorer.version}
             onChange={handleVersionChange}
-            disabled={scorer.name === '*'}
-          >
+            disabled={scorer.name === '*'}>
             <MenuItem value="*">All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version} value={version}>
@@ -783,8 +787,7 @@ const ModelItem: React.FC<{
           <Select
             value={model.version}
             onChange={handleVersionChange}
-            disabled={model.name === '*'}
-          >
+            disabled={model.name === '*'}>
             <MenuItem value="*">All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version} value={version}>
@@ -806,67 +809,3 @@ const ModelItem: React.FC<{
     </Box>
   );
 };
-
-const fetchEvaluationNames = async (): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(["E1", "E2", "E3"]);
-};
-
-const fetchEvaluationVersionsForName = async (
-  name: string
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(["EV1", "EV2", "EV3"]);
-};
-
-const fetchDatasetNamesForSpec = async (
-  spec: FilterAndGroupSpec
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(["D1", "D2", "D3"]);
-};
-
-const fetchDatasetVersionsForSpecAndName = async (
-  spec: FilterAndGroupSpec,
-  name: string
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(["DV1", "DV2", "DV3"]);
-};
-
-const fetchModelNamesForSpec = async (
-  spec: FilterAndGroupSpec
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(["M1", "M2", "M3"]);
-};
-
-const fetchModelVersionsForSpecndName = async (
-  spec: FilterAndGroupSpec,
-  name: string
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(["MV1", "MV2", "MV3"]);
-};
-
-const fetchScorerNamesForSpec = async (
-  spec: FilterAndGroupSpec
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(["S1", "S2", "S3"]);
-};
-
-const fetchScorerVersionsForSpecAndName = async (
-  spec: FilterAndGroupSpec,
-  name: string
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(["SV1", "SV2", "SV3"]);
-};
-
-const fetchMetricPathsForSpec = async (
-  spec: FilterAndGroupSpec
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(["MP1", "MP2", "MP3"]);
-};
\ No newline at end of file
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index 362f021b8e2..e8b827cc692 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -20,7 +20,7 @@ import {
   GroupedLeaderboardData,
   GroupedLeaderboardModelGroup,
   LeaderboardValueRecord,
-} from './leaderboardServerInterface';
+} from './query/leaderboardQuery';
 
 interface LeaderboardGridProps {
   entity: string;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 2ff1ed41d99..59412bbdf2e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -5,11 +5,11 @@ import {useHistory} from 'react-router-dom';
 
 import {useWeaveflowRouteContext} from '../../context';
 import {EditableMarkdown} from './EditableMarkdown';
-import {useLeaderboardData} from './hooks';
-import {LeaderboardConfigType} from './LeaderboardConfigType';
+import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
 import {LeaderboardGrid} from './LeaderboardGrid';
-import {LeaderboardConfig} from './LeaderboardPageConfig';
-import {LeaderboardValueRecord} from './leaderboardServerInterface';
+import {useLeaderboardData} from './query/hookAdapters';
+import {LeaderboardValueRecord} from './query/leaderboardQuery';
+import {LeaderboardConfigType} from './types/leaderboardConfigType';
 
 const USE_COMPARE_EVALUATIONS_PAGE = false;
 
@@ -133,7 +133,7 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
           onCellClick={handleCellClick}
         />
         {showConfig && (
-          <LeaderboardConfig
+          <LeaderboardConfigEditor
             entity={entity}
             project={project}
             config={currentConfig}
@@ -187,6 +187,4 @@ const UnlistedAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
 };
 
 // TODO:
-// * [ ] Make the editable context (i think we should have an object and version and isEditing, etc... Think more like reports)
 // * [ ] Edit panel revisions
-// * [ ] UX Cleanup
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
new file mode 100644
index 00000000000..dfe3c7deda3
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
@@ -0,0 +1,57 @@
+import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
+
+export const fetchEvaluationNames = async (): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(['E1', 'E2', 'E3']);
+};
+export const fetchEvaluationVersionsForName = async (
+  name: string
+): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(['EV1', 'EV2', 'EV3']);
+};
+export const fetchDatasetNamesForSpec = async (
+  spec: FilterAndGroupSpec
+): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(['D1', 'D2', 'D3']);
+};
+export const fetchDatasetVersionsForSpecAndName = async (
+  spec: FilterAndGroupSpec,
+  name: string
+): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(['DV1', 'DV2', 'DV3']);
+};
+export const fetchModelNamesForSpec = async (
+  spec: FilterAndGroupSpec
+): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(['M1', 'M2', 'M3']);
+};
+export const fetchModelVersionsForSpecndName = async (
+  spec: FilterAndGroupSpec,
+  name: string
+): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(['MV1', 'MV2', 'MV3']);
+};
+export const fetchScorerNamesForSpec = async (
+  spec: FilterAndGroupSpec
+): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(['S1', 'S2', 'S3']);
+};
+export const fetchScorerVersionsForSpecAndName = async (
+  spec: FilterAndGroupSpec,
+  name: string
+): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(['SV1', 'SV2', 'SV3']);
+};
+export const fetchMetricPathsForSpec = async (
+  spec: FilterAndGroupSpec
+): Promise<string[]> => {
+  // TODO
+  return Promise.resolve(['MP1', 'MP2', 'MP3']);
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/hookAdapters.ts
similarity index 75%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/hookAdapters.ts
index 7ff48cfcca8..47caaccbf98 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/hooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/hookAdapters.ts
@@ -1,11 +1,8 @@
 import {useEffect, useState} from 'react';
 
-import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
-import {FilterAndGroupSpec} from './LeaderboardConfigType';
-import {
-  getLeaderboardData,
-  GroupedLeaderboardData,
-} from './leaderboardServerInterface';
+import {useGetTraceServerClientContext} from '../../wfReactInterface/traceServerClientContext';
+import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
+import {getLeaderboardData, GroupedLeaderboardData} from './leaderboardQuery';
 
 type LeaderboardDataState = {
   loading: boolean;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
similarity index 96%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
index e3501d13d85..0bbda3b86eb 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardServerInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
@@ -1,20 +1,20 @@
 import {isWeaveObjectRef} from '@wandb/weave/react';
 import _ from 'lodash';
 
-import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
-import {parseRefMaybe} from '../../../Browse2/SmallRef';
-import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
-import {TraceServerClient} from '../wfReactInterface/traceServerClient';
-import {TraceObjSchema} from '../wfReactInterface/traceServerClientTypes';
+import {flattenObjectPreservingWeaveTypes} from '../../../../Browse2/browse2Util';
+import {parseRefMaybe} from '../../../../Browse2/SmallRef';
+import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../../common/heuristics';
+import {TraceServerClient} from '../../wfReactInterface/traceServerClient';
+import {TraceObjSchema} from '../../wfReactInterface/traceServerClientTypes';
 import {
   convertISOToDate,
   projectIdFromParts,
-} from '../wfReactInterface/tsDataModelHooks';
+} from '../../wfReactInterface/tsDataModelHooks';
 import {
   objectVersionKeyToRefUri,
   opVersionKeyToRefUri,
-} from '../wfReactInterface/utilities';
-import {FilterAndGroupSpec} from './LeaderboardConfigType';
+} from '../../wfReactInterface/utilities';
+import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
 
 export type LeaderboardValueRecord = {
   datasetName: string;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts
similarity index 100%
rename from weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigType.tsx
rename to weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts

From de45ea2df343eabf80ac48d2c6224fe983665a7d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 13:28:17 -0700
Subject: [PATCH 055/169] Human: Name refactorrs - lint

---
 .../pages/LeaderboardPage/LeaderboardConfigEditor.tsx       | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 078dea90054..9cebc5dc021 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -26,6 +26,8 @@ import {
   fetchScorerVersionsForSpecAndName,
 } from './query/configEditorQuery';
 import {
+  FilterAndGroupDatasetScorerMetricSpec,
+  FilterAndGroupDatasetScorerSpec,
   FilterAndGroupDatasetSpec,
   FilterAndGroupSourceEvaluationSpec,
   FilterAndGroupSpec,
@@ -428,7 +430,7 @@ const DatasetItem: React.FC<{
 };
 
 const ScorerItem: React.FC<{
-  scorer: FilterAndGroupDatasetSpec['scorers'][0];
+  scorer: FilterAndGroupDatasetScorerSpec;
   scorerNames: string[];
   updateConfig: (
     updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
@@ -583,7 +585,7 @@ const ScorerItem: React.FC<{
 };
 
 const MetricItem: React.FC<{
-  metric: FilterAndGroupDatasetSpec['scorers'][0]['metrics'][0];
+  metric: FilterAndGroupDatasetScorerMetricSpec;
   metricPaths: string[];
   updateConfig: (
     updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec

From 785c29ef4afb3296893770950e57b7f1bc9c09de Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 13:39:45 -0700
Subject: [PATCH 056/169] Human: Better sidebar

---
 .../LeaderboardConfigEditor.tsx               |  7 +-
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 75 ++++++++++---------
 2 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 9cebc5dc021..7aca64572e0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -73,7 +73,7 @@ export const LeaderboardConfigEditor: React.FC<{
   return (
     <Box
       sx={{
-        width: '50%',
+        width: '100%',
         height: '100%',
         display: 'flex',
         flexDirection: 'column',
@@ -100,8 +100,11 @@ export const LeaderboardConfigEditor: React.FC<{
         sx={{
           display: 'flex',
           justifyContent: 'flex-end',
-          p: 2,
+          pt: 1,
+          pb: 1,
+          pr: 1.5,
           borderTop: '1px solid #e0e0e0',
+          height: '51px',
         }}>
         <Button variant="outlined" onClick={handleCancel} sx={{mr: 2}}>
           Cancel
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 59412bbdf2e..28efa602456 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -99,40 +99,47 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   const [showingAlert, setShowingAlert] = useState(true);
 
   return (
-    <Box display="flex" flexDirection="column" height="100%">
-      {showingAlert && <UnlistedAlert onClose={() => setShowingAlert(false)} />}
-      <div
-        style={{
-          position: 'absolute',
-          top: 20 + (showingAlert ? 52 : 0),
-          right: 24,
-        }}>
-        <ToggleLeaderboardConfig
-          isOpen={showConfig}
-          onClick={() => setShowConfig(c => !c)}
-        />
-      </div>
-      <Box flexShrink={0} maxHeight="35%" overflow="auto">
-        <EditableMarkdown
-          value={description}
-          onChange={setDescription}
-          placeholder={DEFAULT_DESCRIPTION}
-        />
-      </Box>
+    <Box display="flex" flexDirection="row" height="100%" flexGrow={1}>
       <Box
-        flexGrow={1}
+        flex={1}
         display="flex"
-        flexDirection="row"
-        overflow="hidden"
-        minHeight="65%">
-        <LeaderboardGrid
-          entity={entity}
-          project={project}
-          loading={loading}
-          data={data}
-          onCellClick={handleCellClick}
-        />
-        {showConfig && (
+        flexDirection="column"
+        height="100%"
+        minWidth="65%">
+        <div
+          style={{
+            position: 'absolute',
+            display: showConfig ? 'none' : 'block',
+            top: 20,
+            right: 24,
+          }}>
+          <ToggleLeaderboardConfig
+            isOpen={showConfig}
+            onClick={() => setShowConfig(c => !c)}
+          />
+        </div>
+        <Box flexShrink={0} maxHeight="35%" overflow="auto">
+          <EditableMarkdown
+            value={description}
+            onChange={setDescription}
+            placeholder={DEFAULT_DESCRIPTION}
+          />
+        </Box>
+        {showingAlert && (
+          <UnlistedAlert onClose={() => setShowingAlert(false)} />
+        )}
+        <Box flexGrow={1} display="flex" flexDirection="row" overflow="hidden">
+          <LeaderboardGrid
+            entity={entity}
+            project={project}
+            loading={loading}
+            data={data}
+            onCellClick={handleCellClick}
+          />
+        </Box>
+      </Box>
+      {showConfig && (
+        <Box flex={1} width="35%" height="100%" overflow="hidden">
           <LeaderboardConfigEditor
             entity={entity}
             project={project}
@@ -147,8 +154,8 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
             }}
             setConfig={setConfigLocal}
           />
-        )}
-      </Box>
+        </Box>
+      )}
     </Box>
   );
 };

From 19f5107f94a023ebf8e01c228d4863ce65bb83c8 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 14:10:04 -0700
Subject: [PATCH 057/169] Human: Nearly there - just need to do data fetch now

---
 .../LeaderboardPage/EditableMarkdown.tsx      | 26 ++++--
 .../LeaderboardConfigEditor.tsx               | 86 ++++++++++++++++++-
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 43 ++++++----
 3 files changed, 130 insertions(+), 25 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
index 12d3262044b..12705870a1b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
@@ -1,6 +1,13 @@
 import {Box, TextField, Typography} from '@mui/material';
-import React, {useCallback, useLayoutEffect, useRef, useState} from 'react';
+import React, {
+  useCallback,
+  useEffect,
+  useLayoutEffect,
+  useRef,
+  useState,
+} from 'react';
 import ReactMarkdown from 'react-markdown';
+import styled from 'styled-components';
 
 interface EditableMarkdownProps {
   value: string;
@@ -8,6 +15,12 @@ interface EditableMarkdownProps {
   placeholder: string;
 }
 
+const StyledReactMarkdown = styled(ReactMarkdown)`
+  > *:first-child {
+    margin-top: 0;
+  }
+`;
+
 export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
   value,
   onChange,
@@ -75,6 +88,12 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
     });
   };
 
+  useEffect(() => {
+    requestAnimationFrame(() => {
+      updateHeight();
+    });
+  }, [updateHeight, value]);
+
   return (
     <Box ref={containerRef} sx={{padding: '16px', transition: 'height 0.3s'}}>
       {isEditing ? (
@@ -84,9 +103,6 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
           value={value}
           onChange={e => {
             onChange(e.target.value);
-            requestAnimationFrame(() => {
-              updateHeight();
-            });
           }}
           onBlur={handleBlur}
           placeholder={placeholder}
@@ -114,7 +130,7 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
             },
           }}
           onClick={handleEdit}>
-          <ReactMarkdown>{value || placeholder}</ReactMarkdown>
+          <StyledReactMarkdown>{value || placeholder}</StyledReactMarkdown>
         </Typography>
       )}
     </Box>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 7aca64572e0..d2ed94eb4ca 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -12,7 +12,7 @@ import {
   Tabs,
   Typography,
 } from '@mui/material';
-import React, {useEffect, useState} from 'react';
+import React, {useEffect, useMemo, useState} from 'react';
 
 import {
   fetchDatasetNamesForSpec,
@@ -66,6 +66,16 @@ export const LeaderboardConfigEditor: React.FC<{
     }));
   };
 
+  const updateDescription = (newDescription: string) => {
+    setConfig(prevConfig => ({
+      ...prevConfig,
+      config: {
+        ...prevConfig.config,
+        description: newDescription,
+      },
+    }));
+  };
+
   const handleTabChange = (event: React.SyntheticEvent, newValue: number) => {
     setActiveTab(newValue);
   };
@@ -81,7 +91,8 @@ export const LeaderboardConfigEditor: React.FC<{
       }}>
       <Box sx={{borderBottom: 1, borderColor: 'divider'}}>
         <Tabs value={activeTab} onChange={handleTabChange}>
-          <Tab label="Config Editor" />
+          <Tab label="Editor" />
+          <Tab label="Description" />
           <Tab label="Config Preview" />
         </Tabs>
       </Box>
@@ -93,7 +104,13 @@ export const LeaderboardConfigEditor: React.FC<{
             updateConfig={updateConfig}
           />
         )}
-        {activeTab === 1 && <ConfigPreview config={config} />}
+        {activeTab === 1 && (
+          <DescriptionEditor
+            description={config.config.description}
+            updateDescription={updateDescription}
+          />
+        )}
+        {activeTab === 2 && <ConfigPreview config={config} />}
       </Box>
 
       <Box
@@ -156,6 +173,69 @@ const ConfigPreview: React.FC<{
   );
 };
 
+const DescriptionEditor: React.FC<{
+  description: string;
+  updateDescription: (newDescription: string) => void;
+}> = ({description, updateDescription}) => {
+  const [localDescription, setLocalDescription] = useState(description);
+
+  const debouncedUpdate = useMemo(
+    () =>
+      debounce((value: string) => {
+        updateDescription(value);
+      }, 300),
+    [updateDescription]
+  );
+
+  useEffect(() => {
+    setLocalDescription(description);
+  }, [description]);
+
+  const handleChange = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
+    const newValue = event.target.value;
+    setLocalDescription(newValue);
+    debouncedUpdate(newValue);
+  };
+
+  return (
+    <Box sx={{height: 'calc(100vh - 200px)'}}>
+      <textarea
+        value={localDescription}
+        onChange={handleChange}
+        style={{
+          width: '100%',
+          height: '100%',
+          fontFamily: 'monospace',
+          fontSize: '14px',
+          padding: '10px',
+          border: '1px solid #ccc',
+          borderRadius: '4px',
+          resize: 'none',
+          backgroundColor: '#f5f5f5',
+          lineHeight: '1.5',
+          whiteSpace: 'pre-wrap',
+          overflowWrap: 'break-word',
+        }}
+        placeholder="Enter markdown description here..."
+      />
+    </Box>
+  );
+};
+
+// Debounce function
+function debounce<T extends (...args: any[]) => any>(
+  func: T,
+  wait: number
+): (...args: Parameters<T>) => void {
+  let timeout: NodeJS.Timeout | null = null;
+  return (...args: Parameters<T>) => {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+    timeout = setTimeout(() => func(...args), wait);
+  };
+}
+
 const SourceEvaluationsConfig: React.FC<{
   sourceEvaluations: FilterAndGroupSourceEvaluationSpec[] | undefined;
   updateConfig: (
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 28efa602456..e115edb2300 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -106,24 +106,33 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
         flexDirection="column"
         height="100%"
         minWidth="65%">
-        <div
-          style={{
-            position: 'absolute',
-            display: showConfig ? 'none' : 'block',
-            top: 20,
-            right: 24,
+        <Box
+          flex={1}
+          display="flex"
+          flexDirection="row"
+          maxHeight="35%"
+          width="100%"
+          sx={{
+            alignItems: 'flex-start',
           }}>
-          <ToggleLeaderboardConfig
-            isOpen={showConfig}
-            onClick={() => setShowConfig(c => !c)}
-          />
-        </div>
-        <Box flexShrink={0} maxHeight="35%" overflow="auto">
-          <EditableMarkdown
-            value={description}
-            onChange={setDescription}
-            placeholder={DEFAULT_DESCRIPTION}
-          />
+          <Box flexShrink={0} flexGrow={1} overflow="auto">
+            <EditableMarkdown
+              value={description}
+              onChange={setDescription}
+              placeholder={DEFAULT_DESCRIPTION}
+            />
+          </Box>
+          <div
+            style={{
+              display: showConfig ? 'none' : 'block',
+              paddingRight: '16px',
+              paddingTop: '16px',
+            }}>
+            <ToggleLeaderboardConfig
+              isOpen={showConfig}
+              onClick={() => setShowConfig(c => !c)}
+            />
+          </div>
         </Box>
         {showingAlert && (
           <UnlistedAlert onClose={() => setShowingAlert(false)} />

From 4142f046d31ba4fa2c570cec5d17d923544b0026 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 14:37:11 -0700
Subject: [PATCH 058/169] Human: A bunch of little style changes because i am
 crazy ocd

---
 .../pages/LeaderboardPage/EditableMarkdown.tsx | 18 ++++++++++++++----
 .../pages/LeaderboardPage/LeaderboardPage.tsx  | 18 ++++++++++++++----
 2 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
index 12705870a1b..c4ea2807f1f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
@@ -19,6 +19,15 @@ const StyledReactMarkdown = styled(ReactMarkdown)`
   > *:first-child {
     margin-top: 0;
   }
+  h1,
+  h2,
+  h3,
+  h4,
+  h5,
+  h6 {
+    font-weight: 600;
+    font-size: 1.25rem;
+  }
 `;
 
 export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
@@ -45,11 +54,10 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
         return; // Exit if we can't determine the height
       }
 
-      // Add padding (16px top + 16px bottom)
-      newHeight += 32;
+      newHeight -= 1;
 
       // Set a minimum height to prevent collapsing
-      newHeight = Math.max(newHeight, 56);
+      newHeight = Math.max(newHeight, 25);
 
       containerRef.current.style.height = `${newHeight}px`;
     }
@@ -95,7 +103,9 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
   }, [updateHeight, value]);
 
   return (
-    <Box ref={containerRef} sx={{padding: '16px', transition: 'height 0.3s'}}>
+    <Box
+      ref={containerRef}
+      sx={{padding: '0px', transition: 'height 0.3s', flex: 1}}>
       {isEditing ? (
         <TextField
           fullWidth
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index e115edb2300..f2f766360e4 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -114,8 +114,18 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
           width="100%"
           sx={{
             alignItems: 'flex-start',
+            padding: '12px',
+            gap: '12px',
           }}>
-          <Box flexShrink={0} flexGrow={1} overflow="auto">
+          <Box
+            flexShrink={0}
+            flexGrow={1}
+            sx={{
+              overflow: 'auto',
+              height: '100%',
+              display: 'flex',
+              alignItems: 'center',
+            }}>
             <EditableMarkdown
               value={description}
               onChange={setDescription}
@@ -125,8 +135,8 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
           <div
             style={{
               display: showConfig ? 'none' : 'block',
-              paddingRight: '16px',
-              paddingTop: '16px',
+              // paddingRight: '12px',
+              // paddingTop: '12px',
             }}>
             <ToggleLeaderboardConfig
               isOpen={showConfig}
@@ -182,7 +192,7 @@ export const ToggleLeaderboardConfig: React.FC<{
       }}>
       <Button
         variant="ghost"
-        size="medium"
+        size="small"
         onClick={onClick}
         tooltip={isOpen ? 'Discard Changes' : 'Configure Leaderboard'}
         icon={isOpen ? 'close' : 'settings'}

From 30beb311fb86b1348d53137f87a8a4697c68d86e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 14:53:14 -0700
Subject: [PATCH 059/169] Human: Layer 1 of queries complete

---
 .../LeaderboardConfigEditor.tsx               | 37 ++++++++++-----
 .../query/configEditorQuery.ts                | 45 ++++++++++++++++---
 .../LeaderboardPage/query/leaderboardQuery.ts |  5 ++-
 3 files changed, 69 insertions(+), 18 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index d2ed94eb4ca..24c261897f2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -33,6 +33,7 @@ import {
   FilterAndGroupSpec,
   LeaderboardConfigType,
 } from './types/leaderboardConfigType';
+import { useGetTraceServerClientContext } from '../wfReactInterface/traceServerClientContext';
 
 export const LeaderboardConfigEditor: React.FC<{
   entity: string;
@@ -100,6 +101,8 @@ export const LeaderboardConfigEditor: React.FC<{
       <Box sx={{flex: 1, overflowY: 'auto', p: 2}}>
         {activeTab === 0 && (
           <ConfigEditor
+            entity={entity}
+            project={project}
             config={config.config.dataSelectionSpec}
             updateConfig={updateConfig}
           />
@@ -135,14 +138,18 @@ export const LeaderboardConfigEditor: React.FC<{
 };
 
 const ConfigEditor: React.FC<{
+  entity: string;
+  project: string;
   config: FilterAndGroupSpec;
   updateConfig: (
     updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
   ) => void;
-}> = ({config, updateConfig}) => {
+}> = ({entity, project, config, updateConfig}) => {
   return (
     <>
       <SourceEvaluationsConfig
+        entity={entity}
+        project={project}
         sourceEvaluations={config.sourceEvaluations}
         updateConfig={updateConfig}
       />
@@ -237,23 +244,26 @@ function debounce<T extends (...args: any[]) => any>(
 }
 
 const SourceEvaluationsConfig: React.FC<{
+  entity: string;
+  project: string;
   sourceEvaluations: FilterAndGroupSourceEvaluationSpec[] | undefined;
   updateConfig: (
     updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
   ) => void;
-}> = ({sourceEvaluations, updateConfig}) => {
+}> = ({sourceEvaluations, updateConfig, entity, project}) => {
+  const getTraceServerClient = useGetTraceServerClientContext();
   const [evaluationNames, setEvaluationNames] = useState<string[]>([]);
 
   useEffect(() => {
-    fetchEvaluationNames().then(setEvaluationNames);
-  }, []);
+    fetchEvaluationNames(getTraceServerClient(), entity, project).then(setEvaluationNames);
+  }, [entity, getTraceServerClient, project]);
 
   const handleAddSourceEvaluation = () => {
     updateConfig(spec => ({
       ...spec,
       sourceEvaluations: [
         ...(spec.sourceEvaluations || []),
-        {name: '', version: '*'},
+        {name: '*', version: '*'},
       ],
     }));
   };
@@ -266,6 +276,8 @@ const SourceEvaluationsConfig: React.FC<{
       {sourceEvaluations?.map((evaluation, index) => (
         <SourceEvaluationItem
           key={index}
+          entity={entity}
+          project={project}
           evaluation={evaluation}
           evaluationNames={evaluationNames}
           updateConfig={updateConfig}
@@ -280,20 +292,23 @@ const SourceEvaluationsConfig: React.FC<{
 };
 
 const SourceEvaluationItem: React.FC<{
+  entity: string;
+  project: string;
   evaluation: FilterAndGroupSourceEvaluationSpec;
   evaluationNames: string[];
   updateConfig: (
     updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
   ) => void;
   index: number;
-}> = ({evaluation, evaluationNames, updateConfig, index}) => {
-  const [versions, setVersions] = useState<string[]>([]);
+}> = ({entity, project, evaluation, evaluationNames, updateConfig, index}) => {
+  const getTraceServerClient = useGetTraceServerClientContext();
+  const [versions, setVersions] = useState<{digest: string, index: number}[]>([]);
 
   useEffect(() => {
     if (evaluation.name && evaluation.name !== '*') {
-      fetchEvaluationVersionsForName(evaluation.name).then(setVersions);
+      fetchEvaluationVersionsForName(getTraceServerClient(), entity, project, evaluation.name).then(setVersions);
     }
-  }, [evaluation.name]);
+  }, [entity, evaluation.name, getTraceServerClient, project]);
 
   const handleNameChange = (event: SelectChangeEvent<string>) => {
     const newName = event.target.value as string;
@@ -336,8 +351,8 @@ const SourceEvaluationItem: React.FC<{
           disabled={evaluation.name === '*'}>
           <MenuItem value="*">All</MenuItem>
           {versions.map(version => (
-            <MenuItem key={version} value={version}>
-              {version}
+            <MenuItem key={version.digest} value={version.digest}>
+              v{version.index} ({version.digest.slice(0, 6)})
             </MenuItem>
           ))}
         </Select>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
index dfe3c7deda3..8145aeee03e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
@@ -1,14 +1,47 @@
+/**
+ * This file is not performant yet.
+ */
+
+import { TraceServerClient } from '../../wfReactInterface/traceServerClient';
+import { projectIdFromParts } from '../../wfReactInterface/tsDataModelHooks';
 import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
 
-export const fetchEvaluationNames = async (): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(['E1', 'E2', 'E3']);
+export const fetchEvaluationNames = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
+): Promise<string[]> => {
+  return client.objsQuery({
+    project_id: projectIdFromParts({entity, project}),
+    filter: {
+      base_object_classes: ['Evaluation'],
+      is_op: false,
+      latest_only: true
+    },
+    metadata_only: true,
+    sort_by: [{field: 'created_at', direction: 'desc'}],
+  }).then(res => {
+    return res.objs.map(obj => obj.object_id);
+  });
 };
 export const fetchEvaluationVersionsForName = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
   name: string
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(['EV1', 'EV2', 'EV3']);
+): Promise<{digest: string, index: number}[]> => {
+  return client.objsQuery({
+    project_id: projectIdFromParts({entity, project}),
+    filter: {
+      base_object_classes: ['Evaluation'],
+      is_op: false,
+      object_ids: [name]
+    },
+    metadata_only: true,
+    sort_by: [{field: 'created_at', direction: 'desc'}],
+  }).then(res => {
+    return res.objs.map(obj => ({digest: obj.digest, index: obj.version_index}));
+  });
 };
 export const fetchDatasetNamesForSpec = async (
   spec: FilterAndGroupSpec
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
index 0bbda3b86eb..862ad6bd5a8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
@@ -67,13 +67,16 @@ export type GroupedLeaderboardModelGroup = {
   };
 };
 
+
+
 export const getLeaderboardData = async (
   client: TraceServerClient,
   entity: string,
   project: string,
   spec: FilterAndGroupSpec = {}
 ): Promise<GroupedLeaderboardData> => {
-  const sourceEvals = spec.sourceEvaluations ?? [];
+  const atLeastOneStarName = spec.sourceEvaluations?.some(sourceEvaluation => sourceEvaluation.name === '*');
+  const sourceEvals = atLeastOneStarName ? [] : (spec.sourceEvaluations ?? []);
   const evalNames = sourceEvals.map(sourceEvaluation => sourceEvaluation.name);
   // const fullyQualifiedEvalRefs = sourceEvals.map(sourceEvaluation => {
   //   return objectVersionKeyToRefUri({

From ae2f9c4e7b12e88e74e56bebe9b90cc38edd9c5c Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 15:00:56 -0700
Subject: [PATCH 060/169] Human: Little styling

---
 .../LeaderboardPage/EditableMarkdown.tsx      |   4 +-
 .../LeaderboardConfigEditor.tsx               | 106 ++++++++++++++++--
 .../query/configEditorQuery.ts                |  63 ++++++-----
 .../LeaderboardPage/query/leaderboardQuery.ts |   8 +-
 4 files changed, 135 insertions(+), 46 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
index c4ea2807f1f..930917ff6a2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
@@ -54,10 +54,10 @@ export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
         return; // Exit if we can't determine the height
       }
 
-      newHeight -= 1;
+      newHeight -= 3;
 
       // Set a minimum height to prevent collapsing
-      newHeight = Math.max(newHeight, 25);
+      newHeight = Math.max(newHeight, 24);
 
       containerRef.current.style.height = `${newHeight}px`;
     }
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 24c261897f2..ca7122dcda5 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -1,3 +1,4 @@
+import DeleteIcon from '@mui/icons-material/Delete';
 import {
   Box,
   Button,
@@ -12,8 +13,10 @@ import {
   Tabs,
   Typography,
 } from '@mui/material';
+import IconButton from '@mui/material/IconButton';
 import React, {useEffect, useMemo, useState} from 'react';
 
+import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {
   fetchDatasetNamesForSpec,
   fetchDatasetVersionsForSpecAndName,
@@ -33,7 +36,6 @@ import {
   FilterAndGroupSpec,
   LeaderboardConfigType,
 } from './types/leaderboardConfigType';
-import { useGetTraceServerClientContext } from '../wfReactInterface/traceServerClientContext';
 
 export const LeaderboardConfigEditor: React.FC<{
   entity: string;
@@ -88,7 +90,7 @@ export const LeaderboardConfigEditor: React.FC<{
         height: '100%',
         display: 'flex',
         flexDirection: 'column',
-        border: '1px solid #e0e0e0',
+        borderLeft: '1px solid #e0e0e0',
       }}>
       <Box sx={{borderBottom: 1, borderColor: 'divider'}}>
         <Tabs value={activeTab} onChange={handleTabChange}>
@@ -122,9 +124,9 @@ export const LeaderboardConfigEditor: React.FC<{
           justifyContent: 'flex-end',
           pt: 1,
           pb: 1,
-          pr: 1.5,
+          pr: 1,
           borderTop: '1px solid #e0e0e0',
-          height: '51px',
+          height: '52px',
         }}>
         <Button variant="outlined" onClick={handleCancel} sx={{mr: 2}}>
           Cancel
@@ -255,7 +257,9 @@ const SourceEvaluationsConfig: React.FC<{
   const [evaluationNames, setEvaluationNames] = useState<string[]>([]);
 
   useEffect(() => {
-    fetchEvaluationNames(getTraceServerClient(), entity, project).then(setEvaluationNames);
+    fetchEvaluationNames(getTraceServerClient(), entity, project).then(
+      setEvaluationNames
+    );
   }, [entity, getTraceServerClient, project]);
 
   const handleAddSourceEvaluation = () => {
@@ -302,11 +306,18 @@ const SourceEvaluationItem: React.FC<{
   index: number;
 }> = ({entity, project, evaluation, evaluationNames, updateConfig, index}) => {
   const getTraceServerClient = useGetTraceServerClientContext();
-  const [versions, setVersions] = useState<{digest: string, index: number}[]>([]);
+  const [versions, setVersions] = useState<
+    Array<{digest: string; index: number}>
+  >([]);
 
   useEffect(() => {
     if (evaluation.name && evaluation.name !== '*') {
-      fetchEvaluationVersionsForName(getTraceServerClient(), entity, project, evaluation.name).then(setVersions);
+      fetchEvaluationVersionsForName(
+        getTraceServerClient(),
+        entity,
+        project,
+        evaluation.name
+      ).then(setVersions);
     }
   }, [entity, evaluation.name, getTraceServerClient, project]);
 
@@ -330,8 +341,15 @@ const SourceEvaluationItem: React.FC<{
     }));
   };
 
+  const handleDelete = () => {
+    updateConfig(spec => ({
+      ...spec,
+      sourceEvaluations: spec.sourceEvaluations?.filter((_, i) => i !== index),
+    }));
+  };
+
   return (
-    <Box sx={{display: 'flex', gap: 2, mb: 2}}>
+    <Box sx={{display: 'flex', gap: 2, mb: 2, alignItems: 'center'}}>
       <FormControl fullWidth>
         <InputLabel>Name</InputLabel>
         <Select value={evaluation.name || '*'} onChange={handleNameChange}>
@@ -357,6 +375,9 @@ const SourceEvaluationItem: React.FC<{
           ))}
         </Select>
       </FormControl>
+      <IconButton onClick={handleDelete} size="small">
+        <DeleteIcon />
+      </IconButton>
     </Box>
   );
 };
@@ -471,9 +492,16 @@ const DatasetItem: React.FC<{
     }));
   };
 
+  const handleDelete = () => {
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.filter((_, i) => i !== index),
+    }));
+  };
+
   return (
     <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, mb: 2}}>
-      <Box sx={{display: 'flex', gap: 2}}>
+      <Box sx={{display: 'flex', gap: 2, alignItems: 'center'}}>
         <FormControl fullWidth>
           <InputLabel>Name</InputLabel>
           <Select value={dataset.name || '*'} onChange={handleNameChange}>
@@ -499,6 +527,9 @@ const DatasetItem: React.FC<{
             ))}
           </Select>
         </FormControl>
+        <IconButton onClick={handleDelete} size="small">
+          <DeleteIcon />
+        </IconButton>
       </Box>
       <FormControlLabel
         control={
@@ -625,9 +656,23 @@ const ScorerItem: React.FC<{
     }));
   };
 
+  const handleDelete = () => {
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === datasetIndex
+          ? {
+              ...d,
+              scorers: d.scorers?.filter((_, j) => j !== scorerIndex),
+            }
+          : d
+      ),
+    }));
+  };
+
   return (
     <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, ml: 2, mb: 2}}>
-      <Box sx={{display: 'flex', gap: 2}}>
+      <Box sx={{display: 'flex', gap: 2, alignItems: 'center'}}>
         <FormControl fullWidth>
           <InputLabel>Name</InputLabel>
           <Select value={scorer.name || '*'} onChange={handleNameChange}>
@@ -653,6 +698,9 @@ const ScorerItem: React.FC<{
             ))}
           </Select>
         </FormControl>
+        <IconButton onClick={handleDelete} size="small">
+          <DeleteIcon />
+        </IconButton>
       </Box>
       <FormControlLabel
         control={
@@ -751,8 +799,29 @@ const MetricItem: React.FC<{
     }));
   };
 
+  const handleDelete = () => {
+    updateConfig(spec => ({
+      ...spec,
+      datasets: spec.datasets?.map((d, i) =>
+        i === datasetIndex
+          ? {
+              ...d,
+              scorers: d.scorers?.map((s, j) =>
+                j === scorerIndex
+                  ? {
+                      ...s,
+                      metrics: s.metrics?.filter((_, k) => k !== metricIndex),
+                    }
+                  : s
+              ),
+            }
+          : d
+      ),
+    }));
+  };
+
   return (
-    <Box sx={{display: 'flex', gap: 2, ml: 2, mb: 2}}>
+    <Box sx={{display: 'flex', gap: 2, ml: 2, mb: 2, alignItems: 'center'}}>
       <FormControl fullWidth>
         <InputLabel>Metric Path</InputLabel>
         <Select value={metric.path} onChange={handlePathChange}>
@@ -773,6 +842,9 @@ const MetricItem: React.FC<{
         }
         label="Should Minimize"
       />
+      <IconButton onClick={handleDelete} size="small">
+        <DeleteIcon />
+      </IconButton>
     </Box>
   );
 };
@@ -868,9 +940,16 @@ const ModelItem: React.FC<{
     }));
   };
 
+  const handleDelete = () => {
+    updateConfig(spec => ({
+      ...spec,
+      models: spec.models?.filter((_, i) => i !== index),
+    }));
+  };
+
   return (
     <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, mb: 2}}>
-      <Box sx={{display: 'flex', gap: 2}}>
+      <Box sx={{display: 'flex', gap: 2, alignItems: 'center'}}>
         <FormControl fullWidth>
           <InputLabel>Name</InputLabel>
           <Select value={model.name || '*'} onChange={handleNameChange}>
@@ -896,6 +975,9 @@ const ModelItem: React.FC<{
             ))}
           </Select>
         </FormControl>
+        <IconButton onClick={handleDelete} size="small">
+          <DeleteIcon />
+        </IconButton>
       </Box>
       <FormControlLabel
         control={
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
index 8145aeee03e..5fc7e3a6aac 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
@@ -2,46 +2,53 @@
  * This file is not performant yet.
  */
 
-import { TraceServerClient } from '../../wfReactInterface/traceServerClient';
-import { projectIdFromParts } from '../../wfReactInterface/tsDataModelHooks';
+import {TraceServerClient} from '../../wfReactInterface/traceServerClient';
+import {projectIdFromParts} from '../../wfReactInterface/tsDataModelHooks';
 import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
 
 export const fetchEvaluationNames = async (
   client: TraceServerClient,
   entity: string,
-  project: string,
+  project: string
 ): Promise<string[]> => {
-  return client.objsQuery({
-    project_id: projectIdFromParts({entity, project}),
-    filter: {
-      base_object_classes: ['Evaluation'],
-      is_op: false,
-      latest_only: true
-    },
-    metadata_only: true,
-    sort_by: [{field: 'created_at', direction: 'desc'}],
-  }).then(res => {
-    return res.objs.map(obj => obj.object_id);
-  });
+  return client
+    .objsQuery({
+      project_id: projectIdFromParts({entity, project}),
+      filter: {
+        base_object_classes: ['Evaluation'],
+        is_op: false,
+        latest_only: true,
+      },
+      metadata_only: true,
+      sort_by: [{field: 'created_at', direction: 'desc'}],
+    })
+    .then(res => {
+      return res.objs.map(obj => obj.object_id);
+    });
 };
 export const fetchEvaluationVersionsForName = async (
   client: TraceServerClient,
   entity: string,
   project: string,
   name: string
-): Promise<{digest: string, index: number}[]> => {
-  return client.objsQuery({
-    project_id: projectIdFromParts({entity, project}),
-    filter: {
-      base_object_classes: ['Evaluation'],
-      is_op: false,
-      object_ids: [name]
-    },
-    metadata_only: true,
-    sort_by: [{field: 'created_at', direction: 'desc'}],
-  }).then(res => {
-    return res.objs.map(obj => ({digest: obj.digest, index: obj.version_index}));
-  });
+): Promise<Array<{digest: string; index: number}>> => {
+  return client
+    .objsQuery({
+      project_id: projectIdFromParts({entity, project}),
+      filter: {
+        base_object_classes: ['Evaluation'],
+        is_op: false,
+        object_ids: [name],
+      },
+      metadata_only: true,
+      sort_by: [{field: 'created_at', direction: 'desc'}],
+    })
+    .then(res => {
+      return res.objs.map(obj => ({
+        digest: obj.digest,
+        index: obj.version_index,
+      }));
+    });
 };
 export const fetchDatasetNamesForSpec = async (
   spec: FilterAndGroupSpec
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
index 862ad6bd5a8..83b7135ad4b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
@@ -67,16 +67,16 @@ export type GroupedLeaderboardModelGroup = {
   };
 };
 
-
-
 export const getLeaderboardData = async (
   client: TraceServerClient,
   entity: string,
   project: string,
   spec: FilterAndGroupSpec = {}
 ): Promise<GroupedLeaderboardData> => {
-  const atLeastOneStarName = spec.sourceEvaluations?.some(sourceEvaluation => sourceEvaluation.name === '*');
-  const sourceEvals = atLeastOneStarName ? [] : (spec.sourceEvaluations ?? []);
+  const atLeastOneStarName = spec.sourceEvaluations?.some(
+    sourceEvaluation => sourceEvaluation.name === '*'
+  );
+  const sourceEvals = atLeastOneStarName ? [] : spec.sourceEvaluations ?? [];
   const evalNames = sourceEvals.map(sourceEvaluation => sourceEvaluation.name);
   // const fullyQualifiedEvalRefs = sourceEvals.map(sourceEvaluation => {
   //   return objectVersionKeyToRefUri({

From 3a023210929ca9156398c526ae01c2ae4a3b23b6 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 15:08:12 -0700
Subject: [PATCH 061/169] Human: Subtle improvements

---
 .../LeaderboardConfigEditor.tsx               | 78 +++++++++++--------
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  2 +-
 .../LeaderboardPage/query/leaderboardQuery.ts | 35 +++++----
 .../types/leaderboardConfigType.ts            |  3 +-
 4 files changed, 66 insertions(+), 52 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index ca7122dcda5..55565c5ab40 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -29,6 +29,7 @@ import {
   fetchScorerVersionsForSpecAndName,
 } from './query/configEditorQuery';
 import {
+  ALL_VALUE,
   FilterAndGroupDatasetScorerMetricSpec,
   FilterAndGroupDatasetScorerSpec,
   FilterAndGroupDatasetSpec,
@@ -267,7 +268,7 @@ const SourceEvaluationsConfig: React.FC<{
       ...spec,
       sourceEvaluations: [
         ...(spec.sourceEvaluations || []),
-        {name: '*', version: '*'},
+        {name: ALL_VALUE, version: ALL_VALUE},
       ],
     }));
   };
@@ -311,7 +312,7 @@ const SourceEvaluationItem: React.FC<{
   >([]);
 
   useEffect(() => {
-    if (evaluation.name && evaluation.name !== '*') {
+    if (evaluation.name && evaluation.name !== ALL_VALUE) {
       fetchEvaluationVersionsForName(
         getTraceServerClient(),
         entity,
@@ -326,7 +327,7 @@ const SourceEvaluationItem: React.FC<{
     updateConfig(spec => ({
       ...spec,
       sourceEvaluations: spec.sourceEvaluations?.map((e, i) =>
-        i === index ? {...e, name: newName, version: '*'} : e
+        i === index ? {...e, name: newName, version: ALL_VALUE} : e
       ),
     }));
   };
@@ -352,8 +353,10 @@ const SourceEvaluationItem: React.FC<{
     <Box sx={{display: 'flex', gap: 2, mb: 2, alignItems: 'center'}}>
       <FormControl fullWidth>
         <InputLabel>Name</InputLabel>
-        <Select value={evaluation.name || '*'} onChange={handleNameChange}>
-          <MenuItem value="*">All</MenuItem>
+        <Select
+          value={evaluation.name || ALL_VALUE}
+          onChange={handleNameChange}>
+          <MenuItem value={ALL_VALUE}>All</MenuItem>
           {evaluationNames.map(name => (
             <MenuItem key={name} value={name}>
               {name}
@@ -361,13 +364,13 @@ const SourceEvaluationItem: React.FC<{
           ))}
         </Select>
       </FormControl>
-      <FormControl fullWidth disabled={evaluation.name === '*'}>
+      <FormControl fullWidth disabled={evaluation.name === ALL_VALUE}>
         <InputLabel>Version</InputLabel>
         <Select
           value={evaluation.version}
           onChange={handleVersionChange}
-          disabled={evaluation.name === '*'}>
-          <MenuItem value="*">All</MenuItem>
+          disabled={evaluation.name === ALL_VALUE}>
+          <MenuItem value={ALL_VALUE}>All</MenuItem>
           {versions.map(version => (
             <MenuItem key={version.digest} value={version.digest}>
               v{version.index} ({version.digest.slice(0, 6)})
@@ -399,7 +402,7 @@ const DatasetsConfig: React.FC<{
       ...spec,
       datasets: [
         ...(spec.datasets || []),
-        {name: '', version: '*', groupAllVersions: false},
+        {name: ALL_VALUE, version: ALL_VALUE, groupAllVersions: false},
       ],
     }));
   };
@@ -437,7 +440,7 @@ const DatasetItem: React.FC<{
   const [scorerNames, setScorerNames] = useState<string[]>([]);
 
   useEffect(() => {
-    if (dataset.name && dataset.name !== '*') {
+    if (dataset.name && dataset.name !== ALL_VALUE) {
       fetchDatasetVersionsForSpecAndName({}, dataset.name).then(setVersions);
     }
     fetchScorerNamesForSpec({}).then(setScorerNames);
@@ -448,7 +451,7 @@ const DatasetItem: React.FC<{
     updateConfig(spec => ({
       ...spec,
       datasets: spec.datasets?.map((d, i) =>
-        i === index ? {...d, name: newName, version: '*'} : d
+        i === index ? {...d, name: newName, version: ALL_VALUE} : d
       ),
     }));
   };
@@ -484,7 +487,12 @@ const DatasetItem: React.FC<{
               ...d,
               scorers: [
                 ...(d.scorers || []),
-                {name: '', version: '*', groupAllVersions: false, metrics: []},
+                {
+                  name: ALL_VALUE,
+                  version: ALL_VALUE,
+                  groupAllVersions: false,
+                  metrics: [],
+                },
               ],
             }
           : d
@@ -504,8 +512,8 @@ const DatasetItem: React.FC<{
       <Box sx={{display: 'flex', gap: 2, alignItems: 'center'}}>
         <FormControl fullWidth>
           <InputLabel>Name</InputLabel>
-          <Select value={dataset.name || '*'} onChange={handleNameChange}>
-            <MenuItem value="*">All</MenuItem>
+          <Select value={dataset.name || ALL_VALUE} onChange={handleNameChange}>
+            <MenuItem value={ALL_VALUE}>All</MenuItem>
             {datasetNames.map(name => (
               <MenuItem key={name} value={name}>
                 {name}
@@ -513,13 +521,13 @@ const DatasetItem: React.FC<{
             ))}
           </Select>
         </FormControl>
-        <FormControl fullWidth disabled={dataset.name === '*'}>
+        <FormControl fullWidth disabled={dataset.name === ALL_VALUE}>
           <InputLabel>Version</InputLabel>
           <Select
             value={dataset.version}
             onChange={handleVersionChange}
-            disabled={dataset.name === '*'}>
-            <MenuItem value="*">All</MenuItem>
+            disabled={dataset.name === ALL_VALUE}>
+            <MenuItem value={ALL_VALUE}>All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version} value={version}>
                 {version}
@@ -571,7 +579,7 @@ const ScorerItem: React.FC<{
   const [metricPaths, setMetricPaths] = useState<string[]>([]);
 
   useEffect(() => {
-    if (scorer.name && scorer.name !== '*') {
+    if (scorer.name && scorer.name !== ALL_VALUE) {
       fetchScorerVersionsForSpecAndName({}, scorer.name).then(setVersions);
     }
     fetchMetricPathsForSpec({}).then(setMetricPaths);
@@ -586,7 +594,9 @@ const ScorerItem: React.FC<{
           ? {
               ...d,
               scorers: d.scorers?.map((s, j) =>
-                j === scorerIndex ? {...s, name: newName, version: '*'} : s
+                j === scorerIndex
+                  ? {...s, name: newName, version: ALL_VALUE}
+                  : s
               ),
             }
           : d
@@ -645,7 +655,7 @@ const ScorerItem: React.FC<{
                       ...s,
                       metrics: [
                         ...(s.metrics || []),
-                        {path: '', shouldMinimize: false},
+                        {path: ALL_VALUE, shouldMinimize: false},
                       ],
                     }
                   : s
@@ -675,8 +685,8 @@ const ScorerItem: React.FC<{
       <Box sx={{display: 'flex', gap: 2, alignItems: 'center'}}>
         <FormControl fullWidth>
           <InputLabel>Name</InputLabel>
-          <Select value={scorer.name || '*'} onChange={handleNameChange}>
-            <MenuItem value="*">All</MenuItem>
+          <Select value={scorer.name || ALL_VALUE} onChange={handleNameChange}>
+            <MenuItem value={ALL_VALUE}>All</MenuItem>
             {scorerNames.map(name => (
               <MenuItem key={name} value={name}>
                 {name}
@@ -684,13 +694,13 @@ const ScorerItem: React.FC<{
             ))}
           </Select>
         </FormControl>
-        <FormControl fullWidth disabled={scorer.name === '*'}>
+        <FormControl fullWidth disabled={scorer.name === ALL_VALUE}>
           <InputLabel>Version</InputLabel>
           <Select
             value={scorer.version}
             onChange={handleVersionChange}
-            disabled={scorer.name === '*'}>
-            <MenuItem value="*">All</MenuItem>
+            disabled={scorer.name === ALL_VALUE}>
+            <MenuItem value={ALL_VALUE}>All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version} value={version}>
                 {version}
@@ -825,7 +835,7 @@ const MetricItem: React.FC<{
       <FormControl fullWidth>
         <InputLabel>Metric Path</InputLabel>
         <Select value={metric.path} onChange={handlePathChange}>
-          <MenuItem value="*">All</MenuItem>
+          <MenuItem value={ALL_VALUE}>All</MenuItem>
           {metricPaths.map(path => (
             <MenuItem key={path} value={path}>
               {path}
@@ -866,7 +876,7 @@ const ModelsConfig: React.FC<{
       ...spec,
       models: [
         ...(spec.models || []),
-        {name: '', version: '*', groupAllVersions: false},
+        {name: ALL_VALUE, version: ALL_VALUE, groupAllVersions: false},
       ],
     }));
   };
@@ -903,7 +913,7 @@ const ModelItem: React.FC<{
   const [versions, setVersions] = useState<string[]>([]);
 
   useEffect(() => {
-    if (model.name && model.name !== '*') {
+    if (model.name && model.name !== ALL_VALUE) {
       fetchModelVersionsForSpecndName({}, model.name).then(setVersions);
     }
   }, [model.name]);
@@ -913,7 +923,7 @@ const ModelItem: React.FC<{
     updateConfig(spec => ({
       ...spec,
       models: spec.models?.map((m, i) =>
-        i === index ? {...m, name: newName, version: '*'} : m
+        i === index ? {...m, name: newName, version: ALL_VALUE} : m
       ),
     }));
   };
@@ -952,8 +962,8 @@ const ModelItem: React.FC<{
       <Box sx={{display: 'flex', gap: 2, alignItems: 'center'}}>
         <FormControl fullWidth>
           <InputLabel>Name</InputLabel>
-          <Select value={model.name || '*'} onChange={handleNameChange}>
-            <MenuItem value="*">All</MenuItem>
+          <Select value={model.name || ALL_VALUE} onChange={handleNameChange}>
+            <MenuItem value={ALL_VALUE}>All</MenuItem>
             {modelNames.map(name => (
               <MenuItem key={name} value={name}>
                 {name}
@@ -961,13 +971,13 @@ const ModelItem: React.FC<{
             ))}
           </Select>
         </FormControl>
-        <FormControl fullWidth disabled={model.name === '*'}>
+        <FormControl fullWidth disabled={model.name === ALL_VALUE}>
           <InputLabel>Version</InputLabel>
           <Select
             value={model.version}
             onChange={handleVersionChange}
-            disabled={model.name === '*'}>
-            <MenuItem value="*">All</MenuItem>
+            disabled={model.name === ALL_VALUE}>
+            <MenuItem value={ALL_VALUE}>All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version} value={version}>
                 {version}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index f2f766360e4..45ced5a88ad 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -105,7 +105,7 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
         display="flex"
         flexDirection="column"
         height="100%"
-        minWidth="65%">
+        minWidth="50%">
         <Box
           flex={1}
           display="flex"
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
index 83b7135ad4b..61732a9f603 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
@@ -14,7 +14,7 @@ import {
   objectVersionKeyToRefUri,
   opVersionKeyToRefUri,
 } from '../../wfReactInterface/utilities';
-import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
+import {ALL_VALUE, FilterAndGroupSpec} from '../types/leaderboardConfigType';
 
 export type LeaderboardValueRecord = {
   datasetName: string;
@@ -74,7 +74,7 @@ export const getLeaderboardData = async (
   spec: FilterAndGroupSpec = {}
 ): Promise<GroupedLeaderboardData> => {
   const atLeastOneStarName = spec.sourceEvaluations?.some(
-    sourceEvaluation => sourceEvaluation.name === '*'
+    sourceEvaluation => sourceEvaluation.name === ALL_VALUE
   );
   const sourceEvals = atLeastOneStarName ? [] : spec.sourceEvaluations ?? [];
   const evalNames = sourceEvals.map(sourceEvaluation => sourceEvaluation.name);
@@ -109,7 +109,8 @@ export const getLeaderboardData = async (
       return sourceEvals.some(sourceEval => {
         return (
           obj.object_id === sourceEval.name &&
-          (obj.digest === sourceEval.version || sourceEval.version === '*')
+          (obj.digest === sourceEval.version ||
+            sourceEval.version === ALL_VALUE)
         );
       });
     });
@@ -158,7 +159,7 @@ export const getLeaderboardData = async (
           entity,
           project,
           opId: EVALUATE_OP_NAME_POST_PYDANTIC,
-          versionHash: '*',
+          versionHash: ALL_VALUE,
         }),
       ],
       input_refs: fullyQualifiedEvalRefs,
@@ -302,14 +303,14 @@ export const getLeaderboardData = async (
         spec.models.find(
           model =>
             model.name === row.modelName &&
-            (model.version === '*' || model.version === row.modelVersion)
+            (model.version === ALL_VALUE || model.version === row.modelVersion)
         );
       modelSpec =
         modelSpec ||
         spec.models.find(
           model =>
-            (model.name === '*' || model.name === row.modelName) &&
-            (model.version === '*' || model.version === row.modelVersion)
+            (model.name === ALL_VALUE || model.name === row.modelName) &&
+            (model.version === ALL_VALUE || model.version === row.modelVersion)
         );
       if (!modelSpec) {
         return {include: false, groupableRow};
@@ -325,7 +326,7 @@ export const getLeaderboardData = async (
     if (spec.datasets.length === 0) {
       return {include: true, groupableRow};
     }
-    if (spec.datasets.some(dataset => dataset.name === '*')) {
+    if (spec.datasets.some(dataset => dataset.name === ALL_VALUE)) {
       return {include: true, groupableRow};
     }
 
@@ -339,14 +340,16 @@ export const getLeaderboardData = async (
       spec.datasets.find(
         dataset =>
           dataset.name === row.datasetName &&
-          (dataset.version === '*' || dataset.version === row.datasetVersion)
+          (dataset.version === ALL_VALUE ||
+            dataset.version === row.datasetVersion)
       );
     datasetSpec =
       datasetSpec ||
       spec.datasets.find(
         dataset =>
-          (dataset.name === '*' || dataset.name === row.datasetName) &&
-          (dataset.version === '*' || dataset.version === row.datasetVersion)
+          (dataset.name === ALL_VALUE || dataset.name === row.datasetName) &&
+          (dataset.version === ALL_VALUE ||
+            dataset.version === row.datasetVersion)
       );
     if (!datasetSpec) {
       return {include: false, groupableRow};
@@ -364,14 +367,16 @@ export const getLeaderboardData = async (
         datasetSpec.scorers.find(
           scorer =>
             scorer.name === row.scorerName &&
-            (scorer.version === '*' || scorer.version === row.scorerVersion)
+            (scorer.version === ALL_VALUE ||
+              scorer.version === row.scorerVersion)
         );
       scorerSpec =
         scorerSpec ||
         datasetSpec.scorers.find(
           scorer =>
-            (scorer.name === '*' || scorer.name === row.scorerName) &&
-            (scorer.version === '*' || scorer.version === row.scorerVersion)
+            (scorer.name === ALL_VALUE || scorer.name === row.scorerName) &&
+            (scorer.version === ALL_VALUE ||
+              scorer.version === row.scorerVersion)
         );
       if (!scorerSpec) {
         return {include: false, groupableRow};
@@ -381,7 +386,7 @@ export const getLeaderboardData = async (
       }
       if (scorerSpec.metrics) {
         const metricSpec = scorerSpec.metrics.find(
-          metric => metric.path === '*' || metric.path === row.metricPath
+          metric => metric.path === ALL_VALUE || metric.path === row.metricPath
         );
         if (!metricSpec) {
           return {include: false, groupableRow};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts
index f1eebbd6526..bb4587966c9 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts
@@ -1,5 +1,4 @@
-export type VersionSpec = string | 'latest' | 'all';
-export type Direction = 'asc' | 'desc';
+export const ALL_VALUE = '*';
 
 export type LeaderboardConfigType = {
   version: number;

From 455101abb62370a0f7e3226f178e4ff92429ac0e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 15:19:30 -0700
Subject: [PATCH 062/169] Human: Made subtle logic fixes to the query

---
 .../LeaderboardPage/query/leaderboardQuery.ts     | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
index 61732a9f603..fb70ac457f5 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
@@ -326,9 +326,9 @@ export const getLeaderboardData = async (
     if (spec.datasets.length === 0) {
       return {include: true, groupableRow};
     }
-    if (spec.datasets.some(dataset => dataset.name === ALL_VALUE)) {
-      return {include: true, groupableRow};
-    }
+    // if (spec.datasets.some(dataset => dataset.name === ALL_VALUE)) {
+    //   return {include: true, groupableRow};
+    // }
 
     let datasetSpec = spec.datasets.find(
       dataset =>
@@ -357,7 +357,7 @@ export const getLeaderboardData = async (
     if (datasetSpec.groupAllVersions) {
       groupableRow.datasetGroup = `${row.datasetName}`;
     }
-    if (datasetSpec.scorers) {
+    if (datasetSpec.scorers && datasetSpec.scorers.length > 0) {
       let scorerSpec = datasetSpec.scorers.find(
         scorer =>
           scorer.name === row.scorerName && scorer.version === row.scorerVersion
@@ -378,13 +378,18 @@ export const getLeaderboardData = async (
             (scorer.version === ALL_VALUE ||
               scorer.version === row.scorerVersion)
         );
+      console.log({
+        scorerSpec,
+        dss: datasetSpec.scorers,
+        row,
+      });
       if (!scorerSpec) {
         return {include: false, groupableRow};
       }
       if (scorerSpec.groupAllVersions) {
         groupableRow.scorerGroup = `${row.scorerName}`;
       }
-      if (scorerSpec.metrics) {
+      if (scorerSpec.metrics && scorerSpec.metrics.length > 0) {
         const metricSpec = scorerSpec.metrics.find(
           metric => metric.path === ALL_VALUE || metric.path === row.metricPath
         );

From a185e3bf39c85e9e469c9b87e3a5a149a0125234 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 17:30:29 -0700
Subject: [PATCH 063/169] Checkpoint

---
 .../LeaderboardConfigEditor.tsx               | 78 +++++++++++++++----
 .../query/configEditorQuery.ts                | 47 +++++++++--
 .../LeaderboardPage/query/leaderboardQuery.ts | 47 ++++++++---
 3 files changed, 139 insertions(+), 33 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 55565c5ab40..1b72371f87a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -24,9 +24,10 @@ import {
   fetchEvaluationVersionsForName,
   fetchMetricPathsForSpec,
   fetchModelNamesForSpec,
-  fetchModelVersionsForSpecndName,
+  fetchModelVersionsForSpecAndName,
   fetchScorerNamesForSpec,
   fetchScorerVersionsForSpecAndName,
+  VersionDetails,
 } from './query/configEditorQuery';
 import {
   ALL_VALUE,
@@ -156,7 +157,16 @@ const ConfigEditor: React.FC<{
         sourceEvaluations={config.sourceEvaluations}
         updateConfig={updateConfig}
       />
-      <DatasetsConfig datasets={config.datasets} updateConfig={updateConfig} />
+      <DatasetsConfig
+        entity={entity}
+        project={project}
+        parentSpec={{
+          sourceEvaluations: config.sourceEvaluations,
+          // Purposely exclude models and datasets
+        }}
+        datasets={config.datasets}
+        updateConfig={updateConfig}
+      />
       <ModelsConfig models={config.models} updateConfig={updateConfig} />
     </>
   );
@@ -307,9 +317,7 @@ const SourceEvaluationItem: React.FC<{
   index: number;
 }> = ({entity, project, evaluation, evaluationNames, updateConfig, index}) => {
   const getTraceServerClient = useGetTraceServerClientContext();
-  const [versions, setVersions] = useState<
-    Array<{digest: string; index: number}>
-  >([]);
+  const [versions, setVersions] = useState<VersionDetails[]>([]);
 
   useEffect(() => {
     if (evaluation.name && evaluation.name !== ALL_VALUE) {
@@ -386,16 +394,25 @@ const SourceEvaluationItem: React.FC<{
 };
 
 const DatasetsConfig: React.FC<{
+  entity: string;
+  project: string;
+  parentSpec: FilterAndGroupSpec;
   datasets: FilterAndGroupDatasetSpec[] | undefined;
   updateConfig: (
     updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
   ) => void;
-}> = ({datasets, updateConfig}) => {
+}> = ({datasets, updateConfig, entity, project, parentSpec}) => {
+  const getTraceServerClient = useGetTraceServerClientContext();
   const [datasetNames, setDatasetNames] = useState<string[]>([]);
 
   useEffect(() => {
-    fetchDatasetNamesForSpec({}).then(setDatasetNames);
-  }, []);
+    fetchDatasetNamesForSpec(
+      getTraceServerClient(),
+      entity,
+      project,
+      parentSpec
+    ).then(setDatasetNames);
+  }, [entity, getTraceServerClient, project, parentSpec]);
 
   const handleAddDataset = () => {
     updateConfig(spec => ({
@@ -415,6 +432,17 @@ const DatasetsConfig: React.FC<{
       {datasets?.map((dataset, index) => (
         <DatasetItem
           key={index}
+          entity={entity}
+          project={project}
+          parentSpec={{
+            sourceEvaluations: parentSpec.sourceEvaluations,
+            datasets: [
+              {
+                name: dataset.name,
+                version: '*',
+              },
+            ],
+          }}
           dataset={dataset}
           datasetNames={datasetNames}
           updateConfig={updateConfig}
@@ -429,22 +457,40 @@ const DatasetsConfig: React.FC<{
 };
 
 const DatasetItem: React.FC<{
+  entity: string;
+  project: string;
+  parentSpec: FilterAndGroupSpec;
   dataset: FilterAndGroupDatasetSpec;
   datasetNames: string[];
   updateConfig: (
     updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
   ) => void;
   index: number;
-}> = ({dataset, datasetNames, updateConfig, index}) => {
-  const [versions, setVersions] = useState<string[]>([]);
+}> = ({
+  entity,
+  project,
+  parentSpec,
+  dataset,
+  datasetNames,
+  updateConfig,
+  index,
+}) => {
+  const getTraceServerClient = useGetTraceServerClientContext();
+  const [versions, setVersions] = useState<VersionDetails[]>([]);
   const [scorerNames, setScorerNames] = useState<string[]>([]);
 
   useEffect(() => {
     if (dataset.name && dataset.name !== ALL_VALUE) {
-      fetchDatasetVersionsForSpecAndName({}, dataset.name).then(setVersions);
+      fetchDatasetVersionsForSpecAndName(
+        getTraceServerClient(),
+        entity,
+        project,
+        parentSpec,
+        dataset.name
+      ).then(setVersions);
     }
-    fetchScorerNamesForSpec({}).then(setScorerNames);
-  }, [dataset.name]);
+    fetchScorerNamesForSpec(parentSpec).then(setScorerNames);
+  }, [dataset.name, entity, getTraceServerClient, project, parentSpec]);
 
   const handleNameChange = (event: SelectChangeEvent<string>) => {
     const newName = event.target.value as string;
@@ -529,8 +575,8 @@ const DatasetItem: React.FC<{
             disabled={dataset.name === ALL_VALUE}>
             <MenuItem value={ALL_VALUE}>All</MenuItem>
             {versions.map(version => (
-              <MenuItem key={version} value={version}>
-                {version}
+              <MenuItem key={version.digest} value={version.digest}>
+                v{version.index} ({version.digest.slice(0, 6)})
               </MenuItem>
             ))}
           </Select>
@@ -914,7 +960,7 @@ const ModelItem: React.FC<{
 
   useEffect(() => {
     if (model.name && model.name !== ALL_VALUE) {
-      fetchModelVersionsForSpecndName({}, model.name).then(setVersions);
+      fetchModelVersionsForSpecAndName({}, model.name).then(setVersions);
     }
   }, [model.name]);
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
index 5fc7e3a6aac..2dbbfb1edb6 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
@@ -2,9 +2,20 @@
  * This file is not performant yet.
  */
 
+import {ObjectRef} from '@wandb/weave/react';
+import _ from 'lodash';
+
+import {parseRefMaybe} from '../../../../Browse2/SmallRef';
 import {TraceServerClient} from '../../wfReactInterface/traceServerClient';
 import {projectIdFromParts} from '../../wfReactInterface/tsDataModelHooks';
 import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
+import {
+  getEvaluationObjectsForSpec,
+  getLeaderboardData,
+  getLeaderboardGroupableData,
+} from './leaderboardQuery';
+
+export type VersionDetails = {digest: string; index: number};
 
 export const fetchEvaluationNames = async (
   client: TraceServerClient,
@@ -31,7 +42,7 @@ export const fetchEvaluationVersionsForName = async (
   entity: string,
   project: string,
   name: string
-): Promise<Array<{digest: string; index: number}>> => {
+): Promise<VersionDetails[]> => {
   return client
     .objsQuery({
       project_id: projectIdFromParts({entity, project}),
@@ -51,25 +62,47 @@ export const fetchEvaluationVersionsForName = async (
     });
 };
 export const fetchDatasetNamesForSpec = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
   spec: FilterAndGroupSpec
 ): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(['D1', 'D2', 'D3']);
+  // This is def a hacky/slow solution, just getting it working.
+  const groupableData = await getLeaderboardGroupableData(
+    client,
+    entity,
+    project,
+    spec
+  );
+  return _.uniq(groupableData.map(g => g.row.datasetName));
 };
 export const fetchDatasetVersionsForSpecAndName = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
   spec: FilterAndGroupSpec,
   name: string
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(['DV1', 'DV2', 'DV3']);
+): Promise<VersionDetails[]> => {
+  const groupableData = await getLeaderboardGroupableData(
+    client,
+    entity,
+    project,
+    spec
+  );
+
+  return _.uniqBy(
+    groupableData.map(g => ({digest: g.row.datasetVersion, index: -1})),
+    o => `${o.digest}-${o.index}`
+  );
 };
+
 export const fetchModelNamesForSpec = async (
   spec: FilterAndGroupSpec
 ): Promise<string[]> => {
   // TODO
   return Promise.resolve(['M1', 'M2', 'M3']);
 };
-export const fetchModelVersionsForSpecndName = async (
+export const fetchModelVersionsForSpecAndName = async (
   spec: FilterAndGroupSpec,
   name: string
 ): Promise<string[]> => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
index fb70ac457f5..8e11c5a3805 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
@@ -53,13 +53,15 @@ export type GroupedLeaderboardData = {
   };
 };
 
-export type GroupedLeaderboardModelGroup = {
+export type GroupedLeaderboardModelGroup<
+  T extends any = LeaderboardValueRecord[]
+> = {
   datasetGroups: {
     [datasetGroup: string]: {
       scorerGroups: {
         [scorerGroup: string]: {
           metricPathGroups: {
-            [metricPathGroup: string]: LeaderboardValueRecord[];
+            [metricPathGroup: string]: T;
           };
         };
       };
@@ -67,12 +69,12 @@ export type GroupedLeaderboardModelGroup = {
   };
 };
 
-export const getLeaderboardData = async (
+export const getEvaluationObjectsForSpec = async (
   client: TraceServerClient,
   entity: string,
   project: string,
-  spec: FilterAndGroupSpec = {}
-): Promise<GroupedLeaderboardData> => {
+  spec: FilterAndGroupSpec
+) => {
   const atLeastOneStarName = spec.sourceEvaluations?.some(
     sourceEvaluation => sourceEvaluation.name === ALL_VALUE
   );
@@ -116,6 +118,22 @@ export const getLeaderboardData = async (
     });
   }
 
+  return allEvaluationObjectsRes;
+};
+
+export const getLeaderboardGroupableData = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
+  spec: FilterAndGroupSpec = {}
+): Promise<GroupableLeaderboardValueRecord[]> => {
+  const allEvaluationObjectsRes = await getEvaluationObjectsForSpec(
+    client,
+    entity,
+    project,
+    spec
+  );
+
   const evaluationObjectDigestMap = new Map<
     string,
     {versions: Map<string, TraceObjSchema>; versionOrder: string[]}
@@ -402,10 +420,22 @@ export const getLeaderboardData = async (
     return {include: true, groupableRow};
   });
 
-  const groupableData = filterableGroupableData
+  return filterableGroupableData
     .filter(entry => entry.include)
     .map(entry => entry.groupableRow);
-
+};
+export const getLeaderboardData = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
+  spec: FilterAndGroupSpec = {}
+): Promise<GroupedLeaderboardData> => {
+  const groupableData = await getLeaderboardGroupableData(
+    client,
+    entity,
+    project,
+    spec
+  );
   const finalData: GroupedLeaderboardData = {
     modelGroups: _.mapValues(
       _.groupBy(groupableData, 'modelGroup'),
@@ -423,9 +453,6 @@ export const getLeaderboardData = async (
                         _.groupBy(scorerGroup, 'metricPathGroup'),
                         metricPathGroup => {
                           return metricPathGroup.map(row => row.row);
-                          //   .sort(
-                          //     (a, b) => a.sortKey - b.sortKey
-                          //   )[0].row;
                         }
                       ),
                     };

From 04523e4139d4f1b921ab05923cfd323ad48763ac Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 17:58:22 -0700
Subject: [PATCH 064/169] Yuk that was a lot

---
 .../LeaderboardConfigEditor.tsx               | 157 +++++++++++++++---
 .../pages/LeaderboardPage/LeaderboardGrid.tsx |  13 +-
 .../pages/LeaderboardPage/LeaderboardPage.tsx |   1 +
 .../query/configEditorQuery.ts                | 101 ++++++++---
 .../LeaderboardPage/query/leaderboardQuery.ts |   6 +-
 5 files changed, 227 insertions(+), 51 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 1b72371f87a..3ff3a3b1d8d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -167,7 +167,17 @@ const ConfigEditor: React.FC<{
         datasets={config.datasets}
         updateConfig={updateConfig}
       />
-      <ModelsConfig models={config.models} updateConfig={updateConfig} />
+      <ModelsConfig
+        entity={entity}
+        project={project}
+        parentSpec={{
+          sourceEvaluations: config.sourceEvaluations,
+          datasets: config.datasets,
+          // Purposely exclude models
+        }}
+        models={config.models}
+        updateConfig={updateConfig}
+      />
     </>
   );
 };
@@ -381,7 +391,7 @@ const SourceEvaluationItem: React.FC<{
           <MenuItem value={ALL_VALUE}>All</MenuItem>
           {versions.map(version => (
             <MenuItem key={version.digest} value={version.digest}>
-              v{version.index} ({version.digest.slice(0, 6)})
+              <VersionItem version={version} />
             </MenuItem>
           ))}
         </Select>
@@ -439,7 +449,7 @@ const DatasetsConfig: React.FC<{
             datasets: [
               {
                 name: dataset.name,
-                version: '*',
+                version: ALL_VALUE,
               },
             ],
           }}
@@ -489,8 +499,18 @@ const DatasetItem: React.FC<{
         dataset.name
       ).then(setVersions);
     }
-    fetchScorerNamesForSpec(parentSpec).then(setScorerNames);
-  }, [dataset.name, entity, getTraceServerClient, project, parentSpec]);
+    fetchScorerNamesForSpec(getTraceServerClient(), entity, project, {
+      ...parentSpec,
+      datasets: [dataset],
+    }).then(setScorerNames);
+  }, [
+    dataset.name,
+    entity,
+    getTraceServerClient,
+    project,
+    parentSpec,
+    dataset,
+  ]);
 
   const handleNameChange = (event: SelectChangeEvent<string>) => {
     const newName = event.target.value as string;
@@ -576,7 +596,7 @@ const DatasetItem: React.FC<{
             <MenuItem value={ALL_VALUE}>All</MenuItem>
             {versions.map(version => (
               <MenuItem key={version.digest} value={version.digest}>
-                v{version.index} ({version.digest.slice(0, 6)})
+                <VersionItem version={version} />
               </MenuItem>
             ))}
           </Select>
@@ -598,6 +618,19 @@ const DatasetItem: React.FC<{
       {dataset.scorers?.map((scorer, scorerIndex) => (
         <ScorerItem
           key={scorerIndex}
+          entity={entity}
+          project={project}
+          parentSpec={{
+            sourceEvaluations: parentSpec.sourceEvaluations,
+            datasets: [
+              {
+                ...(parentSpec.datasets?.[0] ?? {
+                  name: ALL_VALUE,
+                  version: ALL_VALUE,
+                }),
+              },
+            ],
+          }}
           scorer={scorer}
           scorerNames={scorerNames}
           updateConfig={updateConfig}
@@ -613,6 +646,9 @@ const DatasetItem: React.FC<{
 };
 
 const ScorerItem: React.FC<{
+  entity: string;
+  project: string;
+  parentSpec: FilterAndGroupSpec;
   scorer: FilterAndGroupDatasetScorerSpec;
   scorerNames: string[];
   updateConfig: (
@@ -620,16 +656,43 @@ const ScorerItem: React.FC<{
   ) => void;
   datasetIndex: number;
   scorerIndex: number;
-}> = ({scorer, scorerNames, updateConfig, datasetIndex, scorerIndex}) => {
-  const [versions, setVersions] = useState<string[]>([]);
+}> = ({
+  entity,
+  project,
+  parentSpec,
+  scorer,
+  scorerNames,
+  updateConfig,
+  datasetIndex,
+  scorerIndex,
+}) => {
+  const getTraceServerClient = useGetTraceServerClientContext();
+  const [versions, setVersions] = useState<VersionDetails[]>([]);
   const [metricPaths, setMetricPaths] = useState<string[]>([]);
 
   useEffect(() => {
     if (scorer.name && scorer.name !== ALL_VALUE) {
-      fetchScorerVersionsForSpecAndName({}, scorer.name).then(setVersions);
+      fetchScorerVersionsForSpecAndName(
+        getTraceServerClient(),
+        entity,
+        project,
+        parentSpec,
+        scorer.name
+      ).then(setVersions);
     }
-    fetchMetricPathsForSpec({}).then(setMetricPaths);
-  }, [scorer.name]);
+    fetchMetricPathsForSpec(getTraceServerClient(), entity, project, {
+      ...parentSpec,
+      datasets: [
+        {
+          ...(parentSpec.datasets?.[0] ?? {
+            name: ALL_VALUE,
+            version: ALL_VALUE,
+          }),
+          scorers: [scorer],
+        },
+      ],
+    }).then(setMetricPaths);
+  }, [entity, getTraceServerClient, parentSpec, project, scorer, scorer.name]);
 
   const handleNameChange = (event: SelectChangeEvent<string>) => {
     const newName = event.target.value as string;
@@ -748,8 +811,8 @@ const ScorerItem: React.FC<{
             disabled={scorer.name === ALL_VALUE}>
             <MenuItem value={ALL_VALUE}>All</MenuItem>
             {versions.map(version => (
-              <MenuItem key={version} value={version}>
-                {version}
+              <MenuItem key={version.digest} value={version.digest}>
+                <VersionItem version={version} />
               </MenuItem>
             ))}
           </Select>
@@ -906,16 +969,25 @@ const MetricItem: React.FC<{
 };
 
 const ModelsConfig: React.FC<{
+  entity: string;
+  project: string;
+  parentSpec: FilterAndGroupSpec;
   models: FilterAndGroupDatasetSpec[] | undefined;
   updateConfig: (
     updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
   ) => void;
-}> = ({models, updateConfig}) => {
+}> = ({entity, project, parentSpec, models, updateConfig}) => {
+  const getTraceServerClient = useGetTraceServerClientContext();
   const [modelNames, setModelNames] = useState<string[]>([]);
 
   useEffect(() => {
-    fetchModelNamesForSpec({}).then(setModelNames);
-  }, []);
+    fetchModelNamesForSpec(
+      getTraceServerClient(),
+      entity,
+      project,
+      parentSpec
+    ).then(setModelNames);
+  }, [entity, project, getTraceServerClient, parentSpec]);
 
   const handleAddModel = () => {
     updateConfig(spec => ({
@@ -935,6 +1007,18 @@ const ModelsConfig: React.FC<{
       {models?.map((model, index) => (
         <ModelItem
           key={index}
+          entity={entity}
+          project={project}
+          parentSpec={{
+            sourceEvaluations: parentSpec.sourceEvaluations,
+            datasets: parentSpec.datasets,
+            models: [
+              {
+                name: model.name,
+                version: ALL_VALUE,
+              },
+            ],
+          }}
           model={model}
           modelNames={modelNames}
           updateConfig={updateConfig}
@@ -949,20 +1033,39 @@ const ModelsConfig: React.FC<{
 };
 
 const ModelItem: React.FC<{
+  entity: string;
+  project: string;
+  parentSpec: FilterAndGroupSpec;
   model: FilterAndGroupDatasetSpec;
   modelNames: string[];
   updateConfig: (
     updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
   ) => void;
   index: number;
-}> = ({model, modelNames, updateConfig, index}) => {
-  const [versions, setVersions] = useState<string[]>([]);
+}> = ({
+  entity,
+  project,
+  parentSpec,
+  model,
+  modelNames,
+  updateConfig,
+  index,
+}) => {
+  const getTraceServerClient = useGetTraceServerClientContext();
+
+  const [versions, setVersions] = useState<VersionDetails[]>([]);
 
   useEffect(() => {
     if (model.name && model.name !== ALL_VALUE) {
-      fetchModelVersionsForSpecAndName({}, model.name).then(setVersions);
+      fetchModelVersionsForSpecAndName(
+        getTraceServerClient(),
+        entity,
+        project,
+        parentSpec,
+        model.name
+      ).then(setVersions);
     }
-  }, [model.name]);
+  }, [model.name, entity, project, getTraceServerClient, parentSpec]);
 
   const handleNameChange = (event: SelectChangeEvent<string>) => {
     const newName = event.target.value as string;
@@ -1025,8 +1128,8 @@ const ModelItem: React.FC<{
             disabled={model.name === ALL_VALUE}>
             <MenuItem value={ALL_VALUE}>All</MenuItem>
             {versions.map(version => (
-              <MenuItem key={version} value={version}>
-                {version}
+              <MenuItem key={version.digest} value={version.digest}>
+                <VersionItem version={version} />
               </MenuItem>
             ))}
           </Select>
@@ -1047,3 +1150,13 @@ const ModelItem: React.FC<{
     </Box>
   );
 };
+
+const VersionItem: React.FC<{
+  version: VersionDetails;
+}> = ({version}) => {
+  return (
+    <div>
+      v{version.index} ({version.digest.slice(0, 6)})
+    </div>
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index e8b827cc692..e71aad99dbb 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -21,10 +21,12 @@ import {
   GroupedLeaderboardModelGroup,
   LeaderboardValueRecord,
 } from './query/leaderboardQuery';
+import {FilterAndGroupSpec} from './types/leaderboardConfigType';
 
 interface LeaderboardGridProps {
   entity: string;
   project: string;
+  config: FilterAndGroupSpec;
   data: GroupedLeaderboardData;
   loading: boolean;
   onCellClick: (record: LeaderboardValueRecord) => void;
@@ -39,6 +41,7 @@ type RowData = {
 export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   entity,
   project,
+  config,
   data,
   loading,
   onCellClick,
@@ -47,16 +50,22 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
 
   const getColorForScore = useCallback(
     (datasetGroup, scorerGroup, metricPathGroup, score) => {
+      const shouldMinimize = !!config.datasets
+        ?.find(d => d.name === datasetGroup)
+        ?.scorers?.find(s => s.name === scorerGroup)
+        ?.metrics?.find(m => m.path === metricPathGroup)?.shouldMinimize;
       if (score == null) {
         return 'transparent';
       }
       const {min, max} =
         columnStats.datasetGroups[datasetGroup].scorerGroups[scorerGroup]
           .metricPathGroups[metricPathGroup];
-      const normalizedScore = (score - min) / (max - min);
+      const normalizedScore = shouldMinimize
+        ? (max - score) / (max - min)
+        : (score - min) / (max - min);
       return `hsl(${120 * normalizedScore}, 70%, 85%)`;
     },
-    [columnStats.datasetGroups]
+    [columnStats.datasetGroups, config.datasets]
   );
 
   const rows: RowData[] = useMemo(() => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 45ced5a88ad..07cce3282f3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -151,6 +151,7 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
           <LeaderboardGrid
             entity={entity}
             project={project}
+            config={currentConfig.config.dataSelectionSpec}
             loading={loading}
             data={data}
             onCellClick={handleCellClick}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
index 2dbbfb1edb6..45c8657b5f4 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
@@ -1,22 +1,31 @@
 /**
- * This file is not performant yet.
+ * This file implements a memoized version of getLeaderboardGroupableData for improved performance.
  */
 
-import {ObjectRef} from '@wandb/weave/react';
 import _ from 'lodash';
 
-import {parseRefMaybe} from '../../../../Browse2/SmallRef';
 import {TraceServerClient} from '../../wfReactInterface/traceServerClient';
 import {projectIdFromParts} from '../../wfReactInterface/tsDataModelHooks';
 import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
-import {
-  getEvaluationObjectsForSpec,
-  getLeaderboardData,
-  getLeaderboardGroupableData,
-} from './leaderboardQuery';
+import {getLeaderboardGroupableData} from './leaderboardQuery';
 
 export type VersionDetails = {digest: string; index: number};
 
+const memoedGetLeaderboardGroupableData = _.memoize(
+  async (
+    client: TraceServerClient,
+    entity: string,
+    project: string,
+    spec: FilterAndGroupSpec = {}
+  ) => {
+    return getLeaderboardGroupableData(client, entity, project, spec);
+  },
+  (client, entity, project, spec) => {
+    // Create a cache key based on the function arguments
+    return JSON.stringify({entity, project, spec});
+  }
+);
+
 export const fetchEvaluationNames = async (
   client: TraceServerClient,
   entity: string,
@@ -68,7 +77,7 @@ export const fetchDatasetNamesForSpec = async (
   spec: FilterAndGroupSpec
 ): Promise<string[]> => {
   // This is def a hacky/slow solution, just getting it working.
-  const groupableData = await getLeaderboardGroupableData(
+  const groupableData = await memoedGetLeaderboardGroupableData(
     client,
     entity,
     project,
@@ -83,7 +92,7 @@ export const fetchDatasetVersionsForSpecAndName = async (
   spec: FilterAndGroupSpec,
   name: string
 ): Promise<VersionDetails[]> => {
-  const groupableData = await getLeaderboardGroupableData(
+  const groupableData = await memoedGetLeaderboardGroupableData(
     client,
     entity,
     project,
@@ -97,34 +106,82 @@ export const fetchDatasetVersionsForSpecAndName = async (
 };
 
 export const fetchModelNamesForSpec = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
   spec: FilterAndGroupSpec
 ): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(['M1', 'M2', 'M3']);
+  // This is def a hacky/slow solution, just getting it working.
+  const groupableData = await memoedGetLeaderboardGroupableData(
+    client,
+    entity,
+    project,
+    spec
+  );
+  return _.uniq(groupableData.map(g => g.row.modelName));
 };
 export const fetchModelVersionsForSpecAndName = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
   spec: FilterAndGroupSpec,
   name: string
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(['MV1', 'MV2', 'MV3']);
+): Promise<VersionDetails[]> => {
+  // This is def a hacky/slow solution, just getting it working.
+  const groupableData = await memoedGetLeaderboardGroupableData(
+    client,
+    entity,
+    project,
+    spec
+  );
+  return _.uniqBy(
+    groupableData.map(g => ({digest: g.row.modelVersion, index: -1})),
+    o => `${o.digest}-${o.index}`
+  );
 };
 export const fetchScorerNamesForSpec = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
   spec: FilterAndGroupSpec
 ): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(['S1', 'S2', 'S3']);
+  const groupableData = await memoedGetLeaderboardGroupableData(
+    client,
+    entity,
+    project,
+    spec
+  );
+  return _.uniq(groupableData.map(g => g.row.scorerName));
 };
 export const fetchScorerVersionsForSpecAndName = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
   spec: FilterAndGroupSpec,
   name: string
-): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(['SV1', 'SV2', 'SV3']);
+): Promise<VersionDetails[]> => {
+  const groupableData = await memoedGetLeaderboardGroupableData(
+    client,
+    entity,
+    project,
+    spec
+  );
+  return _.uniqBy(
+    groupableData.map(g => ({digest: g.row.scorerVersion, index: -1})),
+    o => `${o.digest}-${o.index}`
+  );
 };
 export const fetchMetricPathsForSpec = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
   spec: FilterAndGroupSpec
 ): Promise<string[]> => {
-  // TODO
-  return Promise.resolve(['MP1', 'MP2', 'MP3']);
+  const groupableData = await memoedGetLeaderboardGroupableData(
+    client,
+    entity,
+    project,
+    spec
+  );
+  return _.uniq(groupableData.map(g => g.row.metricPath));
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
index 8e11c5a3805..e18ddb8ccfc 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
@@ -396,11 +396,7 @@ export const getLeaderboardGroupableData = async (
             (scorer.version === ALL_VALUE ||
               scorer.version === row.scorerVersion)
         );
-      console.log({
-        scorerSpec,
-        dss: datasetSpec.scorers,
-        row,
-      });
+
       if (!scorerSpec) {
         return {include: false, groupableRow};
       }

From 04112496760fa82d7acd3155519959aab2babad3 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 10 Oct 2024 18:05:29 -0700
Subject: [PATCH 065/169] Checkpoint

---
 .../LeaderboardConfigEditor.tsx               |   7 +-
 .../pages/LeaderboardPage/LeaderboardGrid.tsx | 105 +++++++++---------
 2 files changed, 60 insertions(+), 52 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 3ff3a3b1d8d..5e4edafb4fd 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -688,7 +688,12 @@ const ScorerItem: React.FC<{
             name: ALL_VALUE,
             version: ALL_VALUE,
           }),
-          scorers: [scorer],
+          scorers: [
+            {
+              ...scorer,
+              metrics: [],
+            },
+          ],
         },
       ],
     }).then(setMetricPaths);
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
index e71aad99dbb..04b6e96e182 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
@@ -2,8 +2,8 @@ import {Box} from '@mui/material';
 import {
   GridColDef,
   GridColumnGroup,
-  GridColumnGroupingModel,
-  GridColumnNode,
+  // GridColumnGroupingModel,
+  // GridColumnNode,
   GridLeafColumn,
   GridRenderCellParams,
   GridSortDirection,
@@ -118,7 +118,8 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
                 ([metricPathGroupName, metricPathGroup]) => {
                   return {
                     field: `${datasetGroupName}--${scorerGroupName}--${metricPathGroupName}`,
-                    headerName: `${metricPathGroupName.split('.').pop()}`,
+                    headerName: `${metricPathGroupName}`,
+                    // headerName: `${metricPathGroupName.split('.').pop()}`,
                     minWidth: 100,
                     flex: 1,
                     valueGetter: (value: any, row: RowData) => {
@@ -257,43 +258,45 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
       }
     );
 
-    let finalGroupingModel: GridColumnGroupingModel = datasetGroups.filter(
-      c => 'groupId' in c
-    ) as GridColumnGroup[];
-    finalGroupingModel = walkGroupingModel(finalGroupingModel, node => {
-      if ('groupId' in node) {
-        if (node.children.length === 1) {
-          if (
-            'groupId' in node.children[0] &&
-            !node.headerName?.includes(':') &&
-            !node.children[0].headerName?.includes(':')
-          ) {
-            const currNode = node;
-            node = node.children[0];
-            node.headerName = currNode.headerName + '.' + node.headerName;
-          } else {
-            // pass
-            // node = node.children[0];
-          }
-        }
-      }
-      return node;
-    }) as GridColumnGroup[];
-    finalGroupingModel = walkGroupingModel(finalGroupingModel, node => {
-      if ('groupId' in node) {
-        node.renderHeaderGroup = params => {
-          const ref = parseRefMaybe(
-            `weave:///${entity}/${project}/object/${node.headerName}` ?? ''
-          );
-          // console.log(node.headerName, ref);
-          if (ref) {
-            return <SmallRef objRef={ref} />;
-          }
-          return <div>{node.headerName}</div>;
-        };
-      }
-      return node;
-    }) as GridColumnGroup[];
+    const finalGroupingModel = datasetGroups;
+
+    // let finalGroupingModel: GridColumnGroupingModel = datasetGroups.filter(
+    //   c => 'groupId' in c
+    // ) as GridColumnGroup[];
+    // finalGroupingModel = walkGroupingModel(finalGroupingModel, node => {
+    //   if ('groupId' in node) {
+    //     if (node.children.length === 1) {
+    //       if (
+    //         'groupId' in node.children[0] &&
+    //         !node.headerName?.includes(':') &&
+    //         !node.children[0].headerName?.includes(':')
+    //       ) {
+    //         const currNode = node;
+    //         node = node.children[0];
+    //         node.headerName = currNode.headerName + '.' + node.headerName;
+    //       } else {
+    //         // pass
+    //         // node = node.children[0];
+    //       }
+    //     }
+    //   }
+    //   return node;
+    // }) as GridColumnGroup[];
+    // finalGroupingModel = walkGroupingModel(finalGroupingModel, node => {
+    //   if ('groupId' in node) {
+    //     node.renderHeaderGroup = params => {
+    //       const ref = parseRefMaybe(
+    //         `weave:///${entity}/${project}/object/${node.headerName}` ?? ''
+    //       );
+    //       // console.log(node.headerName, ref);
+    //       if (ref) {
+    //         return <SmallRef objRef={ref} />;
+    //       }
+    //       return <div>{node.headerName}</div>;
+    //     };
+    //   }
+    //   return node;
+    // }) as GridColumnGroup[];
 
     return finalGroupingModel;
   }, [columnStats.datasetGroups, entity, project]);
@@ -491,17 +494,17 @@ const defaultGetSortComparator =
     return aValue.localeCompare(bValue);
   };
 
-const walkGroupingModel = (
-  nodes: GridColumnNode[],
-  fn: (node: GridColumnNode) => GridColumnNode
-) => {
-  return nodes.map(node => {
-    if ('children' in node) {
-      node.children = walkGroupingModel(node.children, fn);
-    }
-    return fn(node);
-  });
-};
+// const walkGroupingModel = (
+//   nodes: GridColumnNode[],
+//   fn: (node: GridColumnNode) => GridColumnNode
+// ) => {
+//   return nodes.map(node => {
+//     if ('children' in node) {
+//       node.children = walkGroupingModel(node.children, fn);
+//     }
+//     return fn(node);
+//   });
+// };
 
 // // TODO: refactor to accumulate common root columns in a single pass
 //   const splitCommonRootLeafColumns = (columns: GridLeafColumn[], groupIdPrefix: string, depth: number = 0): GridColumnNode[] => {

From c266f1e346bcfb7c6ef87425929cde85d123cc28 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 18:15:30 -0700
Subject: [PATCH 066/169] blend it all together

---
 .../LeaderboardConfigEditor.tsx               |  18 +-
 .../pages/LeaderboardPage/LeaderboardGrid.tsx | 539 ------------------
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  31 +-
 .../query/configEditorQuery.ts                |   4 +-
 .../LeaderboardPage/query/hookAdapters.ts     |  35 --
 .../LeaderboardPage/query/leaderboardQuery.ts | 466 ---------------
 .../types/leaderboardConfigType.ts            |  45 --
 7 files changed, 14 insertions(+), 1124 deletions(-)
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/hookAdapters.ts
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 5e4edafb4fd..8fb55d1de84 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -16,6 +16,15 @@ import {
 import IconButton from '@mui/material/IconButton';
 import React, {useEffect, useMemo, useState} from 'react';
 
+import {
+  ALL_VALUE,
+  FilterAndGroupDatasetScorerMetricSpec,
+  FilterAndGroupDatasetScorerSpec,
+  FilterAndGroupDatasetSpec,
+  FilterAndGroupSourceEvaluationSpec,
+  FilterAndGroupSpec,
+  LeaderboardConfigType,
+} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {
   fetchDatasetNamesForSpec,
@@ -29,15 +38,6 @@ import {
   fetchScorerVersionsForSpecAndName,
   VersionDetails,
 } from './query/configEditorQuery';
-import {
-  ALL_VALUE,
-  FilterAndGroupDatasetScorerMetricSpec,
-  FilterAndGroupDatasetScorerSpec,
-  FilterAndGroupDatasetSpec,
-  FilterAndGroupSourceEvaluationSpec,
-  FilterAndGroupSpec,
-  LeaderboardConfigType,
-} from './types/leaderboardConfigType';
 
 export const LeaderboardConfigEditor: React.FC<{
   entity: string;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
deleted file mode 100644
index 04b6e96e182..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardGrid.tsx
+++ /dev/null
@@ -1,539 +0,0 @@
-import {Box} from '@mui/material';
-import {
-  GridColDef,
-  GridColumnGroup,
-  // GridColumnGroupingModel,
-  // GridColumnNode,
-  GridLeafColumn,
-  GridRenderCellParams,
-  GridSortDirection,
-  GridSortItem,
-} from '@mui/x-data-grid-pro';
-import _ from 'lodash';
-import React, {useCallback, useEffect, useMemo, useState} from 'react';
-
-import {NotApplicable} from '../../../Browse2/NotApplicable';
-import {parseRefMaybe, SmallRef} from '../../../Browse2/SmallRef';
-import {StyledDataGrid} from '../../StyledDataGrid';
-import {PaginationButtons} from '../CallsPage/CallsTableButtons';
-import {
-  GroupedLeaderboardData,
-  GroupedLeaderboardModelGroup,
-  LeaderboardValueRecord,
-} from './query/leaderboardQuery';
-import {FilterAndGroupSpec} from './types/leaderboardConfigType';
-
-interface LeaderboardGridProps {
-  entity: string;
-  project: string;
-  config: FilterAndGroupSpec;
-  data: GroupedLeaderboardData;
-  loading: boolean;
-  onCellClick: (record: LeaderboardValueRecord) => void;
-}
-
-type RowData = {
-  id: string;
-  modelGroupName: string;
-  modelGroup: GroupedLeaderboardModelGroup;
-};
-
-export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
-  entity,
-  project,
-  config,
-  data,
-  loading,
-  onCellClick,
-}) => {
-  const columnStats = useMemo(() => getColumnStats(data), [data]);
-
-  const getColorForScore = useCallback(
-    (datasetGroup, scorerGroup, metricPathGroup, score) => {
-      const shouldMinimize = !!config.datasets
-        ?.find(d => d.name === datasetGroup)
-        ?.scorers?.find(s => s.name === scorerGroup)
-        ?.metrics?.find(m => m.path === metricPathGroup)?.shouldMinimize;
-      if (score == null) {
-        return 'transparent';
-      }
-      const {min, max} =
-        columnStats.datasetGroups[datasetGroup].scorerGroups[scorerGroup]
-          .metricPathGroups[metricPathGroup];
-      const normalizedScore = shouldMinimize
-        ? (max - score) / (max - min)
-        : (score - min) / (max - min);
-      return `hsl(${120 * normalizedScore}, 70%, 85%)`;
-    },
-    [columnStats.datasetGroups, config.datasets]
-  );
-
-  const rows: RowData[] = useMemo(() => {
-    const rowData: RowData[] = [];
-    Object.entries(data.modelGroups).forEach(([modelGroupName, modelGroup]) => {
-      rowData.push({
-        id: modelGroupName,
-        modelGroupName,
-        modelGroup,
-      });
-    });
-    return rowData;
-  }, [data]);
-
-  const columns: Array<GridColDef<RowData>> = useMemo(
-    () => [
-      {
-        field: 'modelGroupName',
-        headerName: 'Model',
-        minWidth: 150,
-        flex: 1,
-        renderCell: (params: GridRenderCellParams) => {
-          const modelRef = parseRefMaybe(
-            `weave:///${entity}/${project}/object/${params.value}` ?? ''
-          );
-          if (modelRef) {
-            return (
-              <div
-                style={{
-                  width: '100%',
-                  height: '100%',
-                  alignContent: 'center',
-                  display: 'flex',
-                  alignItems: 'center',
-                  justifyContent: 'center',
-                  lineHeight: '20px',
-                }}>
-                <SmallRef objRef={modelRef} />
-              </div>
-            );
-          }
-          return <div>{params.value}</div>;
-        },
-      },
-      ...Object.entries(columnStats.datasetGroups).flatMap(
-        ([datasetGroupName, datasetGroup]) =>
-          Object.entries(datasetGroup.scorerGroups).flatMap(
-            ([scorerGroupName, scorerGroup]) => {
-              return Object.entries(scorerGroup.metricPathGroups).map(
-                ([metricPathGroupName, metricPathGroup]) => {
-                  return {
-                    field: `${datasetGroupName}--${scorerGroupName}--${metricPathGroupName}`,
-                    headerName: `${metricPathGroupName}`,
-                    // headerName: `${metricPathGroupName.split('.').pop()}`,
-                    minWidth: 100,
-                    flex: 1,
-                    valueGetter: (value: any, row: RowData) => {
-                      return valueFromRowData(
-                        row,
-                        datasetGroupName,
-                        scorerGroupName,
-                        metricPathGroupName
-                      );
-                    },
-                    getSortComparator: defaultGetSortComparator,
-
-                    renderCell: (params: GridRenderCellParams) => {
-                      const value = valueFromRowData(
-                        params.row,
-                        datasetGroupName,
-                        scorerGroupName,
-                        metricPathGroupName
-                      );
-                      let inner: React.ReactNode = value;
-                      if (inner == null) {
-                        inner = <NotApplicable />;
-                      } else if (typeof inner === 'number') {
-                        if (inner < 1) {
-                          inner = `${(inner * 100).toFixed(2)}%`;
-                        } else {
-                          inner = `${inner.toFixed(2)}`;
-                        }
-                      } else {
-                        inner = JSON.stringify(params.value);
-                      }
-                      const record = recordFromRowData(
-                        params.row,
-                        datasetGroupName,
-                        scorerGroupName,
-                        metricPathGroupName
-                      );
-                      return (
-                        <div
-                          style={{
-                            width: '100%',
-                            height: '100%',
-                            display: 'flex',
-                            alignItems: 'center',
-                            justifyContent: 'center',
-                            backgroundColor: getColorForScore(
-                              datasetGroupName,
-                              scorerGroupName,
-                              metricPathGroupName,
-                              value
-                            ),
-                          }}
-                          onClick={() => record && onCellClick(record)}>
-                          {inner}
-                        </div>
-                      );
-                    },
-                  };
-                }
-              );
-            }
-          )
-      ),
-    ],
-    [columnStats.datasetGroups, entity, getColorForScore, onCellClick, project]
-  );
-
-  const groupingModel: GridColumnGroup[] = useMemo(() => {
-    const datasetGroups: GridColumnGroup[] = [];
-    Object.entries(columnStats.datasetGroups).forEach(
-      ([datasetGroupName, datasetGroup]) => {
-        const datasetColGroup: GridColumnGroup = {
-          groupId: datasetGroupName,
-          headerName: datasetGroupName,
-          children: [],
-          renderHeaderGroup: params => {
-            const ref = parseRefMaybe(
-              `weave:///${entity}/${project}/object/${datasetGroupName}` ?? ''
-            );
-            if (ref) {
-              return <SmallRef objRef={ref} />;
-            }
-            return <div>{datasetGroupName}</div>;
-          },
-        };
-        datasetGroups.push(datasetColGroup);
-        Object.entries(datasetGroup.scorerGroups).forEach(
-          ([scorerGroupName, scorerGroup]) => {
-            const scorerColGroup: GridColumnGroup = {
-              groupId: `${datasetGroupName}--${scorerGroupName}`,
-              headerName: scorerGroupName,
-              children: [],
-              renderHeaderGroup: params => {
-                const ref = parseRefMaybe(
-                  `weave:///${entity}/${project}/op/${scorerGroupName}` ?? ''
-                );
-                // console.log(node.headerName, ref);
-                if (ref) {
-                  return <SmallRef objRef={ref} />;
-                }
-                return <div>{scorerGroupName}</div>;
-              },
-            };
-            datasetColGroup.children.push(scorerColGroup);
-            Object.entries(scorerGroup.metricPathGroups).forEach(
-              ([metricPathGroupName, metricPathGroup]) => {
-                // const prefix = `${datasetGroupName}--${scorerGroupName}--`
-                const metricPathParts = metricPathGroupName.split('.');
-                let targetParentGroup = scorerColGroup;
-                for (let i = 0; i < metricPathParts.length - 2; i++) {
-                  const part = metricPathParts[i];
-                  let existingChild = targetParentGroup.children.find(
-                    child => 'groupId' in child && child.groupId === part
-                  );
-                  if (!existingChild) {
-                    existingChild = {
-                      groupId: part,
-                      headerName: part,
-                      children: [],
-                    };
-                    targetParentGroup.children.push(existingChild);
-                  }
-                  targetParentGroup = existingChild as GridColumnGroup;
-                }
-                // const finalPart = metricPathParts[metricPathParts.length - 1];
-                const metricPathColGroup: GridLeafColumn = {
-                  field: `${datasetGroupName}--${scorerGroupName}--${metricPathGroupName}`,
-                };
-                targetParentGroup.children.push(metricPathColGroup);
-              }
-            );
-
-            // scorerColGroup.children = splitCommonRootLeafColumns(scorerColGroup.children as GridLeafColumn[], scorerColGroup.groupId);
-          }
-        );
-      }
-    );
-
-    const finalGroupingModel = datasetGroups;
-
-    // let finalGroupingModel: GridColumnGroupingModel = datasetGroups.filter(
-    //   c => 'groupId' in c
-    // ) as GridColumnGroup[];
-    // finalGroupingModel = walkGroupingModel(finalGroupingModel, node => {
-    //   if ('groupId' in node) {
-    //     if (node.children.length === 1) {
-    //       if (
-    //         'groupId' in node.children[0] &&
-    //         !node.headerName?.includes(':') &&
-    //         !node.children[0].headerName?.includes(':')
-    //       ) {
-    //         const currNode = node;
-    //         node = node.children[0];
-    //         node.headerName = currNode.headerName + '.' + node.headerName;
-    //       } else {
-    //         // pass
-    //         // node = node.children[0];
-    //       }
-    //     }
-    //   }
-    //   return node;
-    // }) as GridColumnGroup[];
-    // finalGroupingModel = walkGroupingModel(finalGroupingModel, node => {
-    //   if ('groupId' in node) {
-    //     node.renderHeaderGroup = params => {
-    //       const ref = parseRefMaybe(
-    //         `weave:///${entity}/${project}/object/${node.headerName}` ?? ''
-    //       );
-    //       // console.log(node.headerName, ref);
-    //       if (ref) {
-    //         return <SmallRef objRef={ref} />;
-    //       }
-    //       return <div>{node.headerName}</div>;
-    //     };
-    //   }
-    //   return node;
-    // }) as GridColumnGroup[];
-
-    return finalGroupingModel;
-  }, [columnStats.datasetGroups, entity, project]);
-
-  const [sortModel, setSortModel] = useState<GridSortItem[]>([]);
-
-  useEffect(() => {
-    if (sortModel.length === 0 && columns.length > 1 && !loading) {
-      setSortModel([{field: columns[1].field, sort: 'desc'}]);
-    }
-  }, [columns, loading, sortModel]);
-
-  return (
-    <Box
-      sx={{
-        height: '100%',
-        width: '100%',
-        display: 'flex',
-        flexDirection: 'column',
-        overflow: 'hidden',
-      }}>
-      <StyledDataGrid
-        rows={rows}
-        columns={columns}
-        columnGroupingModel={groupingModel}
-        disableRowSelectionOnClick
-        hideFooterSelectedRowCount
-        disableMultipleColumnsSorting={false}
-        columnHeaderHeight={40}
-        loading={loading}
-        sortModel={sortModel}
-        onSortModelChange={setSortModel}
-        sx={{
-          borderRadius: 0,
-          '& .MuiDataGrid-footerContainer': {
-            justifyContent: 'flex-start',
-          },
-          '& .MuiDataGrid-cell': {
-            cursor: 'pointer',
-          },
-          flexGrow: 1,
-          width: 'calc(100% + 1px)', // Add 1px to account for the right border
-          '& .MuiDataGrid-main': {
-            overflow: 'hidden',
-          },
-          '& .MuiDataGrid-virtualScroller': {
-            overflow: 'auto',
-          },
-          '& .MuiDataGrid-columnHeaders': {
-            overflow: 'hidden',
-          },
-        }}
-        slots={{
-          pagination: PaginationButtons,
-        }}
-      />
-    </Box>
-  );
-};
-
-type ColumnStats = {
-  datasetGroups: {
-    [datasetGroup: string]: {
-      scorerGroups: {
-        [scorerGroup: string]: {
-          metricPathGroups: {
-            [metricPathGroup: string]: {
-              min: number;
-              max: number;
-            };
-          };
-        };
-      };
-    };
-  };
-};
-const getColumnStats = (data: GroupedLeaderboardData): ColumnStats => {
-  const stats: ColumnStats = {
-    datasetGroups: {},
-  };
-
-  Object.values(data.modelGroups).forEach(modelGroup => {
-    Object.entries(modelGroup.datasetGroups).forEach(
-      ([datasetGroupName, datasetGroup]) => {
-        if (stats.datasetGroups[datasetGroupName] == null) {
-          stats.datasetGroups[datasetGroupName] = {
-            scorerGroups: {},
-          };
-        }
-        const currDatasetGroup = stats.datasetGroups[datasetGroupName];
-
-        Object.entries(datasetGroup.scorerGroups).forEach(
-          ([scorerGroupName, scorerGroup]) => {
-            if (currDatasetGroup.scorerGroups[scorerGroupName] == null) {
-              currDatasetGroup.scorerGroups[scorerGroupName] = {
-                metricPathGroups: {},
-              };
-            }
-            const currScorerGroup =
-              currDatasetGroup.scorerGroups[scorerGroupName];
-            Object.entries(scorerGroup.metricPathGroups).forEach(
-              ([metricPathGroupName, metricPathGroup]) => {
-                const groupMin = _.min(
-                  metricPathGroup.map(g => g.metricValue)
-                ) as number;
-                const groupMax = _.max(
-                  metricPathGroup.map(g => g.metricValue)
-                ) as number;
-                if (
-                  currScorerGroup.metricPathGroups[metricPathGroupName] == null
-                ) {
-                  currScorerGroup.metricPathGroups[metricPathGroupName] = {
-                    min: groupMin,
-                    max: groupMax,
-                  };
-                } else {
-                  currScorerGroup.metricPathGroups[metricPathGroupName].min =
-                    Math.min(
-                      currScorerGroup.metricPathGroups[metricPathGroupName].min,
-                      groupMin
-                    );
-                  currScorerGroup.metricPathGroups[metricPathGroupName].max =
-                    Math.max(
-                      currScorerGroup.metricPathGroups[metricPathGroupName].max,
-                      groupMax
-                    );
-                }
-              }
-            );
-          }
-        );
-      }
-    );
-  });
-
-  return stats;
-};
-
-const valueFromRowData = (
-  rowData: RowData,
-  datasetGroupName: string,
-  scorerGroupName: string,
-  metricPathGroupName: string
-): number | string | boolean | null => {
-  return getAggregatedResults(
-    rowData.modelGroup.datasetGroups[datasetGroupName]?.scorerGroups[
-      scorerGroupName
-    ]?.metricPathGroups[metricPathGroupName] ?? []
-  );
-};
-const recordFromRowData = (
-  rowData: RowData,
-  datasetGroupName: string,
-  scorerGroupName: string,
-  metricPathGroupName: string
-): LeaderboardValueRecord | null => {
-  return rowData.modelGroup.datasetGroups[datasetGroupName]?.scorerGroups[
-    scorerGroupName
-  ]?.metricPathGroups[metricPathGroupName]?.[0];
-};
-const getAggregatedResults = (
-  data: null | LeaderboardValueRecord[]
-): number | string | boolean | null => {
-  if (data == null || data.length === 0) {
-    return null;
-  }
-  if (data.length === 1) {
-    return data[0].metricValue;
-  }
-  return data.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())[0]
-    .metricValue;
-};
-
-const defaultGetSortComparator =
-  (dir: GridSortDirection) => (a: any, b: any) => {
-    const aValue = a;
-    const bValue = b;
-    if (aValue == null && bValue == null) {
-      return 0;
-    }
-    // Ignoring direction here allows nulls to always sort to the end
-    if (aValue == null) {
-      return 1;
-    }
-    if (bValue == null) {
-      return -1;
-    }
-    if (typeof aValue === 'number' && typeof bValue === 'number') {
-      if (dir === 'asc') {
-        return aValue - bValue;
-      } else {
-        return bValue - aValue;
-      }
-    }
-    return aValue.localeCompare(bValue);
-  };
-
-// const walkGroupingModel = (
-//   nodes: GridColumnNode[],
-//   fn: (node: GridColumnNode) => GridColumnNode
-// ) => {
-//   return nodes.map(node => {
-//     if ('children' in node) {
-//       node.children = walkGroupingModel(node.children, fn);
-//     }
-//     return fn(node);
-//   });
-// };
-
-// // TODO: refactor to accumulate common root columns in a single pass
-//   const splitCommonRootLeafColumns = (columns: GridLeafColumn[], groupIdPrefix: string, depth: number = 0): GridColumnNode[] => {
-//     if (columns.length < 2) {
-//       return columns;
-//     }
-//     const groups: {[key: string]: GridLeafColumn[]} = {};
-//     columns.forEach(col => {
-//       const key = col.field.split('.')[depth];
-//       if (groups[key] == null) {
-//         groups[key] = [];
-//       }
-//       groups[key].push(col);
-//     });
-
-//     return Object.entries(groups).map(([key, group]) => {
-//       if (group.length === 1) {
-//         return group[0];
-//       }
-//       const newGroupId = `${groupIdPrefix}.${key}`
-
-//       const groupCol: GridColumnGroup = {
-//         groupId: newGroupId,
-//         headerName: key,
-//         renderHeaderGroup: params => {
-//           return <div>{key}</div>
-//         },
-//         children: splitCommonRootLeafColumns(group, newGroupId, depth + 1),
-//       };
-//       return groupCol;
-//     });
-//   }
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 07cce3282f3..8e365e987bd 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,17 +1,12 @@
 import {Alert, Box, Typography} from '@mui/material';
 import {Button} from '@wandb/weave/components/Button';
 import React, {useCallback, useState} from 'react';
-import {useHistory} from 'react-router-dom';
 
-import {useWeaveflowRouteContext} from '../../context';
+import {LeaderboardGrid} from '../../views/Leaderboard/LeaderboardGrid';
+import {useLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
+import {LeaderboardConfigType} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {EditableMarkdown} from './EditableMarkdown';
 import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
-import {LeaderboardGrid} from './LeaderboardGrid';
-import {useLeaderboardData} from './query/hookAdapters';
-import {LeaderboardValueRecord} from './query/leaderboardQuery';
-import {LeaderboardConfigType} from './types/leaderboardConfigType';
-
-const USE_COMPARE_EVALUATIONS_PAGE = false;
 
 type LeaderboardPageProps = {
   entity: string;
@@ -52,9 +47,6 @@ const usePersistedLeaderboardConfig = () => {
 export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
   const {entity, project} = props;
 
-  const {peekingRouter} = useWeaveflowRouteContext();
-  const history = useHistory();
-
   const [showConfig, setShowConfig] = useState(false);
 
   const {
@@ -81,21 +73,6 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     currentConfig.config.dataSelectionSpec
   );
 
-  const handleCellClick = (record: LeaderboardValueRecord) => {
-    const sourceCallId = record.sourceEvaluationCallId;
-    if (sourceCallId) {
-      let to: string;
-      if (USE_COMPARE_EVALUATIONS_PAGE) {
-        to = peekingRouter.compareEvaluationsUri(entity, project, [
-          sourceCallId,
-        ]);
-      } else {
-        to = peekingRouter.callUIUrl(entity, project, '', sourceCallId, null);
-      }
-      history.push(to);
-    }
-  };
-
   const [showingAlert, setShowingAlert] = useState(true);
 
   return (
@@ -151,10 +128,8 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
           <LeaderboardGrid
             entity={entity}
             project={project}
-            config={currentConfig.config.dataSelectionSpec}
             loading={loading}
             data={data}
-            onCellClick={handleCellClick}
           />
         </Box>
       </Box>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
index 45c8657b5f4..31d65bd9eee 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
@@ -4,10 +4,10 @@
 
 import _ from 'lodash';
 
+import {getLeaderboardGroupableData} from '../../../views/Leaderboard/query/leaderboardQuery';
+import {FilterAndGroupSpec} from '../../../views/Leaderboard/types/leaderboardConfigType';
 import {TraceServerClient} from '../../wfReactInterface/traceServerClient';
 import {projectIdFromParts} from '../../wfReactInterface/tsDataModelHooks';
-import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
-import {getLeaderboardGroupableData} from './leaderboardQuery';
 
 export type VersionDetails = {digest: string; index: number};
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/hookAdapters.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/hookAdapters.ts
deleted file mode 100644
index 47caaccbf98..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/hookAdapters.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-import {useEffect, useState} from 'react';
-
-import {useGetTraceServerClientContext} from '../../wfReactInterface/traceServerClientContext';
-import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
-import {getLeaderboardData, GroupedLeaderboardData} from './leaderboardQuery';
-
-type LeaderboardDataState = {
-  loading: boolean;
-  data: GroupedLeaderboardData;
-};
-export const useLeaderboardData = (
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec
-): LeaderboardDataState => {
-  const getTraceServerClient = useGetTraceServerClientContext();
-  const [state, setState] = useState<LeaderboardDataState>({
-    loading: true,
-    data: {modelGroups: {}},
-  });
-  useEffect(() => {
-    let mounted = true;
-    getLeaderboardData(getTraceServerClient(), entity, project, spec).then(
-      data => {
-        if (mounted) {
-          setState({loading: false, data});
-        }
-      }
-    );
-    return () => {
-      mounted = false;
-    };
-  }, [entity, project, getTraceServerClient, spec]);
-  return state;
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
deleted file mode 100644
index e18ddb8ccfc..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/leaderboardQuery.ts
+++ /dev/null
@@ -1,466 +0,0 @@
-import {isWeaveObjectRef} from '@wandb/weave/react';
-import _ from 'lodash';
-
-import {flattenObjectPreservingWeaveTypes} from '../../../../Browse2/browse2Util';
-import {parseRefMaybe} from '../../../../Browse2/SmallRef';
-import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../../common/heuristics';
-import {TraceServerClient} from '../../wfReactInterface/traceServerClient';
-import {TraceObjSchema} from '../../wfReactInterface/traceServerClientTypes';
-import {
-  convertISOToDate,
-  projectIdFromParts,
-} from '../../wfReactInterface/tsDataModelHooks';
-import {
-  objectVersionKeyToRefUri,
-  opVersionKeyToRefUri,
-} from '../../wfReactInterface/utilities';
-import {ALL_VALUE, FilterAndGroupSpec} from '../types/leaderboardConfigType';
-
-export type LeaderboardValueRecord = {
-  datasetName: string;
-  datasetVersion: string;
-  metricType:
-    | 'scorerMetric'
-    | 'modelLatency'
-    | 'modelCost'
-    | 'modelTokens'
-    | 'modelErrors';
-  scorerName: string; // modelMetrics repeat the type here
-  scorerVersion: string; // modelMetrics repeat the type here
-  metricPath: string; // modelMetrics repeat the type here
-  metricValue: number | string | boolean | null;
-  modelName: string;
-  modelVersion: string;
-  modelType: 'object' | 'op';
-  trials: number;
-  createdAt: Date;
-  sourceEvaluationCallId: string;
-  sourceEvaluationObjectRef: string;
-};
-
-export type GroupableLeaderboardValueRecord = {
-  modelGroup: string;
-  datasetGroup: string;
-  scorerGroup: string;
-  metricPathGroup: string;
-  sortKey: number;
-  row: LeaderboardValueRecord;
-};
-
-export type GroupedLeaderboardData = {
-  modelGroups: {
-    [modelGroup: string]: GroupedLeaderboardModelGroup;
-  };
-};
-
-export type GroupedLeaderboardModelGroup<
-  T extends any = LeaderboardValueRecord[]
-> = {
-  datasetGroups: {
-    [datasetGroup: string]: {
-      scorerGroups: {
-        [scorerGroup: string]: {
-          metricPathGroups: {
-            [metricPathGroup: string]: T;
-          };
-        };
-      };
-    };
-  };
-};
-
-export const getEvaluationObjectsForSpec = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec
-) => {
-  const atLeastOneStarName = spec.sourceEvaluations?.some(
-    sourceEvaluation => sourceEvaluation.name === ALL_VALUE
-  );
-  const sourceEvals = atLeastOneStarName ? [] : spec.sourceEvaluations ?? [];
-  const evalNames = sourceEvals.map(sourceEvaluation => sourceEvaluation.name);
-  // const fullyQualifiedEvalRefs = sourceEvals.map(sourceEvaluation => {
-  //   return objectVersionKeyToRefUri({
-  //     scheme: 'weave',
-  //     weaveKind: 'object',
-  //     entity,
-  //     project,
-  //     objectId: sourceEvaluation.name,
-  //     versionHash: sourceEvaluation.version,
-  //     path: '',
-  //   });
-  // });
-  // get all the evaluations
-  const allEvaluationObjectsProm = client.objsQuery({
-    project_id: projectIdFromParts({entity, project}),
-    filter: {
-      base_object_classes: ['Evaluation'],
-      is_op: false,
-      // Sad :( we can't actually filter by version here!
-      object_ids: evalNames,
-    },
-    sort_by: [{field: 'created_at', direction: 'desc'}],
-  });
-
-  const allEvaluationObjectsRes = await allEvaluationObjectsProm;
-
-  // This hack to get around the fact that we can't filter by version in the query
-  if (sourceEvals.length > 0) {
-    allEvaluationObjectsRes.objs = allEvaluationObjectsRes.objs.filter(obj => {
-      return sourceEvals.some(sourceEval => {
-        return (
-          obj.object_id === sourceEval.name &&
-          (obj.digest === sourceEval.version ||
-            sourceEval.version === ALL_VALUE)
-        );
-      });
-    });
-  }
-
-  return allEvaluationObjectsRes;
-};
-
-export const getLeaderboardGroupableData = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec = {}
-): Promise<GroupableLeaderboardValueRecord[]> => {
-  const allEvaluationObjectsRes = await getEvaluationObjectsForSpec(
-    client,
-    entity,
-    project,
-    spec
-  );
-
-  const evaluationObjectDigestMap = new Map<
-    string,
-    {versions: Map<string, TraceObjSchema>; versionOrder: string[]}
-  >();
-  allEvaluationObjectsRes.objs.forEach(obj => {
-    const outerKey = obj.object_id;
-    const innerKey = obj.digest;
-    if (!evaluationObjectDigestMap.has(outerKey)) {
-      evaluationObjectDigestMap.set(outerKey, {
-        versions: new Map(),
-        versionOrder: [],
-      });
-    }
-    evaluationObjectDigestMap.get(outerKey)!.versions.set(innerKey, obj);
-  });
-  evaluationObjectDigestMap.forEach((value, key) => {
-    value.versionOrder = Array.from(value.versions.entries())
-      .sort((a, b) => a[1].version_index - b[1].version_index)
-      .map(entry => entry[0]);
-  });
-
-  const fullyQualifiedEvalRefs = allEvaluationObjectsRes.objs.map(obj => {
-    return objectVersionKeyToRefUri({
-      scheme: 'weave',
-      weaveKind: 'object',
-      entity,
-      project,
-      objectId: obj.object_id,
-      versionHash: obj.digest,
-      path: '',
-    });
-  });
-
-  // SAD: if `input_refs` supported `*` wildcard, then we could do the queries in parallel
-  const allEvaluationCallsProm = client.callsStreamQuery({
-    project_id: projectIdFromParts({entity, project}),
-    sort_by: [{field: 'ended_at', direction: 'desc'}],
-    filter: {
-      op_names: [
-        opVersionKeyToRefUri({
-          entity,
-          project,
-          opId: EVALUATE_OP_NAME_POST_PYDANTIC,
-          versionHash: ALL_VALUE,
-        }),
-      ],
-      input_refs: fullyQualifiedEvalRefs,
-    },
-  });
-
-  const allEvaluationCallsRes = await allEvaluationCallsProm;
-
-  const data: LeaderboardValueRecord[] = [];
-  allEvaluationCallsRes.calls.forEach(call => {
-    const evalObjectRefUri = call.inputs.self;
-    const evalObjectRef = parseRefMaybe(evalObjectRefUri ?? '');
-    const modelObjectOrOpRef = parseRefMaybe(call.inputs.model ?? '');
-
-    if (!evalObjectRef || !modelObjectOrOpRef) {
-      console.warn(
-        'Skipping evaluation call with missing eval object ref',
-        call
-      );
-      return;
-    }
-
-    const evalObjectName = evalObjectRef.artifactName;
-    const evalObjectVersion = evalObjectRef.artifactVersion;
-    const evalObject = evaluationObjectDigestMap
-      .get(evalObjectName)
-      ?.versions.get(evalObjectVersion);
-    if (!evalObject) {
-      console.warn('Skipping evaluation call with missing eval object', call);
-      return;
-    }
-
-    const datasetRef = parseRefMaybe(evalObject.val.dataset ?? '');
-    if (!datasetRef) {
-      console.warn('Skipping evaluation call with missing dataset ref', call);
-      return;
-    }
-    const datasetName = datasetRef.artifactName;
-    const datasetVersion = datasetRef.artifactVersion;
-
-    const modelName = modelObjectOrOpRef.artifactName;
-    const modelVersion = modelObjectOrOpRef.artifactVersion;
-    if (!isWeaveObjectRef(modelObjectOrOpRef)) {
-      console.warn('Skipping evaluation call with invalid model ref', call);
-      return;
-    }
-    const modelType = modelObjectOrOpRef.weaveKind === 'op' ? 'op' : 'object';
-    const trials = evalObject.val.trials ?? call.inputs.trials ?? 1;
-
-    const recordPartial: Omit<
-      LeaderboardValueRecord,
-      | 'metricType'
-      | 'scorerName'
-      | 'scorerVersion'
-      | 'metricPath'
-      | 'metricValue'
-    > = {
-      datasetName,
-      datasetVersion,
-      modelName,
-      modelVersion,
-      modelType,
-      trials,
-      createdAt: convertISOToDate(call.started_at),
-      sourceEvaluationCallId: call.id,
-      sourceEvaluationObjectRef: evalObjectRefUri,
-    };
-
-    const scorerRefUris = (evalObject.val.scorers ?? []) as string[];
-    scorerRefUris.forEach(scorerRefUri => {
-      const scorerRef = parseRefMaybe(scorerRefUri);
-      if (!scorerRef || !isWeaveObjectRef(scorerRef)) {
-        console.warn('Skipping scorer ref', scorerRefUri);
-        return;
-      }
-      const scorerName = scorerRef.artifactName;
-      const scorerVersion = scorerRef.artifactVersion;
-      // const scorerType = scorerRef.weaveKind === 'op' ? 'op' : 'object';
-      const scorePayload = (call.output as any)?.[scorerName];
-      if (typeof scorePayload !== 'object' || scorePayload == null) {
-        console.warn(
-          'Skipping scorer call with invalid score payload',
-          scorerName,
-          scorerVersion,
-          call
-        );
-        return;
-      }
-      const flatScorePayload = flattenObjectPreservingWeaveTypes(scorePayload);
-      Object.entries(flatScorePayload).forEach(([metricPath, metricValue]) => {
-        const scoreRecord: LeaderboardValueRecord = {
-          ...recordPartial,
-          metricType: 'scorerMetric',
-          scorerName,
-          scorerVersion,
-          metricPath,
-          metricValue,
-        };
-        data.push(scoreRecord);
-      });
-    });
-
-    const modelLatency = (call.output as any)?.model_latency?.mean;
-    if (modelLatency == null) {
-      console.warn('Skipping model latency', call);
-    } else {
-      const modelLatencyRecord: LeaderboardValueRecord = {
-        ...recordPartial,
-        metricType: 'modelLatency',
-        scorerName: 'modelLatency',
-        scorerVersion: 'modelLatency',
-        metricPath: 'model_latency.mean',
-        metricValue: modelLatency,
-      };
-      data.push(modelLatencyRecord);
-    }
-
-    // TODO: add modelCost, modelTokens, modelErrors
-  });
-
-  const filterableGroupableData = data.map(row => {
-    const groupableRow: GroupableLeaderboardValueRecord = {
-      datasetGroup: `${row.datasetName}:${row.datasetVersion}`,
-      scorerGroup:
-        row.metricType === 'scorerMetric'
-          ? `${row.scorerName}:${row.scorerVersion}`
-          : row.scorerName,
-      modelGroup: `${row.modelName}:${row.modelVersion}`,
-      metricPathGroup: row.metricPath,
-      sortKey: -row.createdAt.getTime(),
-      row,
-    };
-
-    if (spec.models) {
-      let modelSpec = spec.models.find(
-        model =>
-          model.name === row.modelName && model.version === row.modelVersion
-      );
-      modelSpec =
-        modelSpec ||
-        spec.models.find(
-          model =>
-            model.name === row.modelName &&
-            (model.version === ALL_VALUE || model.version === row.modelVersion)
-        );
-      modelSpec =
-        modelSpec ||
-        spec.models.find(
-          model =>
-            (model.name === ALL_VALUE || model.name === row.modelName) &&
-            (model.version === ALL_VALUE || model.version === row.modelVersion)
-        );
-      if (!modelSpec) {
-        return {include: false, groupableRow};
-      }
-      if (modelSpec.groupAllVersions) {
-        groupableRow.modelGroup = `${row.modelName}`;
-      }
-    }
-
-    if (!spec.datasets) {
-      return {include: true, groupableRow};
-    }
-    if (spec.datasets.length === 0) {
-      return {include: true, groupableRow};
-    }
-    // if (spec.datasets.some(dataset => dataset.name === ALL_VALUE)) {
-    //   return {include: true, groupableRow};
-    // }
-
-    let datasetSpec = spec.datasets.find(
-      dataset =>
-        dataset.name === row.datasetName &&
-        dataset.version === row.datasetVersion
-    );
-    datasetSpec =
-      datasetSpec ||
-      spec.datasets.find(
-        dataset =>
-          dataset.name === row.datasetName &&
-          (dataset.version === ALL_VALUE ||
-            dataset.version === row.datasetVersion)
-      );
-    datasetSpec =
-      datasetSpec ||
-      spec.datasets.find(
-        dataset =>
-          (dataset.name === ALL_VALUE || dataset.name === row.datasetName) &&
-          (dataset.version === ALL_VALUE ||
-            dataset.version === row.datasetVersion)
-      );
-    if (!datasetSpec) {
-      return {include: false, groupableRow};
-    }
-    if (datasetSpec.groupAllVersions) {
-      groupableRow.datasetGroup = `${row.datasetName}`;
-    }
-    if (datasetSpec.scorers && datasetSpec.scorers.length > 0) {
-      let scorerSpec = datasetSpec.scorers.find(
-        scorer =>
-          scorer.name === row.scorerName && scorer.version === row.scorerVersion
-      );
-      scorerSpec =
-        scorerSpec ||
-        datasetSpec.scorers.find(
-          scorer =>
-            scorer.name === row.scorerName &&
-            (scorer.version === ALL_VALUE ||
-              scorer.version === row.scorerVersion)
-        );
-      scorerSpec =
-        scorerSpec ||
-        datasetSpec.scorers.find(
-          scorer =>
-            (scorer.name === ALL_VALUE || scorer.name === row.scorerName) &&
-            (scorer.version === ALL_VALUE ||
-              scorer.version === row.scorerVersion)
-        );
-
-      if (!scorerSpec) {
-        return {include: false, groupableRow};
-      }
-      if (scorerSpec.groupAllVersions) {
-        groupableRow.scorerGroup = `${row.scorerName}`;
-      }
-      if (scorerSpec.metrics && scorerSpec.metrics.length > 0) {
-        const metricSpec = scorerSpec.metrics.find(
-          metric => metric.path === ALL_VALUE || metric.path === row.metricPath
-        );
-        if (!metricSpec) {
-          return {include: false, groupableRow};
-        }
-      }
-    }
-
-    return {include: true, groupableRow};
-  });
-
-  return filterableGroupableData
-    .filter(entry => entry.include)
-    .map(entry => entry.groupableRow);
-};
-export const getLeaderboardData = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec = {}
-): Promise<GroupedLeaderboardData> => {
-  const groupableData = await getLeaderboardGroupableData(
-    client,
-    entity,
-    project,
-    spec
-  );
-  const finalData: GroupedLeaderboardData = {
-    modelGroups: _.mapValues(
-      _.groupBy(groupableData, 'modelGroup'),
-      modelGroup => {
-        return {
-          datasetGroups: _.mapValues(
-            _.groupBy(modelGroup, 'datasetGroup'),
-            datasetGroup => {
-              return {
-                scorerGroups: _.mapValues(
-                  _.groupBy(datasetGroup, 'scorerGroup'),
-                  scorerGroup => {
-                    return {
-                      metricPathGroups: _.mapValues(
-                        _.groupBy(scorerGroup, 'metricPathGroup'),
-                        metricPathGroup => {
-                          return metricPathGroup.map(row => row.row);
-                        }
-                      ),
-                    };
-                  }
-                ),
-              };
-            }
-          ),
-        };
-      }
-    ),
-  };
-
-  return finalData;
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts
deleted file mode 100644
index bb4587966c9..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/types/leaderboardConfigType.ts
+++ /dev/null
@@ -1,45 +0,0 @@
-export const ALL_VALUE = '*';
-
-export type LeaderboardConfigType = {
-  version: number;
-  config: {
-    description: string;
-    dataSelectionSpec: FilterAndGroupSpec;
-  };
-};
-
-export type FilterAndGroupSpec = {
-  sourceEvaluations?: FilterAndGroupSourceEvaluationSpec[]; // null is all
-  datasets?: FilterAndGroupDatasetSpec[]; // null is all
-  models?: FilterAndGroupModelSpec[]; // null is all
-};
-
-export type FilterAndGroupSourceEvaluationSpec = {
-  name: string;
-  version: string; // "*" means all
-};
-
-export type FilterAndGroupDatasetSpec = {
-  name: string; // "*" means all
-  version: string; // "*" means all
-  groupAllVersions?: boolean;
-  scorers?: FilterAndGroupDatasetScorerSpec[]; // null is all
-};
-
-export type FilterAndGroupDatasetScorerSpec = {
-  name: string; // "*" means all
-  version: string; // "*" means all
-  groupAllVersions?: boolean;
-  metrics?: FilterAndGroupDatasetScorerMetricSpec[]; // null is all
-};
-
-export type FilterAndGroupDatasetScorerMetricSpec = {
-  path: string; // "*" means all
-  shouldMinimize?: boolean;
-};
-
-export type FilterAndGroupModelSpec = {
-  name: string; // "*" means all
-  version: string; // "*" means all
-  groupAllVersions?: boolean;
-};

From d1fcc9bf6dd85c0c0c5ab7621f00fbf3860248b9 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 18:17:14 -0700
Subject: [PATCH 067/169] pulled initial files

---
 tests/trace/test_leaderboard.py | 121 ++++++++++++++++++++++++++++++++
 weave/flow/leaderboard.py       |  80 +++++++++++++++++++++
 2 files changed, 201 insertions(+)
 create mode 100644 tests/trace/test_leaderboard.py
 create mode 100644 weave/flow/leaderboard.py

diff --git a/tests/trace/test_leaderboard.py b/tests/trace/test_leaderboard.py
new file mode 100644
index 00000000000..ed8b950ad2b
--- /dev/null
+++ b/tests/trace/test_leaderboard.py
@@ -0,0 +1,121 @@
+import pytest
+
+import weave
+from weave.flow import leaderboard
+from weave.trace.weave_client import get_ref
+
+
+def test_emptyleaderboard(client):
+    spec = leaderboard.LeaderboardSpec(
+        name="Test Leaderboard",
+        description="This is a test leaderboard",
+        columns=[
+            leaderboard.LeaderboardColumn(
+                evaluation_object_ref="test_evaluation_object_ref",
+                scorer_name="test_scorer_name",
+                summary_metric_path="test_summary_metric_path",
+            )
+        ]
+    )
+
+    results = leaderboard.get_leaderboard_results(spec, client)
+    assert len(results) == 0
+
+
+async def do_evaluations():
+    @weave.op
+    def my_scorer(target, model_output):
+        return target == model_output
+    
+    evaluation_obj_1 = weave.Evaluation(
+        name="test_evaluation_name",
+        dataset=[{'input': 1, 'target': 1}],
+        scorers=[my_scorer]
+    )
+
+    @weave.op
+    def simple_model(input):
+        return input
+    
+    await evaluation_obj_1.evaluate(simple_model)
+
+    evaluation_obj_2 = weave.Evaluation(
+        name="test_evaluation_name",
+        dataset=[{'input': 1, 'target': 1}, {'input': 2, 'target': 2}],
+        scorers=[my_scorer]
+    )
+
+    @weave.op
+    def static_model(input):
+        return 1
+    
+    @weave.op
+    def bad_model(input):
+        return input + 1
+    
+    await evaluation_obj_2.evaluate(simple_model)
+    await evaluation_obj_2.evaluate(static_model)
+    await evaluation_obj_2.evaluate(bad_model)
+
+    return evaluation_obj_1, evaluation_obj_2, simple_model, static_model, bad_model
+    
+@pytest.mark.asyncio
+async def test_leaderboard_with_results(client):
+    evaluation_obj_1, evaluation_obj_2, simple_model, static_model, bad_model = await do_evaluations()
+
+    spec = leaderboard.LeaderboardSpec(
+        name="Test Leaderboard",
+        description="This is a test leaderboard",
+        columns=[
+            leaderboard.LeaderboardColumn(
+                evaluation_object_ref=get_ref(evaluation_obj_1).uri(),
+                scorer_name="my_scorer",
+                summary_metric_path_parts=["true_fraction"],
+            )
+        ]
+    )
+
+    results = leaderboard.get_leaderboard_results(spec, client)
+    assert len(results) == 1
+    assert results[0].model_ref == get_ref(simple_model).uri()
+    assert results[0].column_scores[0].scores[0].value == 1.0
+
+    spec = leaderboard.LeaderboardSpec(
+        name="Test Leaderboard",
+        description="This is a test leaderboard",
+        columns=[
+            leaderboard.LeaderboardColumn(
+                evaluation_object_ref=get_ref(evaluation_obj_2).uri(),
+                scorer_name="my_scorer",
+                summary_metric_path_parts=["true_count"],
+            ),
+            leaderboard.LeaderboardColumn(
+                evaluation_object_ref=get_ref(evaluation_obj_2).uri(),
+                scorer_name="my_scorer",
+                summary_metric_path_parts=["true_fraction"],
+            ),
+            leaderboard.LeaderboardColumn(
+                evaluation_object_ref=get_ref(evaluation_obj_1).uri(),
+                scorer_name="my_scorer",
+                summary_metric_path_parts=["true_fraction"],
+            )
+        ]
+    )
+
+    results = leaderboard.get_leaderboard_results(spec, client)
+    assert len(results) == 3
+    assert results[0].model_ref == get_ref(simple_model).uri()
+    assert len(results[0].column_scores) == 3
+    assert results[0].column_scores[0].scores[0].value == 2.0
+    assert results[0].column_scores[1].scores[0].value == 1.0
+    assert results[0].column_scores[1].scores[0].value == 1.0
+    assert results[1].model_ref == get_ref(static_model).uri()
+    assert len(results[1].column_scores) == 3
+    assert results[1].column_scores[0].scores[0].value == 1.0
+    assert results[1].column_scores[1].scores[0].value == 0.5
+    assert len(results[1].column_scores[2].scores) == 0
+    assert results[2].model_ref == get_ref(bad_model).uri()
+    assert len(results[1].column_scores) == 3
+    assert results[2].column_scores[0].scores[0].value == 0
+    assert results[2].column_scores[1].scores[0].value == 0
+    assert len(results[2].column_scores[2].scores) == 0
diff --git a/weave/flow/leaderboard.py b/weave/flow/leaderboard.py
new file mode 100644
index 00000000000..dd6061a6981
--- /dev/null
+++ b/weave/flow/leaderboard.py
@@ -0,0 +1,80 @@
+from dataclasses import field
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+import weave
+from weave.trace.refs import OpRef
+from weave.trace.weave_client import Call, WeaveClient, get_ref
+from weave.trace_server.trace_server_interface import CallsFilter
+
+
+class LeaderboardColumn(BaseModel):
+    evaluation_object_ref: str
+    scorer_name: str
+    summary_metric_path_parts: list[str] = field(default_factory=list)
+    should_minimize: Optional[bool] = None
+
+class LeaderboardSpec(weave.Object):
+    columns: list[LeaderboardColumn]
+
+class LeaderboardModelEvaluationResult(BaseModel):
+    evaluate_call_ref: str
+    value: Any
+
+class ModelScoresForColumn(BaseModel):
+    scores: list[LeaderboardModelEvaluationResult]
+
+class LeaderboardModelResult(BaseModel):
+    model_ref: str
+    column_scores: list[ModelScoresForColumn]
+
+
+def get_leaderboard_results(spec: LeaderboardSpec, client: WeaveClient) -> list[LeaderboardModelResult]:
+    entity, project = client._project_id().split("/")
+    calls = client.get_calls(filter=CallsFilter(
+        op_names=[OpRef(
+            entity=entity,
+            project=project,
+            name="Evaluation.evaluate",
+            _digest="*"
+        ).uri()],
+        input_refs=[c.evaluation_object_ref for c  in spec.columns]
+    ))
+    def get_scores(call: Call) -> list[float]:
+        res = []
+        for c in spec.columns:
+            val = call.output.get(c.scorer_name)
+            for part in c.summary_metric_path_parts:
+                if isinstance(val, dict):
+                    val = val.get(part)
+                elif isinstance(val, list):
+                    val = val[int(part)]
+                else:
+                    break
+            res.append(val)
+        return res
+    
+    res_map: dict[str, LeaderboardModelResult] = {}
+    for call in calls:
+        # Frustrating that we have to get the ref like this (such a waste of network calls)
+        model_ref = get_ref(call.inputs["model"]).uri()
+        if model_ref not in res_map:
+            res_map[model_ref] = LeaderboardModelResult(model_ref=model_ref, column_scores=[ModelScoresForColumn(scores=[]) for _ in spec.columns])
+        for col_idx, c in enumerate(spec.columns):
+            eval_obj_ref = get_ref(call.inputs["self"]).uri()
+            if c.evaluation_object_ref != eval_obj_ref:
+                continue
+            val = call.output.get(c.scorer_name)
+            for part in c.summary_metric_path_parts:
+                if isinstance(val, dict):
+                    val = val.get(part)
+                elif isinstance(val, list):
+                    val = val[int(part)]
+                else:
+                    break
+            res_map[model_ref].column_scores[col_idx].scores.append(LeaderboardModelEvaluationResult(
+                evaluate_call_ref=get_ref(call).uri(),
+                value=val
+            ))
+    return list(res_map.values())

From fe63f28cbfa3f5db619e55a2f2e1b4664b1d893d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 18:21:45 -0700
Subject: [PATCH 068/169] MAYBE CHECKPOINT

---
 .../src/components/FancyPage/useProjectSidebar.ts    |  9 +++------
 .../components/PagePanelComponents/Home/Browse3.tsx  | 12 ++++++++----
 .../PagePanelComponents/Home/Browse3/context.tsx     | 10 ++++++++++
 .../pages/LeaderboardPage/LeaderboardListingPage.tsx |  8 ++++++++
 .../pages/LeaderboardPage/LeaderboardPage.tsx        |  2 ++
 5 files changed, 31 insertions(+), 10 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index a38fec4d332..12ee1706bb3 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -10,8 +10,7 @@ export const useProjectSidebar = (
   hasModelsData: boolean,
   hasWeaveData: boolean,
   hasTraceBackend: boolean = true,
-  hasModelsAccess: boolean = true,
-  viewerIsAdmin: boolean = false
+  hasModelsAccess: boolean = true
 ): FancyPageSidebarItem[] => {
   // Should show models sidebar items if we have models data or if we don't have a trace backend
   let showModelsSidebarItems = hasModelsData || !hasTraceBackend;
@@ -135,10 +134,8 @@ export const useProjectSidebar = (
           {
             type: 'button' as const,
             name: 'Leaderboard',
-            slug: 'weave/leaderboard',
-            // TODO: remove this `true`
-            isShown:
-              (viewerIsAdmin || true) && (showWeaveSidebarItems || isShowAll),
+            slug: 'weave/leaderboards',
+            isShown: (showWeaveSidebarItems || isShowAll),
             iconName: IconNames.BenchmarkSquare,
           },
           {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
index 91523af4d10..40728fb5776 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
@@ -102,6 +102,7 @@ import {
 } from './Browse3/pages/wfReactInterface/context';
 import {useHasTraceServerClientContext} from './Browse3/pages/wfReactInterface/traceServerClientContext';
 import {useDrawerResize} from './useDrawerResize';
+import { LeaderboardListingPage } from './Browse3/pages/LeaderboardPage/LeaderboardListingPage';
 
 LicenseInfo.setLicenseKey(
   'c3f549c76a1e054e5e314b2f1ecfca1cTz05OTY3MixFPTE3NjAxMTM3NDAwMDAsUz1wcm8sTE09c3Vic2NyaXB0aW9uLFBWPWluaXRpYWwsS1Y9Mg=='
@@ -152,7 +153,7 @@ const tabOptions = [
   'op-versions',
   'calls',
   'evaluations',
-  'leaderboard',
+  'leaderboards',
   'boards',
   'tables',
 ];
@@ -494,7 +495,7 @@ const Browse3ProjectRoot: FC<{
         <Route path={`${projectRoot}/:tab(compare-evaluations)`}>
           <CompareEvaluationsBinding />
         </Route>
-        <Route path={`${projectRoot}/:tab(leaderboard)`}>
+        <Route path={`${projectRoot}/:tab(leaderboards)`}>
           <LeaderboardPageBinding />
         </Route>
         {/* BOARDS */}
@@ -979,8 +980,11 @@ const CompareEvaluationsBinding = () => {
 };
 
 const LeaderboardPageBinding = () => {
-  const {entity, project} = useParamsDecoded<Browse3TabParams>();
-  return <LeaderboardPage entity={entity} project={project} />;
+  const {entity, project, itemName: leaderboardName} = useParamsDecoded<Browse3TabItemParams>();
+  if (!leaderboardName) {
+    return <LeaderboardListingPage entity={entity} project={project} />;
+  }
+  return <LeaderboardPage entity={entity} project={project} leaderboardName={leaderboardName/>;
 };
 
 const OpsPageBinding = () => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx
index 640e68441b6..f4e86ab15e7 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx
@@ -193,6 +193,9 @@ export const browse2Context = {
   ) => {
     throw new Error('Not implemented');
   },
+  leaderboardsUIUrl: (entityName: string, projectName: string, leaderboardName?: string) => {
+    throw new Error('Not implemented');
+  }
 };
 
 export const browse3ContextGen = (
@@ -422,6 +425,9 @@ export const browse3ContextGen = (
         JSON.stringify(evaluationCallIds)
       )}${metricsPart}`;
     },
+    leaderboardsUIUrl: (entityName: string, projectName: string, leaderboardName?: string) => {
+      return `${projectRoot(entityName, projectName)}/leaderboards${leaderboardName ? `/${leaderboardName}` : ''}`;
+    }
   };
   return browse3Context;
 };
@@ -506,6 +512,7 @@ type RouteType = {
     evaluationCallIds: string[],
     metrics: Record<string, boolean> | null
   ) => string;
+  leaderboardsUIUrl: (entityName: string, projectName: string, leaderboardName?: string) => string;
 };
 
 const useSetSearchParam = () => {
@@ -619,6 +626,9 @@ const useMakePeekingRouter = (): RouteType => {
         baseContext.compareEvaluationsUri(...args)
       );
     },
+    leaderboardsUIUrl: (...args: Parameters<typeof baseContext.leaderboardsUIUrl>) => {
+      return setSearchParam(PEEK_PARAM, baseContext.leaderboardsUIUrl(...args));
+    }
   };
 };
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
new file mode 100644
index 00000000000..75bd023a52e
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -0,0 +1,8 @@
+import React from 'react';
+
+export const LeaderboardListingPage: React.FC<{
+  entity: string;
+  project: string;
+}> = ({entity, project}) => {
+  return <div>LeaderboardListingPage: {entity} {project}</div>;
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 8e365e987bd..3dee462d64d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -11,9 +11,11 @@ import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
 type LeaderboardPageProps = {
   entity: string;
   project: string;
+  leaderboardName: string;
 };
 
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
+  return <>{props.leaderboardName}</>;
   return (
     <LeaderboardPageContent entity={props.entity} project={props.project} />
   );

From b6fda47f7ca3a7846754f52b8c079ac73e4ab403 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 18:35:59 -0700
Subject: [PATCH 069/169] MAYBE CHECKPOINT 2

---
 .../components/FancyPage/useProjectSidebar.ts |  3 +-
 .../PagePanelComponents/Home/Browse3.tsx      | 20 +++++++++---
 .../Home/Browse3/context.tsx                  | 32 ++++++++++++++-----
 .../LeaderboardListingPage.tsx                |  6 +++-
 4 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index 12ee1706bb3..1a020eeb772 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -135,7 +135,7 @@ export const useProjectSidebar = (
             type: 'button' as const,
             name: 'Leaderboard',
             slug: 'weave/leaderboards',
-            isShown: (showWeaveSidebarItems || isShowAll),
+            isShown: showWeaveSidebarItems || isShowAll,
             iconName: IconNames.BenchmarkSquare,
           },
           {
@@ -225,7 +225,6 @@ export const useProjectSidebar = (
     isWeaveOnly,
     viewingRestricted,
     isModelsOnly,
-    viewerIsAdmin,
     showWeaveSidebarItems,
   ]);
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
index 40728fb5776..409972d0e32 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
@@ -82,6 +82,7 @@ import {Empty} from './Browse3/pages/common/Empty';
 import {EMPTY_NO_TRACE_SERVER} from './Browse3/pages/common/EmptyContent';
 import {SimplePageLayoutContext} from './Browse3/pages/common/SimplePageLayout';
 import {CompareEvaluationsPage} from './Browse3/pages/CompareEvaluationsPage/CompareEvaluationsPage';
+import {LeaderboardListingPage} from './Browse3/pages/LeaderboardPage/LeaderboardListingPage';
 import {LeaderboardPage} from './Browse3/pages/LeaderboardPage/LeaderboardPage';
 import {ObjectPage} from './Browse3/pages/ObjectPage';
 import {ObjectVersionPage} from './Browse3/pages/ObjectVersionPage';
@@ -102,7 +103,6 @@ import {
 } from './Browse3/pages/wfReactInterface/context';
 import {useHasTraceServerClientContext} from './Browse3/pages/wfReactInterface/traceServerClientContext';
 import {useDrawerResize} from './useDrawerResize';
-import { LeaderboardListingPage } from './Browse3/pages/LeaderboardPage/LeaderboardListingPage';
 
 LicenseInfo.setLicenseKey(
   'c3f549c76a1e054e5e314b2f1ecfca1cTz05OTY3MixFPTE3NjAxMTM3NDAwMDAsUz1wcm8sTE09c3Vic2NyaXB0aW9uLFBWPWluaXRpYWwsS1Y9Mg=='
@@ -495,7 +495,11 @@ const Browse3ProjectRoot: FC<{
         <Route path={`${projectRoot}/:tab(compare-evaluations)`}>
           <CompareEvaluationsBinding />
         </Route>
-        <Route path={`${projectRoot}/:tab(leaderboards)`}>
+        <Route
+          path={[
+            `${projectRoot}/leaderboards/:itemName`,
+            `${projectRoot}/leaderboards`,
+          ]}>
           <LeaderboardPageBinding />
         </Route>
         {/* BOARDS */}
@@ -980,11 +984,19 @@ const CompareEvaluationsBinding = () => {
 };
 
 const LeaderboardPageBinding = () => {
-  const {entity, project, itemName: leaderboardName} = useParamsDecoded<Browse3TabItemParams>();
+  const params = useParamsDecoded<Browse3TabItemParams>();
+  const {entity, project, itemName: leaderboardName} = params;
+  console.log({params, leaderboardName});
   if (!leaderboardName) {
     return <LeaderboardListingPage entity={entity} project={project} />;
   }
-  return <LeaderboardPage entity={entity} project={project} leaderboardName={leaderboardName/>;
+  return (
+    <LeaderboardPage
+      entity={entity}
+      project={project}
+      leaderboardName={leaderboardName}
+    />
+  );
 };
 
 const OpsPageBinding = () => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx
index f4e86ab15e7..44f207bbb5e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx
@@ -193,9 +193,13 @@ export const browse2Context = {
   ) => {
     throw new Error('Not implemented');
   },
-  leaderboardsUIUrl: (entityName: string, projectName: string, leaderboardName?: string) => {
+  leaderboardsUIUrl: (
+    entityName: string,
+    projectName: string,
+    leaderboardName?: string
+  ) => {
     throw new Error('Not implemented');
-  }
+  },
 };
 
 export const browse3ContextGen = (
@@ -425,9 +429,15 @@ export const browse3ContextGen = (
         JSON.stringify(evaluationCallIds)
       )}${metricsPart}`;
     },
-    leaderboardsUIUrl: (entityName: string, projectName: string, leaderboardName?: string) => {
-      return `${projectRoot(entityName, projectName)}/leaderboards${leaderboardName ? `/${leaderboardName}` : ''}`;
-    }
+    leaderboardsUIUrl: (
+      entityName: string,
+      projectName: string,
+      leaderboardName?: string
+    ) => {
+      return `${projectRoot(entityName, projectName)}/leaderboards${
+        leaderboardName ? `/${leaderboardName}` : ''
+      }`;
+    },
   };
   return browse3Context;
 };
@@ -512,7 +522,11 @@ type RouteType = {
     evaluationCallIds: string[],
     metrics: Record<string, boolean> | null
   ) => string;
-  leaderboardsUIUrl: (entityName: string, projectName: string, leaderboardName?: string) => string;
+  leaderboardsUIUrl: (
+    entityName: string,
+    projectName: string,
+    leaderboardName?: string
+  ) => string;
 };
 
 const useSetSearchParam = () => {
@@ -626,9 +640,11 @@ const useMakePeekingRouter = (): RouteType => {
         baseContext.compareEvaluationsUri(...args)
       );
     },
-    leaderboardsUIUrl: (...args: Parameters<typeof baseContext.leaderboardsUIUrl>) => {
+    leaderboardsUIUrl: (
+      ...args: Parameters<typeof baseContext.leaderboardsUIUrl>
+    ) => {
       return setSearchParam(PEEK_PARAM, baseContext.leaderboardsUIUrl(...args));
-    }
+    },
   };
 };
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index 75bd023a52e..618d5552d8c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -4,5 +4,9 @@ export const LeaderboardListingPage: React.FC<{
   entity: string;
   project: string;
 }> = ({entity, project}) => {
-  return <div>LeaderboardListingPage: {entity} {project}</div>;
+  return (
+    <div>
+      LeaderboardListingPage: {entity} {project}
+    </div>
+  );
 };

From c75be77a7bc14fb6d408868c8b21f6ff07d2d121 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 18:38:20 -0700
Subject: [PATCH 070/169] MAYBE CHECKPOINT 2

---
 weave-js/src/components/PagePanelComponents/Home/Browse3.tsx   | 1 -
 .../Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx     | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
index 409972d0e32..49adab6af2e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
@@ -986,7 +986,6 @@ const CompareEvaluationsBinding = () => {
 const LeaderboardPageBinding = () => {
   const params = useParamsDecoded<Browse3TabItemParams>();
   const {entity, project, itemName: leaderboardName} = params;
-  console.log({params, leaderboardName});
   if (!leaderboardName) {
     return <LeaderboardListingPage entity={entity} project={project} />;
   }
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 3dee462d64d..4858ee9ea03 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -15,9 +15,8 @@ type LeaderboardPageProps = {
 };
 
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
-  return <>{props.leaderboardName}</>;
   return (
-    <LeaderboardPageContent entity={props.entity} project={props.project} />
+    <LeaderboardPageContent {...props} />
   );
 };
 

From 407ebdf4b8ae738725c4c730bb6c289292f9b792 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 18:46:21 -0700
Subject: [PATCH 071/169] Cursor 1, human 0

---
 .../components/FancyPage/useProjectSidebar.ts |  24 ++-
 .../LeaderboardListingPage.tsx                | 145 +++++++++++++++++-
 2 files changed, 159 insertions(+), 10 deletions(-)

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index 1a020eeb772..2f8f0d210cc 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -131,13 +131,6 @@ export const useProjectSidebar = (
             label: 'Weave',
             isShown: isShowAll,
           },
-          {
-            type: 'button' as const,
-            name: 'Leaderboard',
-            slug: 'weave/leaderboards',
-            isShown: showWeaveSidebarItems || isShowAll,
-            iconName: IconNames.BenchmarkSquare,
-          },
           {
             type: 'button' as const,
             name: 'Traces',
@@ -145,6 +138,11 @@ export const useProjectSidebar = (
             isShown: showWeaveSidebarItems || isShowAll,
             iconName: IconNames.LayoutTabs,
           },
+          {
+            type: 'divider' as const,
+            key: 'dividerWithinWeave',
+            isShown: isWeaveOnly,
+          },
           {
             type: 'button' as const,
             name: 'Evals',
@@ -152,6 +150,18 @@ export const useProjectSidebar = (
             isShown: showWeaveSidebarItems || isShowAll,
             iconName: IconNames.BaselineAlt,
           },
+          {
+            type: 'button' as const,
+            name: 'Leads',
+            slug: 'weave/leaderboards',
+            isShown: showWeaveSidebarItems || isShowAll,
+            iconName: IconNames.BenchmarkSquare,
+          },
+          {
+            type: 'divider' as const,
+            key: 'dividerWithinWeave',
+            isShown: isWeaveOnly,
+          },
           {
             type: 'button' as const,
             name: 'Models',
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index 618d5552d8c..af2105b2a61 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -1,12 +1,151 @@
 import React from 'react';
+import styled from 'styled-components';
+
+import { Empty } from '../common/Empty';
+import { EMPTY_PROPS_EVALUATIONS } from '../common/EmptyContent';
 
 export const LeaderboardListingPage: React.FC<{
   entity: string;
   project: string;
 }> = ({entity, project}) => {
+  const annotationQueues = [
+    { name: 'Weather_genApp_experts', description: 'This is the description', traces: '16/45', needsReview: true },
+    { name: 'TidePressure_modeling_team', description: 'This is the description', traces: '30/62', needsReview: true },
+    { name: 'ClimateModel_genApp_experts', description: 'This is the description', traces: '15', needsReview: false },
+    { name: 'OceanCurrent_analysis_team', description: 'This is the description', traces: '70', needsReview: false },
+    { name: 'Windflow_projection_experts', description: 'This is the description', traces: '82', needsReview: false },
+    { name: 'AtmoMetrics_analysis_team', description: 'This is the description', traces: '40', needsReview: false },
+    { name: 'HydroFluor_research_specialists', description: 'This is the description', traces: '202', needsReview: false },
+  ];
+
   return (
-    <div>
-      LeaderboardListingPage: {entity} {project}
-    </div>
+    <Container>
+      <Header>
+        <Title>Annotation queues</Title>
+        <CreateQueueButton>+ Create queue</CreateQueueButton>
+      </Header>
+      <WelcomeMessage>
+        Welcome to annotation queues
+        <DismissButton>Dismiss</DismissButton>
+      </WelcomeMessage>
+      <Section>
+        <SectionTitle>Needs review (2)</SectionTitle>
+        <QueueGrid>
+          {annotationQueues.filter(queue => queue.needsReview).map(queue => (
+            <QueueCard key={queue.name}>
+              <QueueName>{queue.name}</QueueName>
+              <ReviewButton>Review</ReviewButton>
+              <QueueDescription>{queue.description}</QueueDescription>
+              <TracesCount>{queue.traces} Traces</TracesCount>
+            </QueueCard>
+          ))}
+        </QueueGrid>
+      </Section>
+      <Section>
+        <SectionTitle>Up to date (5)</SectionTitle>
+        <QueueGrid>
+          {annotationQueues.filter(queue => !queue.needsReview).map(queue => (
+            <QueueCard key={queue.name}>
+              <QueueName>{queue.name}</QueueName>
+              <QueueDescription>{queue.description}</QueueDescription>
+              <TracesCount>{queue.traces} Traces</TracesCount>
+            </QueueCard>
+          ))}
+        </QueueGrid>
+      </Section>
+    </Container>
   );
 };
+
+const Container = styled.div`
+  padding: 20px;
+`;
+
+const Header = styled.div`
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  margin-bottom: 20px;
+`;
+
+const Title = styled.h1`
+  font-size: 24px;
+  margin: 0;
+`;
+
+const CreateQueueButton = styled.button`
+  background-color: #00A4B8;
+  color: white;
+  border: none;
+  padding: 10px 15px;
+  border-radius: 4px;
+  cursor: pointer;
+`;
+
+const WelcomeMessage = styled.div`
+  background-color: #f0f0f0;
+  padding: 10px;
+  border-radius: 4px;
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  margin-bottom: 20px;
+`;
+
+const DismissButton = styled.button`
+  background: none;
+  border: none;
+  color: #00A4B8;
+  cursor: pointer;
+`;
+
+const Section = styled.div`
+  margin-bottom: 20px;
+`;
+
+const SectionTitle = styled.h2`
+  font-size: 18px;
+  margin-bottom: 10px;
+`;
+
+const QueueGrid = styled.div`
+  display: grid;
+  grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+  gap: 20px;
+`;
+
+const QueueCard = styled.div`
+  border: 1px solid #e0e0e0;
+  border-radius: 4px;
+  padding: 15px;
+`;
+
+const QueueName = styled.h3`
+  font-size: 16px;
+  margin: 0 0 10px 0;
+`;
+
+const ReviewButton = styled.button`
+  background-color: #FFA500;
+  color: white;
+  border: none;
+  padding: 5px 10px;
+  border-radius: 4px;
+  cursor: pointer;
+  float: right;
+`;
+
+const QueueDescription = styled.p`
+  font-size: 14px;
+  color: #666;
+  margin: 0 0 10px 0;
+`;
+
+const TracesCount = styled.div`
+  font-size: 24px;
+  font-weight: bold;
+`;
+
+const NoEvaluations = () => {
+  return <Empty {...EMPTY_PROPS_EVALUATIONS} />;
+};

From 20947ed7de0c13672708b1df9f6c057bd9450b63 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 18:53:08 -0700
Subject: [PATCH 072/169] Cursor 1, human 1 + lint

---
 .../LeaderboardListingPage.tsx                | 138 ++++++++++++++----
 .../pages/LeaderboardPage/LeaderboardPage.tsx |   4 +-
 2 files changed, 110 insertions(+), 32 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index af2105b2a61..fd22d5902c7 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -1,54 +1,134 @@
 import React from 'react';
 import styled from 'styled-components';
 
-import { Empty } from '../common/Empty';
-import { EMPTY_PROPS_EVALUATIONS } from '../common/EmptyContent';
+import {Empty} from '../common/Empty';
+import {EMPTY_PROPS_EVALUATIONS} from '../common/EmptyContent';
+import {SimplePageLayout} from '../common/SimplePageLayout';
 
 export const LeaderboardListingPage: React.FC<{
   entity: string;
   project: string;
-}> = ({entity, project}) => {
-  const annotationQueues = [
-    { name: 'Weather_genApp_experts', description: 'This is the description', traces: '16/45', needsReview: true },
-    { name: 'TidePressure_modeling_team', description: 'This is the description', traces: '30/62', needsReview: true },
-    { name: 'ClimateModel_genApp_experts', description: 'This is the description', traces: '15', needsReview: false },
-    { name: 'OceanCurrent_analysis_team', description: 'This is the description', traces: '70', needsReview: false },
-    { name: 'Windflow_projection_experts', description: 'This is the description', traces: '82', needsReview: false },
-    { name: 'AtmoMetrics_analysis_team', description: 'This is the description', traces: '40', needsReview: false },
-    { name: 'HydroFluor_research_specialists', description: 'This is the description', traces: '202', needsReview: false },
+}> = props => {
+  return (
+    <SimplePageLayout
+      title={`Leaderboards`}
+      hideTabsIfSingle
+      tabs={[
+        {
+          label: 'All',
+          content: <LeaderboardListingPageInner {...props} />,
+        },
+      ]}
+    />
+  );
+};
+
+export const LeaderboardListingPageInner: React.FC<{
+  entity: string;
+  project: string;
+}> = props => {
+  const customLeaderboards = [
+    {
+      name: 'Weather_genApp_experts',
+      description: 'This is the description',
+      modelsEvaluated: '16',
+    },
+    {
+      name: 'TidePressure_modeling_team',
+      description: 'This is the description',
+      modelsEvaluated: '30',
+    },
+    {
+      name: 'ClimateModel_genApp_experts',
+      description: 'This is the description',
+      modelsEvaluated: '15',
+    },
+    {
+      name: 'OceanCurrent_analysis_team',
+      description: 'This is the description',
+      modelsEvaluated: '70',
+    },
+    {
+      name: 'Windflow_projection_experts',
+      description: 'This is the description',
+      modelsEvaluated: '82',
+    },
+    {
+      name: 'AtmoMetrics_analysis_team',
+      description: 'This is the description',
+      modelsEvaluated: '40',
+    },
+    {
+      name: 'HydroFluor_research_specialists',
+      description: 'This is the description',
+      modelsEvaluated: '202',
+    },
+  ];
+
+  const evalBoards = [
+    {
+      name: 'Weather_genApp_experts',
+      description: 'This is the description',
+      modelsEvaluated: '16',
+    },
+    {
+      name: 'TidePressure_modeling_team',
+      description: 'This is the description',
+      modelsEvaluated: '30',
+    },
+    {
+      name: 'ClimateModel_genApp_experts',
+      description: 'This is the description',
+      modelsEvaluated: '15',
+    },
+    {
+      name: 'OceanCurrent_analysis_team',
+      description: 'This is the description',
+      modelsEvaluated: '70',
+    },
+    {
+      name: 'Windflow_projection_experts',
+      description: 'This is the description',
+      modelsEvaluated: '82',
+    },
+    {
+      name: 'AtmoMetrics_analysis_team',
+      description: 'This is the description',
+      modelsEvaluated: '40',
+    },
+    {
+      name: 'HydroFluor_research_specialists',
+      description: 'This is the description',
+      modelsEvaluated: '202',
+    },
   ];
 
   return (
     <Container>
-      <Header>
-        <Title>Annotation queues</Title>
-        <CreateQueueButton>+ Create queue</CreateQueueButton>
-      </Header>
-      <WelcomeMessage>
-        Welcome to annotation queues
-        <DismissButton>Dismiss</DismissButton>
-      </WelcomeMessage>
       <Section>
-        <SectionTitle>Needs review (2)</SectionTitle>
+        <SectionTitle>Custom Leaderboards</SectionTitle>
         <QueueGrid>
-          {annotationQueues.filter(queue => queue.needsReview).map(queue => (
+          {customLeaderboards.map(queue => (
             <QueueCard key={queue.name}>
               <QueueName>{queue.name}</QueueName>
-              <ReviewButton>Review</ReviewButton>
               <QueueDescription>{queue.description}</QueueDescription>
-              <TracesCount>{queue.traces} Traces</TracesCount>
+              <TracesCount>
+                {queue.modelsEvaluated} Models Evaluated
+              </TracesCount>
             </QueueCard>
           ))}
         </QueueGrid>
       </Section>
       <Section>
-        <SectionTitle>Up to date (5)</SectionTitle>
+        <SectionTitle>Evaluation Leaderboards</SectionTitle>
         <QueueGrid>
-          {annotationQueues.filter(queue => !queue.needsReview).map(queue => (
+          {evalBoards.map(queue => (
             <QueueCard key={queue.name}>
               <QueueName>{queue.name}</QueueName>
               <QueueDescription>{queue.description}</QueueDescription>
-              <TracesCount>{queue.traces} Traces</TracesCount>
+              <TracesCount>
+                {queue.modelsEvaluated} Models Evaluated
+              </TracesCount>
             </QueueCard>
           ))}
         </QueueGrid>
@@ -74,7 +154,7 @@ const Title = styled.h1`
 `;
 
 const CreateQueueButton = styled.button`
-  background-color: #00A4B8;
+  background-color: #00a4b8;
   color: white;
   border: none;
   padding: 10px 15px;
@@ -95,7 +175,7 @@ const WelcomeMessage = styled.div`
 const DismissButton = styled.button`
   background: none;
   border: none;
-  color: #00A4B8;
+  color: #00a4b8;
   cursor: pointer;
 `;
 
@@ -126,7 +206,7 @@ const QueueName = styled.h3`
 `;
 
 const ReviewButton = styled.button`
-  background-color: #FFA500;
+  background-color: #ffa500;
   color: white;
   border: none;
   padding: 5px 10px;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 4858ee9ea03..af7f6885336 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -15,9 +15,7 @@ type LeaderboardPageProps = {
 };
 
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
-  return (
-    <LeaderboardPageContent {...props} />
-  );
+  return <LeaderboardPageContent {...props} />;
 };
 
 const DEFAULT_DESCRIPTION = `# Leaderboard`;

From ff8c9584d6af047f5850fa151a4cff0008d81a04 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 18:59:16 -0700
Subject: [PATCH 073/169] basic listing layout

---
 .../LeaderboardListingPage.tsx                | 47 +++++++++++--------
 .../Browse3/pages/common/EmptyContent.tsx     | 20 ++++++++
 2 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index fd22d5902c7..2ac0731b849 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -2,7 +2,10 @@ import React from 'react';
 import styled from 'styled-components';
 
 import {Empty} from '../common/Empty';
-import {EMPTY_PROPS_EVALUATIONS} from '../common/EmptyContent';
+import {
+  EMPTY_PROPS_EVALUATIONS,
+  EMPTY_PROPS_LEADERBOARDS,
+} from '../common/EmptyContent';
 import {SimplePageLayout} from '../common/SimplePageLayout';
 
 export const LeaderboardListingPage: React.FC<{
@@ -103,24 +106,34 @@ export const LeaderboardListingPageInner: React.FC<{
     },
   ];
 
-  return (
+  const hasCustomLeaderboards = customLeaderboards.length > 0;
+  const hasEvalLeaderboards = evalBoards.length > 0;
+  const allEmpty = !hasCustomLeaderboards && !hasEvalLeaderboards;
+
+  return allEmpty ? (
+    <Empty {...EMPTY_PROPS_EVALUATIONS} />
+  ) : (
     <Container>
       <Section>
-        <SectionTitle>Custom Leaderboards</SectionTitle>
-        <QueueGrid>
-          {customLeaderboards.map(queue => (
-            <QueueCard key={queue.name}>
-              <QueueName>{queue.name}</QueueName>
-              <QueueDescription>{queue.description}</QueueDescription>
-              <TracesCount>
-                {queue.modelsEvaluated} Models Evaluated
-              </TracesCount>
-            </QueueCard>
-          ))}
-        </QueueGrid>
+        <SectionTitle>Custom Leaderboards ({customLeaderboards.length})</SectionTitle>
+        {!hasCustomLeaderboards ? (
+          <Empty {...EMPTY_PROPS_LEADERBOARDS} />
+        ) : (
+          <QueueGrid>
+            {customLeaderboards.map(queue => (
+              <QueueCard key={queue.name}>
+                <QueueName>{queue.name}</QueueName>
+                <QueueDescription>{queue.description}</QueueDescription>
+                <TracesCount>
+                  {queue.modelsEvaluated} Models Evaluated
+                </TracesCount>
+              </QueueCard>
+            ))}
+          </QueueGrid>
+        )}
       </Section>
       <Section>
-        <SectionTitle>Evaluation Leaderboards</SectionTitle>
+        <SectionTitle>Evaluation Leaderboards ({evalBoards.length})</SectionTitle>
         <QueueGrid>
           {evalBoards.map(queue => (
             <QueueCard key={queue.name}>
@@ -225,7 +238,3 @@ const TracesCount = styled.div`
   font-size: 24px;
   font-weight: bold;
 `;
-
-const NoEvaluations = () => {
-  return <Empty {...EMPTY_PROPS_EVALUATIONS} />;
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx
index 1db493e546f..08eaa361af6 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx
@@ -43,6 +43,26 @@ export const EMPTY_PROPS_EVALUATIONS: EmptyProps = {
   ),
 };
 
+export const EMPTY_PROPS_LEADERBOARDS: EmptyProps = {
+  icon: 'benchmark-square' as const,
+  heading: 'No leaderboards yet',
+  description:
+    'Use leaderboards to summarize and compare the performance of your models.',
+  moreInformation: (
+    <>
+      Learn{' '}
+      <TargetBlank href="https://wandb.me/weave_evals">
+        evaluation basics
+      </TargetBlank>{' '}
+      or follow our tutorial to{' '}
+      <TargetBlank href="http://wandb.me/weave_eval_tut">
+        set up an evaluation pipeline
+      </TargetBlank>
+      .
+    </>
+  ),
+};
+
 export const EMPTY_PROPS_MODEL: EmptyProps = {
   icon: 'model' as const,
   heading: 'No models yet',

From 43ec46c41b72acd661b5703825361ad6e78bc784 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 19:03:57 -0700
Subject: [PATCH 074/169] added button

---
 .../LeaderboardListingPage.tsx                | 37 ++++++++++++++-----
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index 2ac0731b849..6b73cf2c607 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -1,4 +1,6 @@
-import React from 'react';
+import { Box } from '@material-ui/core';
+import { Button } from '@wandb/weave/components/Button/Button';
+import React, { FC } from 'react';
 import styled from 'styled-components';
 
 import {Empty} from '../common/Empty';
@@ -22,6 +24,7 @@ export const LeaderboardListingPage: React.FC<{
           content: <LeaderboardListingPageInner {...props} />,
         },
       ]}
+      headerExtra={<CreateLeaderboardButton/>}
     />
   );
 };
@@ -166,14 +169,30 @@ const Title = styled.h1`
   margin: 0;
 `;
 
-const CreateQueueButton = styled.button`
-  background-color: #00a4b8;
-  color: white;
-  border: none;
-  padding: 10px 15px;
-  border-radius: 4px;
-  cursor: pointer;
-`;
+
+const CreateLeaderboardButton: FC = (
+) => {
+  return (
+    <Box
+      sx={{
+        height: '100%',
+        display: 'flex',
+        alignItems: 'center',
+      }}>
+      <Button
+        className="mx-16"
+        style={{
+          marginLeft: '0px',
+        }}
+        size="medium"
+        variant="secondary"
+        onClick={console.log}
+        icon="add-new">
+        Create Leaderboard
+      </Button>
+    </Box>
+  );
+};
 
 const WelcomeMessage = styled.div`
   background-color: #f0f0f0;

From ba039a1dea9b8b1f1d0052d4753404eccce84a2d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 17 Oct 2024 19:13:27 -0700
Subject: [PATCH 075/169] a little bit

---
 .../LeaderboardListingPage.tsx                | 100 +++++++-----------
 1 file changed, 40 insertions(+), 60 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index 6b73cf2c607..3083ee70068 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -125,11 +125,11 @@ export const LeaderboardListingPageInner: React.FC<{
           <QueueGrid>
             {customLeaderboards.map(queue => (
               <QueueCard key={queue.name}>
-                <QueueName>{queue.name}</QueueName>
-                <QueueDescription>{queue.description}</QueueDescription>
-                <TracesCount>
-                  {queue.modelsEvaluated} Models Evaluated
-                </TracesCount>
+                <LeaderboardName>{queue.name}</LeaderboardName>
+                <LeaderboardDescription>{queue.description}</LeaderboardDescription>
+                <ModelCount>
+                  {queue.modelsEvaluated} models
+                </ModelCount>
               </QueueCard>
             ))}
           </QueueGrid>
@@ -140,11 +140,11 @@ export const LeaderboardListingPageInner: React.FC<{
         <QueueGrid>
           {evalBoards.map(queue => (
             <QueueCard key={queue.name}>
-              <QueueName>{queue.name}</QueueName>
-              <QueueDescription>{queue.description}</QueueDescription>
-              <TracesCount>
-                {queue.modelsEvaluated} Models Evaluated
-              </TracesCount>
+              <LeaderboardName>{queue.name}</LeaderboardName>
+              <LeaderboardDescription>{queue.description}</LeaderboardDescription>
+              <ModelCount>
+                {queue.modelsEvaluated} models
+              </ModelCount>
             </QueueCard>
           ))}
         </QueueGrid>
@@ -154,20 +154,12 @@ export const LeaderboardListingPageInner: React.FC<{
 };
 
 const Container = styled.div`
-  padding: 20px;
-`;
-
-const Header = styled.div`
-  display: flex;
-  justify-content: space-between;
-  align-items: center;
-  margin-bottom: 20px;
+  padding: 16px;
+  width: 100%;
+  height: 100%;
+  overflow: auto;
 `;
 
-const Title = styled.h1`
-  font-size: 24px;
-  margin: 0;
-`;
 
 
 const CreateLeaderboardButton: FC = (
@@ -194,25 +186,10 @@ const CreateLeaderboardButton: FC = (
   );
 };
 
-const WelcomeMessage = styled.div`
-  background-color: #f0f0f0;
-  padding: 10px;
-  border-radius: 4px;
-  display: flex;
-  justify-content: space-between;
-  align-items: center;
-  margin-bottom: 20px;
-`;
-
-const DismissButton = styled.button`
-  background: none;
-  border: none;
-  color: #00a4b8;
-  cursor: pointer;
-`;
 
 const Section = styled.div`
-  margin-bottom: 20px;
+  margin-top: 0px;
+  margin-bottom: 30px;
 `;
 
 const SectionTitle = styled.h2`
@@ -222,38 +199,41 @@ const SectionTitle = styled.h2`
 
 const QueueGrid = styled.div`
   display: grid;
-  grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
-  gap: 20px;
+  grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
+  gap: 24px;
 `;
 
 const QueueCard = styled.div`
   border: 1px solid #e0e0e0;
-  border-radius: 4px;
-  padding: 15px;
-`;
+  border-radius: 8px;
+  padding: 20px;
+  transition: all 0.3s ease;
+  cursor: pointer;
+  background-color: #fff;
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
 
-const QueueName = styled.h3`
-  font-size: 16px;
-  margin: 0 0 10px 0;
+  &:hover {
+    box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
+    border-color: #00A4B8;
+  }
 `;
 
-const ReviewButton = styled.button`
-  background-color: #ffa500;
-  color: white;
-  border: none;
-  padding: 5px 10px;
-  border-radius: 4px;
-  cursor: pointer;
-  float: right;
+const LeaderboardName = styled.h3`
+  font-size: 18px;
+  font-weight: 600;
+  margin: 0 0 12px 0;
+  color: #00A4B8;
 `;
 
-const QueueDescription = styled.p`
+const LeaderboardDescription = styled.p`
   font-size: 14px;
   color: #666;
-  margin: 0 0 10px 0;
+  margin: 0 0 16px 0;
+  line-height: 1.4;
 `;
 
-const TracesCount = styled.div`
-  font-size: 24px;
-  font-weight: bold;
+const ModelCount = styled.div`
+  font-size: 16px;
+  font-weight: 600;
+  color: #333;
 `;

From 19c931629780daaa8c698020143c817c12dfb835 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Fri, 18 Oct 2024 12:52:36 -0700
Subject: [PATCH 076/169] ok, getting things a bit closer

---
 tests/trace/test_leaderboard.py               |  13 +-
 .../LeaderboardListingPage.tsx                | 317 +++++++++---------
 .../pages/LeaderboardPage/LeaderboardPage.tsx |   2 +-
 .../Home/Browse3/pages/ObjectVersionPage.tsx  |   3 +-
 .../Home/Browse3/pages/ObjectVersionsPage.tsx | 112 ++++---
 .../Home/Browse3/pages/common/Links.tsx       |  54 ++-
 .../pages/common/TypeVersionCategoryChip.tsx  |   1 +
 .../pages/wfReactInterface/constants.ts       |   1 +
 weave/flow/leaderboard.py                     |  19 +-
 9 files changed, 307 insertions(+), 215 deletions(-)

diff --git a/tests/trace/test_leaderboard.py b/tests/trace/test_leaderboard.py
index ed8b950ad2b..be0aa9820c7 100644
--- a/tests/trace/test_leaderboard.py
+++ b/tests/trace/test_leaderboard.py
@@ -5,8 +5,8 @@
 from weave.trace.weave_client import get_ref
 
 
-def test_emptyleaderboard(client):
-    spec = leaderboard.LeaderboardSpec(
+def test_leaderboard_empty(client):
+    spec = leaderboard.Leaderboard(
         name="Test Leaderboard",
         description="This is a test leaderboard",
         columns=[
@@ -63,7 +63,7 @@ def bad_model(input):
 async def test_leaderboard_with_results(client):
     evaluation_obj_1, evaluation_obj_2, simple_model, static_model, bad_model = await do_evaluations()
 
-    spec = leaderboard.LeaderboardSpec(
+    spec = leaderboard.Leaderboard(
         name="Test Leaderboard",
         description="This is a test leaderboard",
         columns=[
@@ -75,12 +75,17 @@ async def test_leaderboard_with_results(client):
         ]
     )
 
+    ref = weave.publish(spec)
+
+    # TODO: this is not working
+    # spec = ref.get()
+
     results = leaderboard.get_leaderboard_results(spec, client)
     assert len(results) == 1
     assert results[0].model_ref == get_ref(simple_model).uri()
     assert results[0].column_scores[0].scores[0].value == 1.0
 
-    spec = leaderboard.LeaderboardSpec(
+    spec = leaderboard.Leaderboard(
         name="Test Leaderboard",
         description="This is a test leaderboard",
         columns=[
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index 3083ee70068..49cdebd3d5d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -1,14 +1,25 @@
-import { Box } from '@material-ui/core';
-import { Button } from '@wandb/weave/components/Button/Button';
-import React, { FC } from 'react';
+import {Box} from '@material-ui/core';
+import {Button} from '@wandb/weave/components/Button/Button';
+import React, {FC, useCallback} from 'react';
+import {useHistory} from 'react-router-dom';
 import styled from 'styled-components';
 
+import {useWeaveflowRouteContext} from '../../context';
+import {StyledDataGrid} from '../../StyledDataGrid';
+import {CallsTable} from '../CallsPage/CallsTable';
+import {useEvaluationsFilter} from '../CallsPage/evaluationsFilter';
 import {Empty} from '../common/Empty';
 import {
   EMPTY_PROPS_EVALUATIONS,
   EMPTY_PROPS_LEADERBOARDS,
 } from '../common/EmptyContent';
 import {SimplePageLayout} from '../common/SimplePageLayout';
+import {
+  FilterableObjectVersionsTable,
+  ObjectVersionsTable,
+} from '../ObjectVersionsPage';
+import {useWFHooks} from '../wfReactInterface/context';
+import {ObjectVersionSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 
 export const LeaderboardListingPage: React.FC<{
   entity: string;
@@ -24,7 +35,7 @@ export const LeaderboardListingPage: React.FC<{
           content: <LeaderboardListingPageInner {...props} />,
         },
       ]}
-      headerExtra={<CreateLeaderboardButton/>}
+      headerExtra={<CreateLeaderboardButton />}
     />
   );
 };
@@ -33,111 +44,116 @@ export const LeaderboardListingPageInner: React.FC<{
   entity: string;
   project: string;
 }> = props => {
-  const customLeaderboards = [
-    {
-      name: 'Weather_genApp_experts',
-      description: 'This is the description',
-      modelsEvaluated: '16',
-    },
-    {
-      name: 'TidePressure_modeling_team',
-      description: 'This is the description',
-      modelsEvaluated: '30',
-    },
-    {
-      name: 'ClimateModel_genApp_experts',
-      description: 'This is the description',
-      modelsEvaluated: '15',
-    },
-    {
-      name: 'OceanCurrent_analysis_team',
-      description: 'This is the description',
-      modelsEvaluated: '70',
-    },
-    {
-      name: 'Windflow_projection_experts',
-      description: 'This is the description',
-      modelsEvaluated: '82',
-    },
-    {
-      name: 'AtmoMetrics_analysis_team',
-      description: 'This is the description',
-      modelsEvaluated: '40',
-    },
-    {
-      name: 'HydroFluor_research_specialists',
-      description: 'This is the description',
-      modelsEvaluated: '202',
-    },
-  ];
-
-  const evalBoards = [
-    {
-      name: 'Weather_genApp_experts',
-      description: 'This is the description',
-      modelsEvaluated: '16',
-    },
-    {
-      name: 'TidePressure_modeling_team',
-      description: 'This is the description',
-      modelsEvaluated: '30',
-    },
-    {
-      name: 'ClimateModel_genApp_experts',
-      description: 'This is the description',
-      modelsEvaluated: '15',
-    },
-    {
-      name: 'OceanCurrent_analysis_team',
-      description: 'This is the description',
-      modelsEvaluated: '70',
-    },
-    {
-      name: 'Windflow_projection_experts',
-      description: 'This is the description',
-      modelsEvaluated: '82',
-    },
-    {
-      name: 'AtmoMetrics_analysis_team',
-      description: 'This is the description',
-      modelsEvaluated: '40',
-    },
-    {
-      name: 'HydroFluor_research_specialists',
-      description: 'This is the description',
-      modelsEvaluated: '202',
-    },
-  ];
+  // const customLeaderboards = [
+  //   {
+  //     name: 'Weather_genApp_experts',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '16',
+  //   },
+  //   {
+  //     name: 'TidePressure_modeling_team',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '30',
+  //   },
+  //   {
+  //     name: 'ClimateModel_genApp_experts',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '15',
+  //   },
+  //   {
+  //     name: 'OceanCurrent_analysis_team',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '70',
+  //   },
+  //   {
+  //     name: 'Windflow_projection_experts',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '82',
+  //   },
+  //   {
+  //     name: 'AtmoMetrics_analysis_team',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '40',
+  //   },
+  //   {
+  //     name: 'HydroFluor_research_specialists',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '202',
+  //   },
+  // ];
+
+  // const evalBoards = [
+  //   {
+  //     name: 'Weather_genApp_experts',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '16',
+  //   },
+  //   {
+  //     name: 'TidePressure_modeling_team',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '30',
+  //   },
+  //   {
+  //     name: 'ClimateModel_genApp_experts',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '15',
+  //   },
+  //   {
+  //     name: 'OceanCurrent_analysis_team',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '70',
+  //   },
+  //   {
+  //     name: 'Windflow_projection_experts',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '82',
+  //   },
+  //   {
+  //     name: 'AtmoMetrics_analysis_team',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '40',
+  //   },
+  //   {
+  //     name: 'HydroFluor_research_specialists',
+  //     description: 'This is the description',
+  //     modelsEvaluated: '202',
+  //   },
+  // ];
+
+  // const hasCustomLeaderboards = customLeaderboards.length > 0;
+  // const hasEvalLeaderboards = evalBoards.length > 0;
+  // const allEmpty = !hasCustomLeaderboards && !hasEvalLeaderboards;
+  const evaluationsFilter = useEvaluationsFilter(props.entity, props.project);
 
-  const hasCustomLeaderboards = customLeaderboards.length > 0;
-  const hasEvalLeaderboards = evalBoards.length > 0;
-  const allEmpty = !hasCustomLeaderboards && !hasEvalLeaderboards;
-
-  return allEmpty ? (
-    <Empty {...EMPTY_PROPS_EVALUATIONS} />
-  ) : (
+  return (
     <Container>
       <Section>
-        <SectionTitle>Custom Leaderboards ({customLeaderboards.length})</SectionTitle>
-        {!hasCustomLeaderboards ? (
-          <Empty {...EMPTY_PROPS_LEADERBOARDS} />
-        ) : (
-          <QueueGrid>
-            {customLeaderboards.map(queue => (
-              <QueueCard key={queue.name}>
-                <LeaderboardName>{queue.name}</LeaderboardName>
-                <LeaderboardDescription>{queue.description}</LeaderboardDescription>
-                <ModelCount>
-                  {queue.modelsEvaluated} models
-                </ModelCount>
-              </QueueCard>
-            ))}
-          </QueueGrid>
-        )}
+        {/* <SectionTitle>Leaderboards</SectionTitle> */}
+        <LeaderboardTable entity={props.entity} project={props.project} />
+        {/* // <QueueGrid>
+          //   {customLeaderboards.map(queue => (
+          //     <QueueCard key={queue.name}>
+          //       <LeaderboardName>{queue.name}</LeaderboardName>
+          //       <LeaderboardDescription>{queue.description}</LeaderboardDescription>
+          //       <ModelCount>
+          //         {queue.modelsEvaluated} models
+          //       </ModelCount>
+          //     </QueueCard>
+          //   ))}
+          // </QueueGrid>
+        )} */}
       </Section>
       <Section>
-        <SectionTitle>Evaluation Leaderboards ({evalBoards.length})</SectionTitle>
-        <QueueGrid>
+        <SectionTitle>Recent Evaluations</SectionTitle>
+        {/* <CallsTable 
+            entity={props.entity}
+            project={props.project}
+            frozenFilter={evaluationsFilter}
+            // hideControls
+            // hideOpSelector
+            allowedColumnPatterns={['status','inputs.self', 'inputs.model', 'output.*']}
+          /> */}
+        {/* <QueueGrid>
           {evalBoards.map(queue => (
             <QueueCard key={queue.name}>
               <LeaderboardName>{queue.name}</LeaderboardName>
@@ -147,23 +163,22 @@ export const LeaderboardListingPageInner: React.FC<{
               </ModelCount>
             </QueueCard>
           ))}
-        </QueueGrid>
+        </QueueGrid> */}
       </Section>
     </Container>
   );
 };
 
 const Container = styled.div`
-  padding: 16px;
+  // padding: 16px;
   width: 100%;
   height: 100%;
-  overflow: auto;
+  overflow: hidden;
+  display: flex;
+  flex-direction: column;
 `;
 
-
-
-const CreateLeaderboardButton: FC = (
-) => {
+const CreateLeaderboardButton: FC = () => {
   return (
     <Box
       sx={{
@@ -186,10 +201,11 @@ const CreateLeaderboardButton: FC = (
   );
 };
 
-
 const Section = styled.div`
   margin-top: 0px;
   margin-bottom: 30px;
+  flex: 1;
+  overflow: hidden;
 `;
 
 const SectionTitle = styled.h2`
@@ -197,43 +213,42 @@ const SectionTitle = styled.h2`
   margin-bottom: 10px;
 `;
 
-const QueueGrid = styled.div`
-  display: grid;
-  grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
-  gap: 24px;
-`;
-
-const QueueCard = styled.div`
-  border: 1px solid #e0e0e0;
-  border-radius: 8px;
-  padding: 20px;
-  transition: all 0.3s ease;
-  cursor: pointer;
-  background-color: #fff;
-  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
-
-  &:hover {
-    box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
-    border-color: #00A4B8;
-  }
-`;
-
-const LeaderboardName = styled.h3`
-  font-size: 18px;
-  font-weight: 600;
-  margin: 0 0 12px 0;
-  color: #00A4B8;
-`;
-
-const LeaderboardDescription = styled.p`
-  font-size: 14px;
-  color: #666;
-  margin: 0 0 16px 0;
-  line-height: 1.4;
-`;
+const LeaderboardTable: React.FC<{
+  entity: string;
+  project: string;
+}> = props => {
+  const history = useHistory();
+  const {peekingRouter} = useWeaveflowRouteContext();
+  const {useRootObjectVersions} = useWFHooks();
+  const leaderboardObjectVersions = useRootObjectVersions(
+    props.entity,
+    props.project,
+    {
+      baseObjectClasses: ['Leaderboard'],
+      latestOnly: true,
+    }
+  );
+  const onClick = useCallback(
+    (obj: ObjectVersionSchema) => {
+      const to = peekingRouter.leaderboardsUIUrl(
+        props.entity,
+        props.project,
+        obj.objectId
+      );
+      history.push(to);
+    },
+    [history, peekingRouter, props.entity, props.project]
+  );
 
-const ModelCount = styled.div`
-  font-size: 16px;
-  font-weight: 600;
-  color: #333;
-`;
+  return (
+    <ObjectVersionsTable
+      objectVersions={leaderboardObjectVersions.result ?? []}
+      objectTitle="Name"
+      hidePropsAsColumns
+      hidePeerVersionsColumn
+      hideCategoryColumn
+      hideVersionSuffix
+      onRowClick={onClick}
+    />
+  );
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index af7f6885336..11b716ce183 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -103,7 +103,7 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
               alignItems: 'center',
             }}>
             <EditableMarkdown
-              value={description}
+              value={props.leaderboardName + '\n' + description}
               onChange={setDescription}
               placeholder={DEFAULT_DESCRIPTION}
             />
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx
index d287d6a3432..10a210ef346 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionPage.tsx
@@ -46,7 +46,8 @@ type ObjectIconProps = {
 const OBJECT_ICONS: Record<KnownBaseObjectClassType, IconName> = {
   Model: 'model',
   Dataset: 'table',
-  Evaluation: 'benchmark-square',
+  Evaluation: 'baseline-alt',
+  Leaderboard: 'benchmark-square',
 };
 const ObjectIcon = ({baseObjectClass}: ObjectIconProps) => {
   if (baseObjectClass in OBJECT_ICONS) {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
index 92bc347c222..101d887ccf5 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
@@ -29,10 +29,16 @@ import {basicField} from './common/DataTable';
 import {Empty} from './common/Empty';
 import {
   EMPTY_PROPS_DATASETS,
+  EMPTY_PROPS_LEADERBOARDS,
   EMPTY_PROPS_MODEL,
   EMPTY_PROPS_OBJECTS,
 } from './common/EmptyContent';
-import {ObjectVersionLink, ObjectVersionsLink} from './common/Links';
+import {
+  CustomLink,
+  ObjectVersionLink,
+  ObjectVersionsLink,
+  objectVersionText,
+} from './common/Links';
 import {FilterLayoutTemplate} from './common/SimpleFilterableDataTable';
 import {SimplePageLayout} from './common/SimplePageLayout';
 import {
@@ -148,11 +154,13 @@ export const FilterableObjectVersionsTable: React.FC<{
   const isEmpty = objectVersions.length === 0;
   if (isEmpty) {
     let propsEmpty = EMPTY_PROPS_OBJECTS;
-    const base = props.initialFilter?.baseObjectClass;
+    const base = effectiveFilter.baseObjectClass;
     if ('Model' === base) {
       propsEmpty = EMPTY_PROPS_MODEL;
     } else if (DATASET_BASE_OBJECT_CLASS === base) {
       propsEmpty = EMPTY_PROPS_DATASETS;
+    } else if (base === 'Leaderboard') {
+      propsEmpty = EMPTY_PROPS_LEADERBOARDS;
     }
     return <Empty {...propsEmpty} />;
   }
@@ -168,18 +176,25 @@ export const FilterableObjectVersionsTable: React.FC<{
       )}>
       <ObjectVersionsTable
         objectVersions={objectVersions}
-        usingLatestFilter={effectivelyLatestOnly}
+        hidePropsAsColumns={!!effectivelyLatestOnly}
+        hidePeerVersionsColumn={!effectivelyLatestOnly}
       />
     </FilterLayoutTemplate>
   );
 };
 
-const ObjectVersionsTable: React.FC<{
+export const ObjectVersionsTable: React.FC<{
   objectVersions: ObjectVersionSchema[];
-  usingLatestFilter?: boolean;
+  objectTitle?: string;
+  hidePropsAsColumns?: boolean;
+  hidePeerVersionsColumn?: boolean;
+  hideCategoryColumn?: boolean;
+  hideCreatedAtColumn?: boolean;
+  hideVersionSuffix?: boolean;
+  onRowClick?: (objectVersion: ObjectVersionSchema) => void;
 }> = props => {
   // `showPropsAsColumns` probably needs to be a bit more robust
-  const showPropsAsColumns = !props.usingLatestFilter;
+  const showPropsAsColumns = !props.hidePropsAsColumns;
   const rows: GridRowsProp = useMemo(() => {
     const vals = props.objectVersions.map(ov => ov.val);
     const flat = prepareFlattenedDataForTable(vals);
@@ -213,11 +228,20 @@ const ObjectVersionsTable: React.FC<{
     const cols: GridColDef[] = [
       // This field name chosen to reduce possibility of conflict
       // with the dynamic fields added below.
-      basicField('weave__object_version_link', 'Object', {
+      basicField('weave__object_version_link', props.objectTitle ?? 'Object', {
         hideable: false,
         renderCell: cellParams => {
           // Icon to indicate navigation to the object version
           const obj: ObjectVersionSchema = cellParams.row.obj;
+          if (props.onRowClick) {
+            const text = props.hideVersionSuffix
+              ? obj.objectId
+              : objectVersionText(obj.objectId, obj.versionIndex);
+
+            return (
+              <CustomLink text={text} onClick={() => props.onRowClick?.(obj)} />
+            );
+          }
           return (
             <ObjectVersionLink
               entityName={obj.entity}
@@ -227,6 +251,7 @@ const ObjectVersionsTable: React.FC<{
               versionIndex={obj.versionIndex}
               fullWidth={true}
               color={TEAL_600}
+              hideVersionSuffix={props.hideVersionSuffix}
             />
           );
         },
@@ -277,40 +302,45 @@ const ObjectVersionsTable: React.FC<{
       groups = groupingModel;
     }
 
-    cols.push(
-      basicField('baseObjectClass', 'Category', {
-        width: 100,
-        display: 'flex',
-        valueGetter: (unused: any, row: any) => {
-          return row.obj.baseObjectClass;
-        },
-        renderCell: cellParams => {
-          const category = cellParams.value;
-          if (
-            category === 'Model' ||
-            category === 'Dataset' ||
-            category === 'Evaluation'
-          ) {
-            return <TypeVersionCategoryChip baseObjectClass={category} />;
-          }
-          return null;
-        },
-      })
-    );
+    if (!props.hideCategoryColumn) {
+      cols.push(
+        basicField('baseObjectClass', 'Category', {
+          width: 100,
+          display: 'flex',
+          valueGetter: (unused: any, row: any) => {
+            return row.obj.baseObjectClass;
+          },
+          renderCell: cellParams => {
+            const category = cellParams.value;
+            if (
+              category === 'Model' ||
+              category === 'Dataset' ||
+              category === 'Evaluation'
+            ) {
+              return <TypeVersionCategoryChip baseObjectClass={category} />;
+            }
+            return null;
+          },
+        })
+      );
+    }
 
-    cols.push(
-      basicField('createdAtMs', 'Created', {
-        width: 100,
-        valueGetter: (unused: any, row: any) => {
-          return row.obj.createdAtMs;
-        },
-        renderCell: cellParams => {
-          const createdAtMs = cellParams.value;
-          return <Timestamp value={createdAtMs / 1000} format="relative" />;
-        },
-      })
-    );
-    if (props.usingLatestFilter) {
+    if (!props.hideCreatedAtColumn) {
+      cols.push(
+        basicField('createdAtMs', 'Created', {
+          width: 100,
+          valueGetter: (unused: any, row: any) => {
+            return row.obj.createdAtMs;
+          },
+          renderCell: cellParams => {
+            const createdAtMs = cellParams.value;
+            return <Timestamp value={createdAtMs / 1000} format="relative" />;
+          },
+        })
+      );
+    }
+
+    if (!props.hidePeerVersionsColumn) {
       cols.push(
         basicField('peerVersions', 'Versions', {
           width: 100,
@@ -325,7 +355,7 @@ const ObjectVersionsTable: React.FC<{
     }
 
     return {cols, groups};
-  }, [showPropsAsColumns, props.usingLatestFilter, rows]);
+  }, [props, showPropsAsColumns, rows]);
 
   // Highlight table row if it matches peek drawer.
   const query = useURLSearchParamsDict();
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
index b33d5f8d125..93a19fa2cb8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
@@ -43,6 +43,15 @@ export const Link = styled(LinkComp)<LinkProps>`
 `;
 Link.displayName = 'S.Link';
 
+export const FakeLink = styled.div<LinkProps>`
+  font-weight: 600;
+  color: ${p => (p.$variant === 'secondary' ? MOON_700 : TEAL_600)};
+  &:hover {
+    color: ${TEAL_500};
+  }
+`;
+Link.displayName = 'S.Link';
+
 const LinkWrapper = styled.div<{fullWidth?: boolean; color?: string}>`
   ${p =>
     p.fullWidth &&
@@ -146,13 +155,16 @@ export const ObjectVersionLink: React.FC<{
   fullWidth?: boolean;
   icon?: React.ReactNode;
   color?: string;
+  hideVersionSuffix?: boolean;
 }> = props => {
   const history = useHistory();
   const {peekingRouter} = useWeaveflowRouteContext();
   // const text = props.hideName
   //   ? props.version
   //   : props.objectName + ': ' + truncateID(props.version);
-  const text = objectVersionText(props.objectName, props.versionIndex);
+  const text = props.hideVersionSuffix
+    ? props.objectName
+    : objectVersionText(props.objectName, props.versionIndex);
   const to = peekingRouter.objectVersionUIUrl(
     props.entityName,
     props.projectName,
@@ -331,6 +343,46 @@ export const CallLink: React.FC<{
   );
 };
 
+export const CustomLink: React.FC<{
+  text: string;
+  onClick: () => void;
+  fullWidth?: boolean;
+  color?: string;
+  variant?: LinkVariant;
+  icon?: React.ReactNode;
+}> = props => {
+  return (
+    <LinkWrapper
+      onClick={props.onClick}
+      fullWidth={props.fullWidth}
+      color={props.color}>
+      <LinkTruncater fullWidth={props.fullWidth}>
+        <FakeLink
+          $variant={props.variant}
+          style={{
+            display: 'flex',
+            alignItems: 'center',
+            gap: '4px',
+            // allow flex items to shrink below their minimum content size
+            minWidth: 0,
+          }}>
+          {props.icon}
+          <span
+            style={{
+              overflow: 'hidden',
+              textOverflow: 'ellipsis',
+              whiteSpace: 'nowrap',
+              flexGrow: 1,
+              flexShrink: 1,
+            }}>
+            {props.text}
+          </span>
+        </FakeLink>
+      </LinkTruncater>
+    </LinkWrapper>
+  );
+};
+
 export const CallsLink: React.FC<{
   entity: string;
   project: string;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/TypeVersionCategoryChip.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/TypeVersionCategoryChip.tsx
index a1c366fd1da..c57f3a9b1fd 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/TypeVersionCategoryChip.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/TypeVersionCategoryChip.tsx
@@ -7,6 +7,7 @@ const colorMap: Record<KnownBaseObjectClassType, TagColorName> = {
   Model: 'blue',
   Dataset: 'green',
   Evaluation: 'cactus',
+  Leaderboard: 'gold',
 };
 
 export const TypeVersionCategoryChip: React.FC<{
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/constants.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/constants.ts
index 8b5aca318c9..6bc05866094 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/constants.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/constants.ts
@@ -23,4 +23,5 @@ export const KNOWN_BASE_OBJECT_CLASSES = [
   'Model',
   'Dataset',
   'Evaluation',
+  'Leaderboard',
 ] as const;
diff --git a/weave/flow/leaderboard.py b/weave/flow/leaderboard.py
index dd6061a6981..827c9c83475 100644
--- a/weave/flow/leaderboard.py
+++ b/weave/flow/leaderboard.py
@@ -5,7 +5,7 @@
 
 import weave
 from weave.trace.refs import OpRef
-from weave.trace.weave_client import Call, WeaveClient, get_ref
+from weave.trace.weave_client import WeaveClient, get_ref
 from weave.trace_server.trace_server_interface import CallsFilter
 
 
@@ -15,7 +15,7 @@ class LeaderboardColumn(BaseModel):
     summary_metric_path_parts: list[str] = field(default_factory=list)
     should_minimize: Optional[bool] = None
 
-class LeaderboardSpec(weave.Object):
+class Leaderboard(weave.Object):
     columns: list[LeaderboardColumn]
 
 class LeaderboardModelEvaluationResult(BaseModel):
@@ -30,7 +30,7 @@ class LeaderboardModelResult(BaseModel):
     column_scores: list[ModelScoresForColumn]
 
 
-def get_leaderboard_results(spec: LeaderboardSpec, client: WeaveClient) -> list[LeaderboardModelResult]:
+def get_leaderboard_results(spec: Leaderboard, client: WeaveClient) -> list[LeaderboardModelResult]:
     entity, project = client._project_id().split("/")
     calls = client.get_calls(filter=CallsFilter(
         op_names=[OpRef(
@@ -41,19 +41,6 @@ def get_leaderboard_results(spec: LeaderboardSpec, client: WeaveClient) -> list[
         ).uri()],
         input_refs=[c.evaluation_object_ref for c  in spec.columns]
     ))
-    def get_scores(call: Call) -> list[float]:
-        res = []
-        for c in spec.columns:
-            val = call.output.get(c.scorer_name)
-            for part in c.summary_metric_path_parts:
-                if isinstance(val, dict):
-                    val = val.get(part)
-                elif isinstance(val, list):
-                    val = val[int(part)]
-                else:
-                    break
-            res.append(val)
-        return res
     
     res_map: dict[str, LeaderboardModelResult] = {}
     for call in calls:

From f327b02b631e4fec991ea37b017d7a7137327018 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Fri, 18 Oct 2024 12:55:46 -0700
Subject: [PATCH 077/169] cleanup leaderbaord

---
 .../LeaderboardListingPage.tsx                | 166 ++----------------
 1 file changed, 17 insertions(+), 149 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index 49cdebd3d5d..45d76916971 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -5,22 +5,27 @@ import {useHistory} from 'react-router-dom';
 import styled from 'styled-components';
 
 import {useWeaveflowRouteContext} from '../../context';
-import {StyledDataGrid} from '../../StyledDataGrid';
-import {CallsTable} from '../CallsPage/CallsTable';
-import {useEvaluationsFilter} from '../CallsPage/evaluationsFilter';
-import {Empty} from '../common/Empty';
-import {
-  EMPTY_PROPS_EVALUATIONS,
-  EMPTY_PROPS_LEADERBOARDS,
-} from '../common/EmptyContent';
 import {SimplePageLayout} from '../common/SimplePageLayout';
-import {
-  FilterableObjectVersionsTable,
-  ObjectVersionsTable,
-} from '../ObjectVersionsPage';
+import {ObjectVersionsTable} from '../ObjectVersionsPage';
 import {useWFHooks} from '../wfReactInterface/context';
 import {ObjectVersionSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 
+const Container = styled.div`
+  // padding: 16px;
+  width: 100%;
+  height: 100%;
+  overflow: hidden;
+  display: flex;
+  flex-direction: column;
+`;
+
+const Section = styled.div`
+  margin-top: 0px;
+  margin-bottom: 30px;
+  flex: 1;
+  overflow: hidden;
+`;
+
 export const LeaderboardListingPage: React.FC<{
   entity: string;
   project: string;
@@ -44,140 +49,15 @@ export const LeaderboardListingPageInner: React.FC<{
   entity: string;
   project: string;
 }> = props => {
-  // const customLeaderboards = [
-  //   {
-  //     name: 'Weather_genApp_experts',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '16',
-  //   },
-  //   {
-  //     name: 'TidePressure_modeling_team',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '30',
-  //   },
-  //   {
-  //     name: 'ClimateModel_genApp_experts',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '15',
-  //   },
-  //   {
-  //     name: 'OceanCurrent_analysis_team',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '70',
-  //   },
-  //   {
-  //     name: 'Windflow_projection_experts',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '82',
-  //   },
-  //   {
-  //     name: 'AtmoMetrics_analysis_team',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '40',
-  //   },
-  //   {
-  //     name: 'HydroFluor_research_specialists',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '202',
-  //   },
-  // ];
-
-  // const evalBoards = [
-  //   {
-  //     name: 'Weather_genApp_experts',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '16',
-  //   },
-  //   {
-  //     name: 'TidePressure_modeling_team',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '30',
-  //   },
-  //   {
-  //     name: 'ClimateModel_genApp_experts',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '15',
-  //   },
-  //   {
-  //     name: 'OceanCurrent_analysis_team',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '70',
-  //   },
-  //   {
-  //     name: 'Windflow_projection_experts',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '82',
-  //   },
-  //   {
-  //     name: 'AtmoMetrics_analysis_team',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '40',
-  //   },
-  //   {
-  //     name: 'HydroFluor_research_specialists',
-  //     description: 'This is the description',
-  //     modelsEvaluated: '202',
-  //   },
-  // ];
-
-  // const hasCustomLeaderboards = customLeaderboards.length > 0;
-  // const hasEvalLeaderboards = evalBoards.length > 0;
-  // const allEmpty = !hasCustomLeaderboards && !hasEvalLeaderboards;
-  const evaluationsFilter = useEvaluationsFilter(props.entity, props.project);
-
   return (
     <Container>
       <Section>
-        {/* <SectionTitle>Leaderboards</SectionTitle> */}
         <LeaderboardTable entity={props.entity} project={props.project} />
-        {/* // <QueueGrid>
-          //   {customLeaderboards.map(queue => (
-          //     <QueueCard key={queue.name}>
-          //       <LeaderboardName>{queue.name}</LeaderboardName>
-          //       <LeaderboardDescription>{queue.description}</LeaderboardDescription>
-          //       <ModelCount>
-          //         {queue.modelsEvaluated} models
-          //       </ModelCount>
-          //     </QueueCard>
-          //   ))}
-          // </QueueGrid>
-        )} */}
-      </Section>
-      <Section>
-        <SectionTitle>Recent Evaluations</SectionTitle>
-        {/* <CallsTable 
-            entity={props.entity}
-            project={props.project}
-            frozenFilter={evaluationsFilter}
-            // hideControls
-            // hideOpSelector
-            allowedColumnPatterns={['status','inputs.self', 'inputs.model', 'output.*']}
-          /> */}
-        {/* <QueueGrid>
-          {evalBoards.map(queue => (
-            <QueueCard key={queue.name}>
-              <LeaderboardName>{queue.name}</LeaderboardName>
-              <LeaderboardDescription>{queue.description}</LeaderboardDescription>
-              <ModelCount>
-                {queue.modelsEvaluated} models
-              </ModelCount>
-            </QueueCard>
-          ))}
-        </QueueGrid> */}
       </Section>
     </Container>
   );
 };
 
-const Container = styled.div`
-  // padding: 16px;
-  width: 100%;
-  height: 100%;
-  overflow: hidden;
-  display: flex;
-  flex-direction: column;
-`;
-
 const CreateLeaderboardButton: FC = () => {
   return (
     <Box
@@ -201,18 +81,6 @@ const CreateLeaderboardButton: FC = () => {
   );
 };
 
-const Section = styled.div`
-  margin-top: 0px;
-  margin-bottom: 30px;
-  flex: 1;
-  overflow: hidden;
-`;
-
-const SectionTitle = styled.h2`
-  font-size: 18px;
-  margin-bottom: 10px;
-`;
-
 const LeaderboardTable: React.FC<{
   entity: string;
   project: string;

From 0a203927027c30dbcb2e2e3a609042304d30129a Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Fri, 18 Oct 2024 13:07:19 -0700
Subject: [PATCH 078/169] a little cleanuo

---
 .../LeaderboardListingPage.tsx                | 31 +++++++++-------
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 37 ++++++++-----------
 2 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index 45d76916971..f4cbbd194b7 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -1,29 +1,23 @@
 import {Box} from '@material-ui/core';
 import {Button} from '@wandb/weave/components/Button/Button';
+import {ErrorPanel} from '@wandb/weave/components/ErrorPanel';
+import {Loading} from '@wandb/weave/components/Loading';
 import React, {FC, useCallback} from 'react';
 import {useHistory} from 'react-router-dom';
 import styled from 'styled-components';
 
 import {useWeaveflowRouteContext} from '../../context';
+import {Empty} from '../common/Empty';
+import {EMPTY_PROPS_LEADERBOARDS} from '../common/EmptyContent';
 import {SimplePageLayout} from '../common/SimplePageLayout';
 import {ObjectVersionsTable} from '../ObjectVersionsPage';
 import {useWFHooks} from '../wfReactInterface/context';
 import {ObjectVersionSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 
 const Container = styled.div`
-  // padding: 16px;
   width: 100%;
   height: 100%;
   overflow: hidden;
-  display: flex;
-  flex-direction: column;
-`;
-
-const Section = styled.div`
-  margin-top: 0px;
-  margin-bottom: 30px;
-  flex: 1;
-  overflow: hidden;
 `;
 
 export const LeaderboardListingPage: React.FC<{
@@ -51,9 +45,7 @@ export const LeaderboardListingPageInner: React.FC<{
 }> = props => {
   return (
     <Container>
-      <Section>
-        <LeaderboardTable entity={props.entity} project={props.project} />
-      </Section>
+      <LeaderboardTable entity={props.entity} project={props.project} />
     </Container>
   );
 };
@@ -108,6 +100,19 @@ const LeaderboardTable: React.FC<{
     [history, peekingRouter, props.entity, props.project]
   );
 
+  if (leaderboardObjectVersions.loading) {
+    return <Loading centered />;
+  }
+  if (leaderboardObjectVersions.error) {
+    return <ErrorPanel />;
+  }
+
+  const objectVersions = leaderboardObjectVersions.result ?? [];
+  const isEmpty = objectVersions.length === 0;
+  if (isEmpty) {
+    return <Empty {...EMPTY_PROPS_LEADERBOARDS} />;
+  }
+
   return (
     <ObjectVersionsTable
       objectVersions={leaderboardObjectVersions.result ?? []}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 11b716ce183..717cdef9ab0 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,10 +1,13 @@
-import {Alert, Box, Typography} from '@mui/material';
+import {Box} from '@mui/material';
 import {Button} from '@wandb/weave/components/Button';
 import React, {useCallback, useState} from 'react';
 
 import {LeaderboardGrid} from '../../views/Leaderboard/LeaderboardGrid';
 import {useLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
 import {LeaderboardConfigType} from '../../views/Leaderboard/types/leaderboardConfigType';
+import {
+  SimplePageLayout,
+} from '../common/SimplePageLayout';
 import {EditableMarkdown} from './EditableMarkdown';
 import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
 
@@ -15,7 +18,18 @@ type LeaderboardPageProps = {
 };
 
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
-  return <LeaderboardPageContent {...props} />;
+  return (
+    <SimplePageLayout
+      title={props.leaderboardName}
+      hideTabsIfSingle
+      tabs={[
+        {
+          label: 'Leaderboard',
+          content: <LeaderboardPageContent {...props} />,
+        },
+      ]}
+    />
+  );
 };
 
 const DEFAULT_DESCRIPTION = `# Leaderboard`;
@@ -72,8 +86,6 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
     currentConfig.config.dataSelectionSpec
   );
 
-  const [showingAlert, setShowingAlert] = useState(true);
-
   return (
     <Box display="flex" flexDirection="row" height="100%" flexGrow={1}>
       <Box
@@ -120,9 +132,6 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
             />
           </div>
         </Box>
-        {showingAlert && (
-          <UnlistedAlert onClose={() => setShowingAlert(false)} />
-        )}
         <Box flexGrow={1} display="flex" flexDirection="row" overflow="hidden">
           <LeaderboardGrid
             entity={entity}
@@ -175,17 +184,3 @@ export const ToggleLeaderboardConfig: React.FC<{
     </Box>
   );
 };
-
-const UnlistedAlert: React.FC<{onClose: () => void}> = ({onClose}) => {
-  return (
-    <Alert severity="info" onClose={onClose}>
-      <Typography variant="body1">
-        You have found an internal, unlisted beta page! Please expect bugs and
-        incomplete features. Permilinks & saving state are not yet supported
-      </Typography>
-    </Alert>
-  );
-};
-
-// TODO:
-// * [ ] Edit panel revisions

From 61e59c157e505fa3f3028fc5dd30c1855738fe85 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Fri, 18 Oct 2024 13:58:59 -0700
Subject: [PATCH 079/169] ok, getting close

---
 tests/trace/test_leaderboard.py               |  44 +--
 .../LeaderboardPage/EditableMarkdown.tsx      |   2 +-
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 264 +++++++++++++++---
 .../views/Leaderboard/LeaderboardGrid.tsx     |   7 +
 weave/flow/leaderboard.py                     |  46 +--
 5 files changed, 290 insertions(+), 73 deletions(-)

diff --git a/tests/trace/test_leaderboard.py b/tests/trace/test_leaderboard.py
index be0aa9820c7..288ed95b294 100644
--- a/tests/trace/test_leaderboard.py
+++ b/tests/trace/test_leaderboard.py
@@ -15,7 +15,7 @@ def test_leaderboard_empty(client):
                 scorer_name="test_scorer_name",
                 summary_metric_path="test_summary_metric_path",
             )
-        ]
+        ],
     )
 
     results = leaderboard.get_leaderboard_results(spec, client)
@@ -26,45 +26,52 @@ async def do_evaluations():
     @weave.op
     def my_scorer(target, model_output):
         return target == model_output
-    
+
     evaluation_obj_1 = weave.Evaluation(
         name="test_evaluation_name",
-        dataset=[{'input': 1, 'target': 1}],
-        scorers=[my_scorer]
+        dataset=[{"input": 1, "target": 1}],
+        scorers=[my_scorer],
     )
 
     @weave.op
     def simple_model(input):
         return input
-    
+
     await evaluation_obj_1.evaluate(simple_model)
 
     evaluation_obj_2 = weave.Evaluation(
         name="test_evaluation_name",
-        dataset=[{'input': 1, 'target': 1}, {'input': 2, 'target': 2}],
-        scorers=[my_scorer]
+        dataset=[{"input": 1, "target": 1}, {"input": 2, "target": 2}],
+        scorers=[my_scorer],
     )
 
     @weave.op
     def static_model(input):
         return 1
-    
+
     @weave.op
     def bad_model(input):
         return input + 1
-    
+
     await evaluation_obj_2.evaluate(simple_model)
     await evaluation_obj_2.evaluate(static_model)
     await evaluation_obj_2.evaluate(bad_model)
 
     return evaluation_obj_1, evaluation_obj_2, simple_model, static_model, bad_model
-    
+
+
 @pytest.mark.asyncio
 async def test_leaderboard_with_results(client):
-    evaluation_obj_1, evaluation_obj_2, simple_model, static_model, bad_model = await do_evaluations()
+    (
+        evaluation_obj_1,
+        evaluation_obj_2,
+        simple_model,
+        static_model,
+        bad_model,
+    ) = await do_evaluations()
 
     spec = leaderboard.Leaderboard(
-        name="Test Leaderboard",
+        name="Simple Leaderboard",
         description="This is a test leaderboard",
         columns=[
             leaderboard.LeaderboardColumn(
@@ -72,7 +79,7 @@ async def test_leaderboard_with_results(client):
                 scorer_name="my_scorer",
                 summary_metric_path_parts=["true_fraction"],
             )
-        ]
+        ],
     )
 
     ref = weave.publish(spec)
@@ -86,7 +93,7 @@ async def test_leaderboard_with_results(client):
     assert results[0].column_scores[0].scores[0].value == 1.0
 
     spec = leaderboard.Leaderboard(
-        name="Test Leaderboard",
+        name="Complex Leaderboard",
         description="This is a test leaderboard",
         columns=[
             leaderboard.LeaderboardColumn(
@@ -103,10 +110,15 @@ async def test_leaderboard_with_results(client):
                 evaluation_object_ref=get_ref(evaluation_obj_1).uri(),
                 scorer_name="my_scorer",
                 summary_metric_path_parts=["true_fraction"],
-            )
-        ]
+            ),
+        ],
     )
 
+    ref = weave.publish(spec)
+
+    # TODO: this is not working
+    # spec = ref.get()
+
     results = leaderboard.get_leaderboard_results(spec, client)
     assert len(results) == 3
     assert results[0].model_ref == get_ref(simple_model).uri()
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
index 930917ff6a2..e9b5f70c7e4 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
@@ -15,7 +15,7 @@ interface EditableMarkdownProps {
   placeholder: string;
 }
 
-const StyledReactMarkdown = styled(ReactMarkdown)`
+export const StyledReactMarkdown = styled(ReactMarkdown)`
   > *:first-child {
     margin-top: 0;
   }
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 717cdef9ab0..566bc46ca56 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,14 +1,21 @@
 import {Box} from '@mui/material';
 import {Button} from '@wandb/weave/components/Button';
-import React, {useCallback, useState} from 'react';
+import {ErrorPanel} from '@wandb/weave/components/ErrorPanel';
+import {Loading} from '@wandb/weave/components/Loading';
+import React, {useCallback, useEffect, useMemo, useState} from 'react';
 
+import {parseRefMaybe} from '../../../Browse2/SmallRef';
+import {NotFoundPanel} from '../../NotFoundPanel';
 import {LeaderboardGrid} from '../../views/Leaderboard/LeaderboardGrid';
 import {useLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
-import {LeaderboardConfigType} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {
-  SimplePageLayout,
-} from '../common/SimplePageLayout';
-import {EditableMarkdown} from './EditableMarkdown';
+  FilterAndGroupSourceEvaluationSpec,
+  FilterAndGroupSpec,
+  LeaderboardConfigType,
+} from '../../views/Leaderboard/types/leaderboardConfigType';
+import {SimplePageLayout} from '../common/SimplePageLayout';
+import {useWFHooks} from '../wfReactInterface/context';
+import {EditableMarkdown, StyledReactMarkdown} from './EditableMarkdown';
 import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
 
 type LeaderboardPageProps = {
@@ -18,14 +25,15 @@ type LeaderboardPageProps = {
 };
 
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
+  const [name, setName] = useState(props.leaderboardName);
   return (
     <SimplePageLayout
-      title={props.leaderboardName}
+      title={name}
       hideTabsIfSingle
       tabs={[
         {
           label: 'Leaderboard',
-          content: <LeaderboardPageContent {...props} />,
+          content: <LeaderboardPageContent {...props} setName={setName} />,
         },
       ]}
     />
@@ -57,34 +65,87 @@ const usePersistedLeaderboardConfig = () => {
   return {config, setConfigLocal, persistConfig, cancelChanges};
 };
 
-export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
+export const LeaderboardPageContent: React.FC<
+  LeaderboardPageProps & {setName: (name: string) => void}
+> = props => {
+  const {useRootObjectVersions} = useWFHooks();
   const {entity, project} = props;
+  const leaderboardObjectVersion = useRootObjectVersions(entity, project, {
+    objectIds: [props.leaderboardName],
+    baseObjectClasses: ['Leaderboard'],
+    latestOnly: true,
+  });
 
-  const [showConfig, setShowConfig] = useState(false);
-
-  const {
-    config: currentConfig,
-    setConfigLocal,
-    persistConfig,
-    cancelChanges,
-  } = usePersistedLeaderboardConfig();
-  const description = currentConfig.config.description;
-  const setDescription = useCallback(
-    (newDescription: string) => {
-      setConfigLocal(newConfig => ({
-        ...newConfig,
-        config: {...newConfig.config, description: newDescription},
-      }));
-      persistConfig();
-    },
-    [setConfigLocal, persistConfig]
+  if (leaderboardObjectVersion.loading) {
+    return <Loading centered />;
+  }
+
+  if (leaderboardObjectVersion.error) {
+    return <ErrorPanel />;
+  }
+
+  if (
+    leaderboardObjectVersion.result == null ||
+    leaderboardObjectVersion.result.length !== 1
+  ) {
+    return (
+      <NotFoundPanel
+        title={`Leaderboard (${props.leaderboardName}) not found`}
+      />
+    );
+  }
+
+  const leaderboardVal = parseLeaderboardVal(
+    leaderboardObjectVersion.result[0].val
   );
 
-  const {loading, data} = useLeaderboardData(
-    entity,
-    project,
-    currentConfig.config.dataSelectionSpec
+  if (leaderboardVal == null) {
+    return (
+      <NotFoundPanel
+        title={`Leaderboard (${props.leaderboardName}) had invalid data`}
+      />
+    );
+  }
+
+  return (
+    <LeaderboardPageContentInner {...props} leaderboardVal={leaderboardVal} />
   );
+};
+
+export const LeaderboardPageContentInner: React.FC<
+  LeaderboardPageProps & {setName: (name: string) => void} & {
+    leaderboardVal: LeaderboardVal;
+  }
+> = props => {
+  useEffect(() => {
+    props.setName(props.leaderboardVal.name);
+  }, [props]);
+
+  // const [showConfig, setShowConfig] = useState(false);
+
+  // const {
+  //   config: currentConfig,
+  //   setConfigLocal,
+  //   persistConfig,
+  //   cancelChanges,
+  // } = usePersistedLeaderboardConfig();
+  // const setDescription = useCallback(
+  //   (newDescription: string) => {
+  //     setConfigLocal(newConfig => ({
+  //       ...newConfig,
+  //       config: {...newConfig.config, description: newDescription},
+  //     }));
+  //     persistConfig();
+  //   },
+  //   [setConfigLocal, persistConfig]
+  // );
+
+  const spec = useMemo(() => {
+    return convertLeaderboardValToFilterAndGroupSpec(props.leaderboardVal);
+  }, [props.leaderboardVal]);
+
+  const description = props.leaderboardVal.description;
+  const {loading, data} = useLeaderboardData(props.entity, props.project, spec);
 
   return (
     <Box display="flex" flexDirection="row" height="100%" flexGrow={1}>
@@ -102,7 +163,7 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
           width="100%"
           sx={{
             alignItems: 'flex-start',
-            padding: '12px',
+            padding: '12px 16px',
             gap: '12px',
           }}>
           <Box
@@ -112,15 +173,18 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
               overflow: 'auto',
               height: '100%',
               display: 'flex',
-              alignItems: 'center',
+              alignItems: 'flex-start',
             }}>
-            <EditableMarkdown
+            {description && (
+              <StyledReactMarkdown>{description}</StyledReactMarkdown>
+            )}
+            {/* <EditableMarkdown
               value={props.leaderboardName + '\n' + description}
               onChange={setDescription}
               placeholder={DEFAULT_DESCRIPTION}
-            />
+            /> */}
           </Box>
-          <div
+          {/* <div
             style={{
               display: showConfig ? 'none' : 'block',
               // paddingRight: '12px',
@@ -130,18 +194,18 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
               isOpen={showConfig}
               onClick={() => setShowConfig(c => !c)}
             />
-          </div>
+          </div> */}
         </Box>
         <Box flexGrow={1} display="flex" flexDirection="row" overflow="hidden">
           <LeaderboardGrid
-            entity={entity}
-            project={project}
+            entity={props.entity}
+            project={props.project}
             loading={loading}
             data={data}
           />
         </Box>
       </Box>
-      {showConfig && (
+      {/* {showConfig && (
         <Box flex={1} width="35%" height="100%" overflow="hidden">
           <LeaderboardConfigEditor
             entity={entity}
@@ -158,7 +222,7 @@ export const LeaderboardPageContent: React.FC<LeaderboardPageProps> = props => {
             setConfig={setConfigLocal}
           />
         </Box>
-      )}
+      )} */}
     </Box>
   );
 };
@@ -184,3 +248,123 @@ export const ToggleLeaderboardConfig: React.FC<{
     </Box>
   );
 };
+
+type LeaderboardVal = {
+  name: string;
+  description: string;
+  columns: Array<{
+    evaluation_object_ref: string;
+    scorer_name: string;
+    should_minimize?: boolean;
+    summary_metric_path_parts: string[];
+  }>;
+};
+
+const parseLeaderboardVal = (leaderboardVal: any): LeaderboardVal | null => {
+  if (typeof leaderboardVal !== 'object' || leaderboardVal == null) {
+    return null;
+  }
+
+  const name = leaderboardVal.name;
+  if (typeof name !== 'string') {
+    return null;
+  }
+
+  const description = leaderboardVal.description;
+  if (typeof description !== 'string') {
+    return null;
+  }
+
+  const columns = leaderboardVal.columns;
+  if (!Array.isArray(columns)) {
+    return null;
+  }
+
+  const finalColumns = columns
+    .map(column => {
+      const evaluationObjectRef = column.evaluation_object_ref;
+      if (typeof evaluationObjectRef !== 'string') {
+        return null;
+      }
+
+      const scorerName = column.scorer_name;
+      if (typeof scorerName !== 'string') {
+        return null;
+      }
+
+      const shouldMinimize = column.should_minimize;
+      if (shouldMinimize != null && typeof shouldMinimize !== 'boolean') {
+        return null;
+      }
+
+      const summaryMetricParts = column.summary_metric_path_parts;
+      if (!Array.isArray(summaryMetricParts)) {
+        return null;
+      } else if (summaryMetricParts.some(part => typeof part !== 'string')) {
+        return null;
+      }
+
+      return {
+        evaluation_object_ref: evaluationObjectRef,
+        scorer_name: scorerName,
+        should_minimize: shouldMinimize,
+        summary_metric_path_parts: summaryMetricParts,
+      };
+    })
+    .filter(column => column != null) as LeaderboardVal['columns'];
+
+  return {
+    name,
+    description,
+    columns: finalColumns,
+  };
+};
+
+/**
+ * Note: this is a breidge between the python type and the js type.
+ */
+const convertLeaderboardValToFilterAndGroupSpec = (
+  leaderboardVal: LeaderboardVal
+): FilterAndGroupSpec => {
+  const allEvaluations = new Set<string>();
+  leaderboardVal.columns.forEach(column => {
+    allEvaluations.add(column.evaluation_object_ref);
+  });
+  const sourceEvaluations = Array.from(allEvaluations)
+    .map(evaluation => {
+      const evalRef = parseRefMaybe(evaluation);
+      if (evalRef == null) {
+        return null;
+      }
+      return {
+        name: evalRef.artifactName,
+        version: evalRef.artifactVersion,
+      };
+    })
+    .filter(
+      evaluation => evaluation != null
+    ) as FilterAndGroupSourceEvaluationSpec[];
+
+  const scorers = leaderboardVal.columns.map(column => {
+    return {
+      name: column.scorer_name,
+      version: '*',
+      metrics: [
+        {
+          path: column.summary_metric_path_parts.join('.'),
+          shouldMinimize: column.should_minimize,
+        },
+      ],
+    };
+  });
+  return {
+    sourceEvaluations,
+    datasets: [
+      {
+        name: '*',
+        version: '*',
+        scorers,
+      },
+    ],
+  };
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index f4df31ca113..04fb8b834ed 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -502,3 +502,10 @@ const defaultGetSortComparator =
     }
     return aValue.localeCompare(bValue);
   };
+
+/**
+TODO:
+* [ ] Support shouldMinimize
+* [ ] When models are Ops, their link is not quite right
+* [ ] There is a not a perfect mapping of the data model (we over select with the current js impl.)
+ */
diff --git a/weave/flow/leaderboard.py b/weave/flow/leaderboard.py
index 827c9c83475..6c67c0fbf16 100644
--- a/weave/flow/leaderboard.py
+++ b/weave/flow/leaderboard.py
@@ -15,39 +15,52 @@ class LeaderboardColumn(BaseModel):
     summary_metric_path_parts: list[str] = field(default_factory=list)
     should_minimize: Optional[bool] = None
 
+
 class Leaderboard(weave.Object):
     columns: list[LeaderboardColumn]
 
+
 class LeaderboardModelEvaluationResult(BaseModel):
     evaluate_call_ref: str
     value: Any
 
+
 class ModelScoresForColumn(BaseModel):
     scores: list[LeaderboardModelEvaluationResult]
 
+
 class LeaderboardModelResult(BaseModel):
     model_ref: str
     column_scores: list[ModelScoresForColumn]
 
 
-def get_leaderboard_results(spec: Leaderboard, client: WeaveClient) -> list[LeaderboardModelResult]:
+def get_leaderboard_results(
+    spec: Leaderboard, client: WeaveClient
+) -> list[LeaderboardModelResult]:
     entity, project = client._project_id().split("/")
-    calls = client.get_calls(filter=CallsFilter(
-        op_names=[OpRef(
-            entity=entity,
-            project=project,
-            name="Evaluation.evaluate",
-            _digest="*"
-        ).uri()],
-        input_refs=[c.evaluation_object_ref for c  in spec.columns]
-    ))
-    
+    calls = client.get_calls(
+        filter=CallsFilter(
+            op_names=[
+                OpRef(
+                    entity=entity,
+                    project=project,
+                    name="Evaluation.evaluate",
+                    _digest="*",
+                ).uri()
+            ],
+            input_refs=[c.evaluation_object_ref for c in spec.columns],
+        )
+    )
+
     res_map: dict[str, LeaderboardModelResult] = {}
     for call in calls:
         # Frustrating that we have to get the ref like this (such a waste of network calls)
         model_ref = get_ref(call.inputs["model"]).uri()
         if model_ref not in res_map:
-            res_map[model_ref] = LeaderboardModelResult(model_ref=model_ref, column_scores=[ModelScoresForColumn(scores=[]) for _ in spec.columns])
+            res_map[model_ref] = LeaderboardModelResult(
+                model_ref=model_ref,
+                column_scores=[ModelScoresForColumn(scores=[]) for _ in spec.columns],
+            )
         for col_idx, c in enumerate(spec.columns):
             eval_obj_ref = get_ref(call.inputs["self"]).uri()
             if c.evaluation_object_ref != eval_obj_ref:
@@ -60,8 +73,9 @@ def get_leaderboard_results(spec: Leaderboard, client: WeaveClient) -> list[Lead
                     val = val[int(part)]
                 else:
                     break
-            res_map[model_ref].column_scores[col_idx].scores.append(LeaderboardModelEvaluationResult(
-                evaluate_call_ref=get_ref(call).uri(),
-                value=val
-            ))
+            res_map[model_ref].column_scores[col_idx].scores.append(
+                LeaderboardModelEvaluationResult(
+                    evaluate_call_ref=get_ref(call).uri(), value=val
+                )
+            )
     return list(res_map.values())

From 1d67f511616aa79a48b53cbc2e123b1226344ef3 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Fri, 18 Oct 2024 14:01:36 -0700
Subject: [PATCH 080/169] lint

---
 weave/flow/leaderboard.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/weave/flow/leaderboard.py b/weave/flow/leaderboard.py
index 6c67c0fbf16..ff0f513e823 100644
--- a/weave/flow/leaderboard.py
+++ b/weave/flow/leaderboard.py
@@ -55,15 +55,26 @@ def get_leaderboard_results(
     res_map: dict[str, LeaderboardModelResult] = {}
     for call in calls:
         # Frustrating that we have to get the ref like this (such a waste of network calls)
-        model_ref = get_ref(call.inputs["model"]).uri()
-        if model_ref not in res_map:
-            res_map[model_ref] = LeaderboardModelResult(
-                model_ref=model_ref,
+        call_ref = get_ref(call)
+        if call_ref is None:
+            continue
+        call_ref_uri = call_ref.uri()
+
+        model_ref = get_ref(call.inputs["model"])
+        if model_ref is None:
+            continue
+        model_ref_uri = model_ref.uri()
+        if model_ref_uri not in res_map:
+            res_map[model_ref_uri] = LeaderboardModelResult(
+                model_ref=model_ref_uri,
                 column_scores=[ModelScoresForColumn(scores=[]) for _ in spec.columns],
             )
         for col_idx, c in enumerate(spec.columns):
-            eval_obj_ref = get_ref(call.inputs["self"]).uri()
-            if c.evaluation_object_ref != eval_obj_ref:
+            eval_obj_ref = get_ref(call.inputs["self"])
+            if eval_obj_ref is None:
+                continue
+            eval_obj_ref_uri = eval_obj_ref.uri()
+            if c.evaluation_object_ref != eval_obj_ref_uri:
                 continue
             val = call.output.get(c.scorer_name)
             for part in c.summary_metric_path_parts:
@@ -73,9 +84,9 @@ def get_leaderboard_results(
                     val = val[int(part)]
                 else:
                     break
-            res_map[model_ref].column_scores[col_idx].scores.append(
+            res_map[model_ref_uri].column_scores[col_idx].scores.append(
                 LeaderboardModelEvaluationResult(
-                    evaluate_call_ref=get_ref(call).uri(), value=val
+                    evaluate_call_ref=call_ref_uri, value=val
                 )
             )
     return list(res_map.values())

From 5de4b6d5a183f1c07ee3f2adf57faadace1cc257 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Fri, 18 Oct 2024 15:55:14 -0700
Subject: [PATCH 081/169] Ok, big changes, but good changes

---
 tests/trace/test_leaderboard.py               |   1 +
 .../components/FancyPage/useProjectSidebar.ts |   4 +-
 .../LeaderboardListingPage.tsx                |   4 +-
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 170 ++----------------
 .../views/Leaderboard/LeaderboardGrid.tsx     |  26 ++-
 .../views/Leaderboard/query/hookAdapters.ts   |  42 ++++-
 .../Leaderboard/query/leaderboardQuery.ts     | 165 ++++++++++++++++-
 .../types/leaderboardConfigType.ts            |  11 ++
 8 files changed, 257 insertions(+), 166 deletions(-)

diff --git a/tests/trace/test_leaderboard.py b/tests/trace/test_leaderboard.py
index 288ed95b294..25cb17c0d2a 100644
--- a/tests/trace/test_leaderboard.py
+++ b/tests/trace/test_leaderboard.py
@@ -104,6 +104,7 @@ async def test_leaderboard_with_results(client):
             leaderboard.LeaderboardColumn(
                 evaluation_object_ref=get_ref(evaluation_obj_2).uri(),
                 scorer_name="my_scorer",
+                should_minimize=True,
                 summary_metric_path_parts=["true_fraction"],
             ),
             leaderboard.LeaderboardColumn(
diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index 2f8f0d210cc..218e0859081 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -154,7 +154,7 @@ export const useProjectSidebar = (
             type: 'button' as const,
             name: 'Leads',
             slug: 'weave/leaderboards',
-            isShown: showWeaveSidebarItems || isShowAll,
+            isShown: isWeaveOnly,
             iconName: IconNames.BenchmarkSquare,
           },
           {
@@ -204,7 +204,7 @@ export const useProjectSidebar = (
             key: 'moreWeave',
             isShown: isShowAll,
             // iconName: IconNames.OverflowHorizontal,
-            menu: ['weave/operations', 'weave/objects'],
+            menu: ['weave/leaderboards', 'weave/operations', 'weave/objects'],
           },
         ];
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index f4cbbd194b7..6600a673d09 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -14,6 +14,8 @@ import {ObjectVersionsTable} from '../ObjectVersionsPage';
 import {useWFHooks} from '../wfReactInterface/context';
 import {ObjectVersionSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 
+const SHOW_LEADERBOARD_OBJECTS = false;
+
 const Container = styled.div`
   width: 100%;
   height: 100%;
@@ -34,7 +36,7 @@ export const LeaderboardListingPage: React.FC<{
           content: <LeaderboardListingPageInner {...props} />,
         },
       ]}
-      headerExtra={<CreateLeaderboardButton />}
+      headerExtra={SHOW_LEADERBOARD_OBJECTS && <CreateLeaderboardButton />}
     />
   );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 566bc46ca56..d7b9e67a940 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -2,21 +2,15 @@ import {Box} from '@mui/material';
 import {Button} from '@wandb/weave/components/Button';
 import {ErrorPanel} from '@wandb/weave/components/ErrorPanel';
 import {Loading} from '@wandb/weave/components/Loading';
-import React, {useCallback, useEffect, useMemo, useState} from 'react';
+import React, {useEffect, useState} from 'react';
 
-import {parseRefMaybe} from '../../../Browse2/SmallRef';
 import {NotFoundPanel} from '../../NotFoundPanel';
 import {LeaderboardGrid} from '../../views/Leaderboard/LeaderboardGrid';
-import {useLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
-import {
-  FilterAndGroupSourceEvaluationSpec,
-  FilterAndGroupSpec,
-  LeaderboardConfigType,
-} from '../../views/Leaderboard/types/leaderboardConfigType';
+import {usePythonLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
+import {PythonLeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {SimplePageLayout} from '../common/SimplePageLayout';
 import {useWFHooks} from '../wfReactInterface/context';
-import {EditableMarkdown, StyledReactMarkdown} from './EditableMarkdown';
-import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
+import {StyledReactMarkdown} from './EditableMarkdown';
 
 type LeaderboardPageProps = {
   entity: string;
@@ -40,31 +34,6 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
   );
 };
 
-const DEFAULT_DESCRIPTION = `# Leaderboard`;
-
-const usePersistedLeaderboardConfig = () => {
-  const [configPersisted, setConfigPersisted] = useState<LeaderboardConfigType>(
-    {
-      version: 1,
-      config: {description: '', dataSelectionSpec: {}},
-    }
-  );
-
-  const [config, setConfigLocal] =
-    useState<LeaderboardConfigType>(configPersisted);
-
-  const persistConfig = useCallback(() => {
-    setConfigPersisted(config);
-    // persistLeaderboardConfig(config);
-  }, [config]);
-
-  const cancelChanges = useCallback(() => {
-    setConfigLocal(configPersisted);
-  }, [configPersisted]);
-
-  return {config, setConfigLocal, persistConfig, cancelChanges};
-};
-
 export const LeaderboardPageContent: React.FC<
   LeaderboardPageProps & {setName: (name: string) => void}
 > = props => {
@@ -114,38 +83,19 @@ export const LeaderboardPageContent: React.FC<
 
 export const LeaderboardPageContentInner: React.FC<
   LeaderboardPageProps & {setName: (name: string) => void} & {
-    leaderboardVal: LeaderboardVal;
+    leaderboardVal: PythonLeaderboardObjectVal;
   }
 > = props => {
   useEffect(() => {
     props.setName(props.leaderboardVal.name);
   }, [props]);
 
-  // const [showConfig, setShowConfig] = useState(false);
-
-  // const {
-  //   config: currentConfig,
-  //   setConfigLocal,
-  //   persistConfig,
-  //   cancelChanges,
-  // } = usePersistedLeaderboardConfig();
-  // const setDescription = useCallback(
-  //   (newDescription: string) => {
-  //     setConfigLocal(newConfig => ({
-  //       ...newConfig,
-  //       config: {...newConfig.config, description: newDescription},
-  //     }));
-  //     persistConfig();
-  //   },
-  //   [setConfigLocal, persistConfig]
-  // );
-
-  const spec = useMemo(() => {
-    return convertLeaderboardValToFilterAndGroupSpec(props.leaderboardVal);
-  }, [props.leaderboardVal]);
-
   const description = props.leaderboardVal.description;
-  const {loading, data} = useLeaderboardData(props.entity, props.project, spec);
+  const {loading, data} = usePythonLeaderboardData(
+    props.entity,
+    props.project,
+    props.leaderboardVal
+  );
 
   return (
     <Box display="flex" flexDirection="row" height="100%" flexGrow={1}>
@@ -178,23 +128,7 @@ export const LeaderboardPageContentInner: React.FC<
             {description && (
               <StyledReactMarkdown>{description}</StyledReactMarkdown>
             )}
-            {/* <EditableMarkdown
-              value={props.leaderboardName + '\n' + description}
-              onChange={setDescription}
-              placeholder={DEFAULT_DESCRIPTION}
-            /> */}
           </Box>
-          {/* <div
-            style={{
-              display: showConfig ? 'none' : 'block',
-              // paddingRight: '12px',
-              // paddingTop: '12px',
-            }}>
-            <ToggleLeaderboardConfig
-              isOpen={showConfig}
-              onClick={() => setShowConfig(c => !c)}
-            />
-          </div> */}
         </Box>
         <Box flexGrow={1} display="flex" flexDirection="row" overflow="hidden">
           <LeaderboardGrid
@@ -205,24 +139,6 @@ export const LeaderboardPageContentInner: React.FC<
           />
         </Box>
       </Box>
-      {/* {showConfig && (
-        <Box flex={1} width="35%" height="100%" overflow="hidden">
-          <LeaderboardConfigEditor
-            entity={entity}
-            project={project}
-            config={currentConfig}
-            onCancel={() => {
-              cancelChanges();
-              setShowConfig(false);
-            }}
-            onPersist={() => {
-              persistConfig();
-              setShowConfig(false);
-            }}
-            setConfig={setConfigLocal}
-          />
-        </Box>
-      )} */}
     </Box>
   );
 };
@@ -249,18 +165,9 @@ export const ToggleLeaderboardConfig: React.FC<{
   );
 };
 
-type LeaderboardVal = {
-  name: string;
-  description: string;
-  columns: Array<{
-    evaluation_object_ref: string;
-    scorer_name: string;
-    should_minimize?: boolean;
-    summary_metric_path_parts: string[];
-  }>;
-};
-
-const parseLeaderboardVal = (leaderboardVal: any): LeaderboardVal | null => {
+const parseLeaderboardVal = (
+  leaderboardVal: any
+): PythonLeaderboardObjectVal | null => {
   if (typeof leaderboardVal !== 'object' || leaderboardVal == null) {
     return null;
   }
@@ -311,7 +218,7 @@ const parseLeaderboardVal = (leaderboardVal: any): LeaderboardVal | null => {
         summary_metric_path_parts: summaryMetricParts,
       };
     })
-    .filter(column => column != null) as LeaderboardVal['columns'];
+    .filter(column => column != null) as PythonLeaderboardObjectVal['columns'];
 
   return {
     name,
@@ -319,52 +226,3 @@ const parseLeaderboardVal = (leaderboardVal: any): LeaderboardVal | null => {
     columns: finalColumns,
   };
 };
-
-/**
- * Note: this is a breidge between the python type and the js type.
- */
-const convertLeaderboardValToFilterAndGroupSpec = (
-  leaderboardVal: LeaderboardVal
-): FilterAndGroupSpec => {
-  const allEvaluations = new Set<string>();
-  leaderboardVal.columns.forEach(column => {
-    allEvaluations.add(column.evaluation_object_ref);
-  });
-  const sourceEvaluations = Array.from(allEvaluations)
-    .map(evaluation => {
-      const evalRef = parseRefMaybe(evaluation);
-      if (evalRef == null) {
-        return null;
-      }
-      return {
-        name: evalRef.artifactName,
-        version: evalRef.artifactVersion,
-      };
-    })
-    .filter(
-      evaluation => evaluation != null
-    ) as FilterAndGroupSourceEvaluationSpec[];
-
-  const scorers = leaderboardVal.columns.map(column => {
-    return {
-      name: column.scorer_name,
-      version: '*',
-      metrics: [
-        {
-          path: column.summary_metric_path_parts.join('.'),
-          shouldMinimize: column.should_minimize,
-        },
-      ],
-    };
-  });
-  return {
-    sourceEvaluations,
-    datasets: [
-      {
-        name: '*',
-        version: '*',
-        scorers,
-      },
-    ],
-  };
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index 04fb8b834ed..8e032736b63 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -73,7 +73,10 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
       if (['Trials', 'Run Date'].includes(metricPathGroup)) {
         return 'transparent';
       }
-      const shouldMinimize = ['Avg. Latency'].includes(metricPathGroup);
+      const shouldMinimize =
+        ['Avg. Latency'].includes(metricPathGroup) ||
+        columnStats.datasetGroups[datasetGroup].scorerGroups[scorerGroup]
+          .metricPathGroups[metricPathGroup].shouldMinimize;
       if (score == null) {
         return 'transparent';
       }
@@ -111,8 +114,23 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         minWidth: 150,
         flex: 1,
         renderCell: (params: GridRenderCellParams) => {
+          let isOp = false;
+          try {
+            isOp =
+              Object.values(
+                Object.values(
+                  Object.values(
+                    (params.row as RowData).modelGroup.datasetGroups
+                  )[0].scorerGroups
+                )[0].metricPathGroups
+              )[0][0].modelType === 'op';
+          } catch (e) {
+            console.log(e);
+          }
           const modelRef = parseRefMaybe(
-            `weave:///${entity}/${project}/object/${params.value}` ?? ''
+            `weave:///${entity}/${project}/${isOp ? 'op' : 'object'}/${
+              params.value
+            }` ?? ''
           );
           if (modelRef) {
             return (
@@ -350,6 +368,7 @@ type ColumnStats = {
               min: number;
               max: number;
               count: number;
+              shouldMinimize: boolean;
             };
           };
         };
@@ -395,6 +414,7 @@ const getColumnStats = (data: GroupedLeaderboardData): ColumnStats => {
                     min: metricValue,
                     max: metricValue,
                     count: metricPathGroup.length,
+                    shouldMinimize: metricPathGroup[0].shouldMinimize ?? false,
                   };
                 } else {
                   currScorerGroup.metricPathGroups[metricPathGroupName].min =
@@ -505,7 +525,5 @@ const defaultGetSortComparator =
 
 /**
 TODO:
-* [ ] Support shouldMinimize
 * [ ] When models are Ops, their link is not quite right
-* [ ] There is a not a perfect mapping of the data model (we over select with the current js impl.)
  */
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
index 6d2b45cb8b4..37aaf6daf9c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
@@ -2,13 +2,21 @@ import {useDeepMemo} from '@wandb/weave/hookUtils';
 import {useEffect, useState} from 'react';
 
 import {useGetTraceServerClientContext} from '../../../pages/wfReactInterface/traceServerClientContext';
-import {FilterAndGroupSpec} from '../types/leaderboardConfigType';
-import {getLeaderboardData, GroupedLeaderboardData} from './leaderboardQuery';
+import {
+  FilterAndGroupSpec,
+  PythonLeaderboardObjectVal,
+} from '../types/leaderboardConfigType';
+import {
+  getLeaderboardData,
+  getPythonLeaderboardData,
+  GroupedLeaderboardData,
+} from './leaderboardQuery';
 
 type LeaderboardDataState = {
   loading: boolean;
   data: GroupedLeaderboardData;
 };
+
 export const useLeaderboardData = (
   entity: string,
   project: string,
@@ -35,3 +43,33 @@ export const useLeaderboardData = (
   }, [entity, project, getTraceServerClient, deepSpec]);
   return state;
 };
+
+export const usePythonLeaderboardData = (
+  entity: string,
+  project: string,
+  val: PythonLeaderboardObjectVal
+): LeaderboardDataState => {
+  const getTraceServerClient = useGetTraceServerClientContext();
+  const [state, setState] = useState<LeaderboardDataState>({
+    loading: true,
+    data: {modelGroups: {}},
+  });
+  const deepVal = useDeepMemo(val);
+  useEffect(() => {
+    let mounted = true;
+    getPythonLeaderboardData(
+      getTraceServerClient(),
+      entity,
+      project,
+      deepVal
+    ).then(data => {
+      if (mounted) {
+        setState({loading: false, data});
+      }
+    });
+    return () => {
+      mounted = false;
+    };
+  }, [entity, project, getTraceServerClient, deepVal]);
+  return state;
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
index cb217274132..7c46d1ea98a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
@@ -14,7 +14,11 @@ import {
   objectVersionKeyToRefUri,
   opVersionKeyToRefUri,
 } from '../../../pages/wfReactInterface/utilities';
-import {ALL_VALUE, FilterAndGroupSpec} from '../types/leaderboardConfigType';
+import {
+  ALL_VALUE,
+  FilterAndGroupSpec,
+  PythonLeaderboardObjectVal,
+} from '../types/leaderboardConfigType';
 
 export type LeaderboardValueRecord = {
   datasetName: string;
@@ -38,6 +42,7 @@ export type LeaderboardValueRecord = {
   createdAt: Date;
   sourceEvaluationCallId: string;
   sourceEvaluationObjectRef: string;
+  shouldMinimize?: boolean; // hack to put this here
 };
 
 export type GroupableLeaderboardValueRecord = {
@@ -437,6 +442,7 @@ export const getLeaderboardGroupableData = async (
     .filter(entry => entry.include)
     .map(entry => entry.groupableRow);
 };
+
 export const getLeaderboardData = async (
   client: TraceServerClient,
   entity: string,
@@ -481,3 +487,160 @@ export const getLeaderboardData = async (
 
   return finalData;
 };
+
+export const getPythonLeaderboardData = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
+  val: PythonLeaderboardObjectVal
+): Promise<GroupedLeaderboardData> => {
+  const groupableData = await getPythonLeaderboardGroupableData(
+    client,
+    entity,
+    project,
+    val
+  );
+
+  const finalData: GroupedLeaderboardData = {
+    modelGroups: _.mapValues(
+      _.groupBy(groupableData, 'modelGroup'),
+      modelGroup => {
+        return {
+          datasetGroups: _.mapValues(
+            _.groupBy(modelGroup, 'datasetGroup'),
+            datasetGroup => {
+              return {
+                scorerGroups: _.mapValues(
+                  _.groupBy(datasetGroup, 'scorerGroup'),
+                  scorerGroup => {
+                    return {
+                      metricPathGroups: _.mapValues(
+                        _.groupBy(scorerGroup, 'metricPathGroup'),
+                        metricPathGroup => {
+                          return metricPathGroup.map(row => row.row);
+                        }
+                      ),
+                    };
+                  }
+                ),
+              };
+            }
+          ),
+        };
+      }
+    ),
+  };
+
+  return finalData;
+};
+
+const getPythonLeaderboardGroupableData = async (
+  client: TraceServerClient,
+  entity: string,
+  project: string,
+  val: PythonLeaderboardObjectVal
+): Promise<GroupableLeaderboardValueRecord[]> => {
+  const evalObjectRefs = _.uniq(
+    val.columns.map(col => col.evaluation_object_ref)
+  );
+
+  const evalObjectRefsValsProm = client.readBatch({refs: evalObjectRefs});
+
+  const allEvaluationCallsProm = client.callsStreamQuery({
+    project_id: projectIdFromParts({entity, project}),
+    sort_by: [{field: 'ended_at', direction: 'desc'}],
+    filter: {
+      op_names: [
+        opVersionKeyToRefUri({
+          entity,
+          project,
+          opId: EVALUATE_OP_NAME_POST_PYDANTIC,
+          versionHash: ALL_VALUE,
+        }),
+      ],
+      input_refs: evalObjectRefs,
+    },
+  });
+
+  const evalObjectRefsVals = await evalObjectRefsValsProm;
+  const evalMap = _.zipObject(evalObjectRefs, evalObjectRefsVals.vals);
+
+  const allEvaluationCallsRes = await allEvaluationCallsProm;
+
+  const data: GroupableLeaderboardValueRecord[] = [];
+  allEvaluationCallsRes.calls.forEach(call => {
+    val.columns.forEach(col => {
+      const evalObjRefUri = call.inputs.self;
+      if (col.evaluation_object_ref === evalObjRefUri) {
+        const evalVal = evalMap[evalObjRefUri];
+        if (evalVal == null) {
+          return;
+        }
+        const modelRefUri = call.inputs.model ?? '';
+        const modelRef = parseRefMaybe(modelRefUri);
+        const datasetRefUri = evalVal.dataset ?? '';
+        const datasetRef = parseRefMaybe(datasetRefUri);
+        if (modelRef?.scheme !== 'weave' || datasetRef?.scheme !== 'weave') {
+          return;
+        }
+        const scorerRefUri = evalVal.scorers.find(
+          (scorer: string) =>
+            parseRefMaybe(scorer ?? '')?.artifactName === col.scorer_name
+        );
+        const scorerRef = parseRefMaybe(scorerRefUri ?? '');
+        if (scorerRef?.scheme !== 'weave') {
+          return;
+        }
+        let value = call.output;
+        if (typeof value !== 'object' || value == null) {
+          value = null;
+        } else {
+          value = (value as any)[col.scorer_name];
+        }
+        col.summary_metric_path_parts.forEach(part => {
+          if (value == null) {
+            return;
+          }
+          if (_.isArray(value)) {
+            try {
+              const index = parseInt(part, 10);
+              value = value[index];
+            } catch (e) {
+              console.warn('Skipping model latency', call, e);
+              value = null;
+            }
+          } else {
+            value = (value as any)[part];
+          }
+        });
+        const row: GroupableLeaderboardValueRecord = {
+          modelGroup: `${modelRef.artifactName}:${modelRef.artifactVersion}`,
+          datasetGroup: `${datasetRef.artifactName}:${datasetRef.artifactVersion}`,
+          scorerGroup: `${scorerRef.artifactName}:${scorerRef.artifactVersion}`,
+          metricPathGroup: col.summary_metric_path_parts.join('.'),
+          sortKey: -convertISOToDate(call.started_at).getTime(),
+          row: {
+            datasetName: datasetRef.artifactName,
+            datasetVersion: datasetRef.artifactVersion,
+            metricType: 'scorerMetric',
+            scorerName: scorerRef.artifactName,
+            scorerVersion: scorerRef.artifactVersion,
+            metricPath: col.summary_metric_path_parts.join('.'),
+            metricValue: value as any,
+            modelName: modelRef.artifactName,
+            modelVersion: modelRef.artifactVersion,
+            modelType: modelRef.weaveKind === 'op' ? 'op' : 'object',
+            trials: evalVal.trials,
+            createdAt: convertISOToDate(call.started_at),
+            sourceEvaluationCallId: call.id,
+            sourceEvaluationObjectRef: col.evaluation_object_ref,
+            shouldMinimize: col.should_minimize,
+          },
+        };
+        data.push(row);
+      }
+    });
+  });
+
+  return data;
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts
index bb4587966c9..4b322733fc5 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts
@@ -43,3 +43,14 @@ export type FilterAndGroupModelSpec = {
   version: string; // "*" means all
   groupAllVersions?: boolean;
 };
+
+export type PythonLeaderboardObjectVal = {
+  name: string;
+  description: string;
+  columns: Array<{
+    evaluation_object_ref: string;
+    scorer_name: string;
+    should_minimize?: boolean;
+    summary_metric_path_parts: string[];
+  }>;
+};

From a83c44078632a2f6a07548eb71ad0497fbf93052 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Fri, 18 Oct 2024 16:07:00 -0700
Subject: [PATCH 082/169] basically done - except for editting

---
 .../components/FancyPage/useProjectSidebar.ts |  2 +-
 .../LeaderboardPage/EditableMarkdown.tsx      | 27 ++++++++++++++-----
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 27 +++++++++----------
 3 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index 218e0859081..5ccb980712b 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -152,7 +152,7 @@ export const useProjectSidebar = (
           },
           {
             type: 'button' as const,
-            name: 'Leads',
+            name: 'Leaders',
             slug: 'weave/leaderboards',
             isShown: isWeaveOnly,
             iconName: IconNames.BenchmarkSquare,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
index e9b5f70c7e4..bb55ca863e2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
@@ -19,14 +19,29 @@ export const StyledReactMarkdown = styled(ReactMarkdown)`
   > *:first-child {
     margin-top: 0;
   }
-  h1,
-  h2,
-  h3,
-  h4,
-  h5,
+  h1 {
+    font-weight: 600;
+    font-size: 1.2rem;
+  }
+  h2 {
+    font-weight: 600;
+    font-size: 1.15rem;
+  }
+  h3 {
+    font-weight: 600;
+    font-size: 1.1rem;
+  }
+  h4 {
+    font-weight: 600;
+    font-size: 1.05rem;
+  }
+  h5 {
+    font-weight: 600;
+    font-size: 1rem;
+  }
   h6 {
     font-weight: 600;
-    font-size: 1.25rem;
+    font-size: 1rem;
   }
 `;
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index d7b9e67a940..7c05c63159a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -106,31 +106,28 @@ export const LeaderboardPageContentInner: React.FC<
         height="100%"
         minWidth="50%">
         <Box
-          flex={1}
           display="flex"
           flexDirection="row"
           maxHeight="35%"
           width="100%"
           sx={{
+            flex: '1 1 auto',
             alignItems: 'flex-start',
             padding: '12px 16px',
             gap: '12px',
+            overflowY: 'auto',
           }}>
-          <Box
-            flexShrink={0}
-            flexGrow={1}
-            sx={{
-              overflow: 'auto',
-              height: '100%',
-              display: 'flex',
-              alignItems: 'flex-start',
-            }}>
-            {description && (
-              <StyledReactMarkdown>{description}</StyledReactMarkdown>
-            )}
-          </Box>
+          {description && (
+            <StyledReactMarkdown>{description}</StyledReactMarkdown>
+          )}
         </Box>
-        <Box flexGrow={1} display="flex" flexDirection="row" overflow="hidden">
+        <Box
+          display="flex"
+          flexDirection="row"
+          overflow="hidden"
+          sx={{
+            flex: '1 1 auto',
+          }}>
           <LeaderboardGrid
             entity={props.entity}
             project={props.project}

From 2294f662ad3c599caafe7329cfc00d73d777d080 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Fri, 18 Oct 2024 16:22:23 -0700
Subject: [PATCH 083/169] Done checkpoint

---
 tests/trace/test_leaderboard.py               | 60 +++++++++++++++++--
 .../components/FancyPage/useProjectSidebar.ts |  6 +-
 .../Leaderboard/query/leaderboardQuery.ts     |  6 +-
 3 files changed, 64 insertions(+), 8 deletions(-)

diff --git a/tests/trace/test_leaderboard.py b/tests/trace/test_leaderboard.py
index 25cb17c0d2a..3ac7ab17f9f 100644
--- a/tests/trace/test_leaderboard.py
+++ b/tests/trace/test_leaderboard.py
@@ -6,9 +6,39 @@
 
 
 def test_leaderboard_empty(client):
+    evaluation_obj_1 = weave.Evaluation(
+        name="test_evaluation_name",
+        dataset=[{"input": -1, "target": -1}],
+        scorers=[],
+    )
+
+    weave.publish(evaluation_obj_1)
+
     spec = leaderboard.Leaderboard(
-        name="Test Leaderboard",
-        description="This is a test leaderboard",
+        name="Empty Leaderboard",
+        description="""This is an empty leaderboard""",
+        columns=[
+            leaderboard.LeaderboardColumn(
+                evaluation_object_ref=get_ref(evaluation_obj_1).uri(),
+                scorer_name="test_scorer_name",
+                summary_metric_path="test_summary_metric_path",
+            )
+        ],
+    )
+
+    ref = weave.publish(spec)
+
+    # TODO: this is not working
+    # spec = ref.get()
+
+    results = leaderboard.get_leaderboard_results(spec, client)
+    assert len(results) == 0
+
+
+def test_leaderboard_mis_configured(client):
+    spec = leaderboard.Leaderboard(
+        name="Misconfigured Leaderboard",
+        description="""This is a misconfigured leaderboard""",
         columns=[
             leaderboard.LeaderboardColumn(
                 evaluation_object_ref="test_evaluation_object_ref",
@@ -18,6 +48,11 @@ def test_leaderboard_empty(client):
         ],
     )
 
+    ref = weave.publish(spec)
+
+    # TODO: this is not working
+    # spec = ref.get()
+
     results = leaderboard.get_leaderboard_results(spec, client)
     assert len(results) == 0
 
@@ -72,7 +107,7 @@ async def test_leaderboard_with_results(client):
 
     spec = leaderboard.Leaderboard(
         name="Simple Leaderboard",
-        description="This is a test leaderboard",
+        description="""This is a simple leaderboard""",
         columns=[
             leaderboard.LeaderboardColumn(
                 evaluation_object_ref=get_ref(evaluation_obj_1).uri(),
@@ -94,7 +129,24 @@ async def test_leaderboard_with_results(client):
 
     spec = leaderboard.Leaderboard(
         name="Complex Leaderboard",
-        description="This is a test leaderboard",
+        description="""
+This leaderboard has multiple columns
+
+### Columns
+
+1. Column 1:
+    - Evaluation Object: test_evaluation_object_ref
+    - Scorer Name: test_scorer_name
+    - Summary Metric Path: test_summary_metric_path
+2. Column 2:
+    - Evaluation Object: test_evaluation_object_ref
+    - Scorer Name: test_scorer_name
+    - Summary Metric Path: test_summary_metric_path
+3. Column 3:
+    - Evaluation Object: test_evaluation_object_ref
+    - Scorer Name: test_scorer_name
+    - Summary Metric Path: test_summary_metric_path
+""",
         columns=[
             leaderboard.LeaderboardColumn(
                 evaluation_object_ref=get_ref(evaluation_obj_2).uri(),
diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts
index 5ccb980712b..12499ba668c 100644
--- a/weave-js/src/components/FancyPage/useProjectSidebar.ts
+++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts
@@ -140,7 +140,7 @@ export const useProjectSidebar = (
           },
           {
             type: 'divider' as const,
-            key: 'dividerWithinWeave',
+            key: 'dividerWithinWeave-1',
             isShown: isWeaveOnly,
           },
           {
@@ -159,7 +159,7 @@ export const useProjectSidebar = (
           },
           {
             type: 'divider' as const,
-            key: 'dividerWithinWeave',
+            key: 'dividerWithinWeave-2',
             isShown: isWeaveOnly,
           },
           {
@@ -178,7 +178,7 @@ export const useProjectSidebar = (
           },
           {
             type: 'divider' as const,
-            key: 'dividerWithinWeave',
+            key: 'dividerWithinWeave-3',
             isShown: isWeaveOnly,
           },
           {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
index 7c46d1ea98a..ace9958c25c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
@@ -542,7 +542,11 @@ const getPythonLeaderboardGroupableData = async (
 ): Promise<GroupableLeaderboardValueRecord[]> => {
   const evalObjectRefs = _.uniq(
     val.columns.map(col => col.evaluation_object_ref)
-  );
+  ).filter(ref => parseRefMaybe(ref)?.scheme === 'weave');
+
+  if (evalObjectRefs.length === 0) {
+    return [];
+  }
 
   const evalObjectRefsValsProm = client.readBatch({refs: evalObjectRefs});
 

From 14fe6650a9d76c7fdd3667ed239dfa1c85e32cfa Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Sun, 20 Oct 2024 18:34:10 -0700
Subject: [PATCH 084/169] buttons added

---
 .../LeaderboardListingPage.tsx                |  6 ++--
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 29 ++++++++++++++++++-
 2 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index 6600a673d09..de42cf9bfd8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -14,8 +14,6 @@ import {ObjectVersionsTable} from '../ObjectVersionsPage';
 import {useWFHooks} from '../wfReactInterface/context';
 import {ObjectVersionSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 
-const SHOW_LEADERBOARD_OBJECTS = false;
-
 const Container = styled.div`
   width: 100%;
   height: 100%;
@@ -36,7 +34,7 @@ export const LeaderboardListingPage: React.FC<{
           content: <LeaderboardListingPageInner {...props} />,
         },
       ]}
-      headerExtra={SHOW_LEADERBOARD_OBJECTS && <CreateLeaderboardButton />}
+      headerExtra={<CreateLeaderboardButton />}
     />
   );
 };
@@ -67,7 +65,7 @@ const CreateLeaderboardButton: FC = () => {
         }}
         size="medium"
         variant="secondary"
-        onClick={console.log}
+        onClick={() => console.log('create leaderboard')}
         icon="add-new">
         Create Leaderboard
       </Button>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 7c05c63159a..ff286b5ce72 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -2,8 +2,9 @@ import {Box} from '@mui/material';
 import {Button} from '@wandb/weave/components/Button';
 import {ErrorPanel} from '@wandb/weave/components/ErrorPanel';
 import {Loading} from '@wandb/weave/components/Loading';
-import React, {useEffect, useState} from 'react';
+import React, {FC, useContext, useEffect, useState} from 'react';
 
+import { WeaveflowPeekContext } from '../../context';
 import {NotFoundPanel} from '../../NotFoundPanel';
 import {LeaderboardGrid} from '../../views/Leaderboard/LeaderboardGrid';
 import {usePythonLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
@@ -20,6 +21,7 @@ type LeaderboardPageProps = {
 
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
   const [name, setName] = useState(props.leaderboardName);
+  const {isPeeking} = useContext(WeaveflowPeekContext);
   return (
     <SimplePageLayout
       title={name}
@@ -30,6 +32,7 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
           content: <LeaderboardPageContent {...props} setName={setName} />,
         },
       ]}
+      headerExtra={!isPeeking && <EditLeaderboardButton />}
     />
   );
 };
@@ -223,3 +226,27 @@ const parseLeaderboardVal = (
     columns: finalColumns,
   };
 };
+
+
+const EditLeaderboardButton: FC = () => {
+  return (
+    <Box
+      sx={{
+        height: '100%',
+        display: 'flex',
+        alignItems: 'center',
+      }}>
+      <Button
+        className="mx-16"
+        style={{
+          marginLeft: '0px',
+        }}
+        size="medium"
+        variant="secondary"
+        onClick={() => console.log('edit leaderboard')}
+        icon='pencil-edit'>
+        Edit
+      </Button>
+    </Box>
+  );
+};

From d607390af7b2fe1b4f8c7cc4f342bb962a4f59c0 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Sun, 20 Oct 2024 18:48:26 -0700
Subject: [PATCH 085/169] updated the client

---
 .../pages/wfReactInterface/traceServerClientTypes.ts | 12 ++++++++++++
 .../wfReactInterface/traceServerDirectClient.ts      |  6 ++++++
 2 files changed, 18 insertions(+)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
index 5ccc3f1d10b..1f5e93edb0b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
@@ -178,6 +178,18 @@ export type TraceObjReadRes = {
   obj: TraceObjSchema;
 };
 
+export type TraceObjCreateReq = {
+  obj: {
+    project_id: string;
+    object_id: string;
+    val: any;
+  }
+}
+
+export type TraceObjCreateRes = {
+  digest: string;
+};
+
 export type TraceRefsReadBatchReq = {
   refs: string[];
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
index caaf63b7f56..c8428efa8e8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
@@ -34,6 +34,8 @@ import {
   TraceCallUpdateReq,
   TraceFileContentReadReq,
   TraceFileContentReadRes,
+  TraceObjCreateReq,
+  TraceObjCreateRes,
   TraceObjQueryReq,
   TraceObjQueryRes,
   TraceObjReadReq,
@@ -224,6 +226,10 @@ export class DirectTraceServerClient {
     return this.makeRequest<TraceObjReadReq, TraceObjReadRes>('/obj/read', req);
   }
 
+  public objCreate(req: TraceObjCreateReq): Promise<TraceObjCreateRes> {
+    return this.makeRequest<TraceObjCreateReq, TraceObjCreateRes>('/obj/create', req);
+  }
+
   public readBatch(req: TraceRefsReadBatchReq): Promise<TraceRefsReadBatchRes> {
     return this.makeRequest<TraceRefsReadBatchReq, TraceRefsReadBatchRes>(
       '/refs/read_batch',

From 3d1e9bd3eb1982f5262ed3bb7358d6b4735279a7 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Sun, 20 Oct 2024 18:55:44 -0700
Subject: [PATCH 086/169] initial config + lint

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 142 ++++++++++++++++--
 .../traceServerClientTypes.ts                 |   4 +-
 .../traceServerDirectClient.ts                |   5 +-
 3 files changed, 135 insertions(+), 16 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index ff286b5ce72..cde7c56c08d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,16 +1,19 @@
 import {Box} from '@mui/material';
+import {MOON_250} from '@wandb/weave/common/css/color.styles';
 import {Button} from '@wandb/weave/components/Button';
 import {ErrorPanel} from '@wandb/weave/components/ErrorPanel';
 import {Loading} from '@wandb/weave/components/Loading';
-import React, {FC, useContext, useEffect, useState} from 'react';
+import React, {FC, useCallback, useContext, useEffect, useState} from 'react';
 
-import { WeaveflowPeekContext } from '../../context';
+import {WeaveflowPeekContext} from '../../context';
 import {NotFoundPanel} from '../../NotFoundPanel';
 import {LeaderboardGrid} from '../../views/Leaderboard/LeaderboardGrid';
 import {usePythonLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
 import {PythonLeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {SimplePageLayout} from '../common/SimplePageLayout';
 import {useWFHooks} from '../wfReactInterface/context';
+import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
+import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
 import {StyledReactMarkdown} from './EditableMarkdown';
 
 type LeaderboardPageProps = {
@@ -22,6 +25,7 @@ type LeaderboardPageProps = {
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
   const [name, setName] = useState(props.leaderboardName);
   const {isPeeking} = useContext(WeaveflowPeekContext);
+  const [isEditing, setIsEditing] = useState(false);
   return (
     <SimplePageLayout
       title={name}
@@ -29,16 +33,35 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
       tabs={[
         {
           label: 'Leaderboard',
-          content: <LeaderboardPageContent {...props} setName={setName} />,
+          content: (
+            <LeaderboardPageContent
+              {...props}
+              setName={setName}
+              isEditing={isEditing}
+              setIsEditing={setIsEditing}
+            />
+          ),
         },
       ]}
-      headerExtra={!isPeeking && <EditLeaderboardButton />}
+      headerExtra={
+        !isPeeking &&
+        !isEditing && (
+          <EditLeaderboardButton
+            isEditing={isEditing}
+            setIsEditing={setIsEditing}
+          />
+        )
+      }
     />
   );
 };
 
 export const LeaderboardPageContent: React.FC<
-  LeaderboardPageProps & {setName: (name: string) => void}
+  LeaderboardPageProps & {
+    setName: (name: string) => void;
+    isEditing: boolean;
+    setIsEditing: (isEditing: boolean) => void;
+  }
 > = props => {
   const {useRootObjectVersions} = useWFHooks();
   const {entity, project} = props;
@@ -80,12 +103,43 @@ export const LeaderboardPageContent: React.FC<
   }
 
   return (
-    <LeaderboardPageContentInner {...props} leaderboardVal={leaderboardVal} />
+    <LeaderboardPageContentInner
+      {...props}
+      leaderboardVal={leaderboardVal}
+      setIsEditing={props.setIsEditing}
+    />
   );
 };
 
+const useUpdateLeaderboard = (
+  entity: string,
+  project: string,
+  objectId: string
+) => {
+  const getTsClient = useGetTraceServerClientContext();
+  const client = getTsClient();
+
+  const updateLeaderboard = async (
+    leaderboardVal: PythonLeaderboardObjectVal
+  ) => {
+    return await client.objCreate({
+      obj: {
+        project_id: projectIdFromParts({entity, project}),
+        object_id: objectId,
+        val: leaderboardVal,
+      },
+    });
+  };
+
+  return updateLeaderboard;
+};
+
 export const LeaderboardPageContentInner: React.FC<
-  LeaderboardPageProps & {setName: (name: string) => void} & {
+  LeaderboardPageProps & {
+    setName: (name: string) => void;
+    isEditing: boolean;
+    setIsEditing: (isEditing: boolean) => void;
+  } & {
     leaderboardVal: PythonLeaderboardObjectVal;
   }
 > = props => {
@@ -94,11 +148,42 @@ export const LeaderboardPageContentInner: React.FC<
   }, [props]);
 
   const description = props.leaderboardVal.description;
-  const {loading, data} = usePythonLeaderboardData(
+  const updateLeaderboard = useUpdateLeaderboard(
     props.entity,
     props.project,
+    props.leaderboardName
+  );
+  const [workingLeaderboardValCopy, setWorkingLeaderboardValCopy] = useState(
     props.leaderboardVal
   );
+  const {loading, data} = usePythonLeaderboardData(
+    props.entity,
+    props.project,
+    workingLeaderboardValCopy
+  );
+  const [saving, setSaving] = useState(false);
+  const discardChanges = useCallback(() => {
+    setWorkingLeaderboardValCopy(props.leaderboardVal);
+    props.setIsEditing(false);
+  }, [props]);
+  const commitChanges = useCallback(() => {
+    const mounted = true;
+    setSaving(true);
+    updateLeaderboard(workingLeaderboardValCopy)
+      .then(() => {
+        if (mounted) {
+          props.setIsEditing(false);
+          setSaving(false);
+        }
+      })
+      .catch(e => {
+        console.error(e);
+        if (mounted) {
+          setWorkingLeaderboardValCopy(props.leaderboardVal);
+          setSaving(false);
+        }
+      });
+  }, [props, updateLeaderboard, workingLeaderboardValCopy]);
 
   return (
     <Box display="flex" flexDirection="row" height="100%" flexGrow={1}>
@@ -139,10 +224,39 @@ export const LeaderboardPageContentInner: React.FC<
           />
         </Box>
       </Box>
+      {props.isEditing && (
+        <Box
+          flex={1}
+          display="flex"
+          flexDirection="column"
+          height="100%"
+          minWidth="50%"
+          sx={{
+            borderLeft: `1px solid ${MOON_250}`,
+          }}>
+          <LeaderboardConfigEditor
+            saving={saving}
+            leaderboardVal={props.leaderboardVal}
+            setWorkingCopy={setWorkingLeaderboardValCopy}
+            discardChanges={discardChanges}
+            commitChanges={commitChanges}
+          />
+        </Box>
+      )}
     </Box>
   );
 };
 
+const LeaderboardConfigEditor: React.FC<{
+  leaderboardVal: PythonLeaderboardObjectVal;
+  saving: boolean;
+  setWorkingCopy: (leaderboardVal: PythonLeaderboardObjectVal) => void;
+  discardChanges: () => void;
+  commitChanges: () => void;
+}> = ({leaderboardVal, setWorkingCopy, discardChanges, commitChanges}) => {
+  return <Box>Config Editor</Box>;
+};
+
 export const ToggleLeaderboardConfig: React.FC<{
   isOpen: boolean;
   onClick: () => void;
@@ -227,8 +341,10 @@ const parseLeaderboardVal = (
   };
 };
 
-
-const EditLeaderboardButton: FC = () => {
+const EditLeaderboardButton: FC<{
+  isEditing: boolean;
+  setIsEditing: (isEditing: boolean) => void;
+}> = ({isEditing, setIsEditing}) => {
   return (
     <Box
       sx={{
@@ -243,9 +359,9 @@ const EditLeaderboardButton: FC = () => {
         }}
         size="medium"
         variant="secondary"
-        onClick={() => console.log('edit leaderboard')}
-        icon='pencil-edit'>
-        Edit
+        onClick={() => setIsEditing(!isEditing)}
+        icon={isEditing ? 'close' : 'pencil-edit'}>
+        {isEditing ? 'Discard Changes' : 'Edit'}
       </Button>
     </Box>
   );
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
index 1f5e93edb0b..293b5ad0015 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
@@ -183,8 +183,8 @@ export type TraceObjCreateReq = {
     project_id: string;
     object_id: string;
     val: any;
-  }
-}
+  };
+};
 
 export type TraceObjCreateRes = {
   digest: string;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
index c8428efa8e8..a8e505a895b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
@@ -227,7 +227,10 @@ export class DirectTraceServerClient {
   }
 
   public objCreate(req: TraceObjCreateReq): Promise<TraceObjCreateRes> {
-    return this.makeRequest<TraceObjCreateReq, TraceObjCreateRes>('/obj/create', req);
+    return this.makeRequest<TraceObjCreateReq, TraceObjCreateRes>(
+      '/obj/create',
+      req
+    );
   }
 
   public readBatch(req: TraceRefsReadBatchReq): Promise<TraceRefsReadBatchRes> {

From 5efa8488b89f25263ea2eae76884db1d08c28027 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Sun, 20 Oct 2024 18:58:34 -0700
Subject: [PATCH 087/169] REMOVED OLD CONFIG EFITOR

---
 .../LeaderboardConfigEditor.tsx               | 1173 +----------------
 .../pages/LeaderboardPage/LeaderboardPage.tsx |   11 +-
 2 files changed, 11 insertions(+), 1173 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 8fb55d1de84..c777cc88044 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -1,1167 +1,14 @@
-import DeleteIcon from '@mui/icons-material/Delete';
-import {
-  Box,
-  Button,
-  Checkbox,
-  FormControl,
-  FormControlLabel,
-  InputLabel,
-  MenuItem,
-  Select,
-  SelectChangeEvent,
-  Tab,
-  Tabs,
-  Typography,
-} from '@mui/material';
-import IconButton from '@mui/material/IconButton';
-import React, {useEffect, useMemo, useState} from 'react';
+import {Box} from '@material-ui/core';
+import React from 'react';
 
-import {
-  ALL_VALUE,
-  FilterAndGroupDatasetScorerMetricSpec,
-  FilterAndGroupDatasetScorerSpec,
-  FilterAndGroupDatasetSpec,
-  FilterAndGroupSourceEvaluationSpec,
-  FilterAndGroupSpec,
-  LeaderboardConfigType,
-} from '../../views/Leaderboard/types/leaderboardConfigType';
-import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
-import {
-  fetchDatasetNamesForSpec,
-  fetchDatasetVersionsForSpecAndName,
-  fetchEvaluationNames,
-  fetchEvaluationVersionsForName,
-  fetchMetricPathsForSpec,
-  fetchModelNamesForSpec,
-  fetchModelVersionsForSpecAndName,
-  fetchScorerNamesForSpec,
-  fetchScorerVersionsForSpecAndName,
-  VersionDetails,
-} from './query/configEditorQuery';
+import {PythonLeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
 
 export const LeaderboardConfigEditor: React.FC<{
-  entity: string;
-  project: string;
-  config: LeaderboardConfigType;
-  onCancel: () => void;
-  onPersist: () => void;
-  setConfig: (
-    updater: (config: LeaderboardConfigType) => LeaderboardConfigType
-  ) => void;
-}> = ({entity, project, config, setConfig, onPersist, onCancel}) => {
-  const [activeTab, setActiveTab] = useState(0);
-
-  const handleSave = () => {
-    onPersist();
-  };
-
-  const handleCancel = () => {
-    onCancel();
-  };
-
-  const updateConfig = (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => {
-    setConfig(prevConfig => ({
-      ...prevConfig,
-      config: {
-        ...prevConfig.config,
-        dataSelectionSpec: updater(prevConfig.config.dataSelectionSpec),
-      },
-    }));
-  };
-
-  const updateDescription = (newDescription: string) => {
-    setConfig(prevConfig => ({
-      ...prevConfig,
-      config: {
-        ...prevConfig.config,
-        description: newDescription,
-      },
-    }));
-  };
-
-  const handleTabChange = (event: React.SyntheticEvent, newValue: number) => {
-    setActiveTab(newValue);
-  };
-
-  return (
-    <Box
-      sx={{
-        width: '100%',
-        height: '100%',
-        display: 'flex',
-        flexDirection: 'column',
-        borderLeft: '1px solid #e0e0e0',
-      }}>
-      <Box sx={{borderBottom: 1, borderColor: 'divider'}}>
-        <Tabs value={activeTab} onChange={handleTabChange}>
-          <Tab label="Editor" />
-          <Tab label="Description" />
-          <Tab label="Config Preview" />
-        </Tabs>
-      </Box>
-
-      <Box sx={{flex: 1, overflowY: 'auto', p: 2}}>
-        {activeTab === 0 && (
-          <ConfigEditor
-            entity={entity}
-            project={project}
-            config={config.config.dataSelectionSpec}
-            updateConfig={updateConfig}
-          />
-        )}
-        {activeTab === 1 && (
-          <DescriptionEditor
-            description={config.config.description}
-            updateDescription={updateDescription}
-          />
-        )}
-        {activeTab === 2 && <ConfigPreview config={config} />}
-      </Box>
-
-      <Box
-        sx={{
-          display: 'flex',
-          justifyContent: 'flex-end',
-          pt: 1,
-          pb: 1,
-          pr: 1,
-          borderTop: '1px solid #e0e0e0',
-          height: '52px',
-        }}>
-        <Button variant="outlined" onClick={handleCancel} sx={{mr: 2}}>
-          Cancel
-        </Button>
-        <Button variant="contained" onClick={handleSave}>
-          Save
-        </Button>
-      </Box>
-    </Box>
-  );
-};
-
-const ConfigEditor: React.FC<{
-  entity: string;
-  project: string;
-  config: FilterAndGroupSpec;
-  updateConfig: (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => void;
-}> = ({entity, project, config, updateConfig}) => {
-  return (
-    <>
-      <SourceEvaluationsConfig
-        entity={entity}
-        project={project}
-        sourceEvaluations={config.sourceEvaluations}
-        updateConfig={updateConfig}
-      />
-      <DatasetsConfig
-        entity={entity}
-        project={project}
-        parentSpec={{
-          sourceEvaluations: config.sourceEvaluations,
-          // Purposely exclude models and datasets
-        }}
-        datasets={config.datasets}
-        updateConfig={updateConfig}
-      />
-      <ModelsConfig
-        entity={entity}
-        project={project}
-        parentSpec={{
-          sourceEvaluations: config.sourceEvaluations,
-          datasets: config.datasets,
-          // Purposely exclude models
-        }}
-        models={config.models}
-        updateConfig={updateConfig}
-      />
-    </>
-  );
-};
-
-const ConfigPreview: React.FC<{
-  config: LeaderboardConfigType;
-}> = ({config}) => {
-  return (
-    <Box>
-      <pre
-        style={{
-          backgroundColor: '#f5f5f5',
-          padding: '10px',
-          borderRadius: '4px',
-          overflowX: 'auto',
-          maxHeight: 'calc(100vh - 200px)',
-          whiteSpace: 'pre-wrap',
-          wordWrap: 'break-word',
-        }}>
-        {JSON.stringify(config, null, 2)}
-      </pre>
-    </Box>
-  );
-};
-
-const DescriptionEditor: React.FC<{
-  description: string;
-  updateDescription: (newDescription: string) => void;
-}> = ({description, updateDescription}) => {
-  const [localDescription, setLocalDescription] = useState(description);
-
-  const debouncedUpdate = useMemo(
-    () =>
-      debounce((value: string) => {
-        updateDescription(value);
-      }, 300),
-    [updateDescription]
-  );
-
-  useEffect(() => {
-    setLocalDescription(description);
-  }, [description]);
-
-  const handleChange = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
-    const newValue = event.target.value;
-    setLocalDescription(newValue);
-    debouncedUpdate(newValue);
-  };
-
-  return (
-    <Box sx={{height: 'calc(100vh - 200px)'}}>
-      <textarea
-        value={localDescription}
-        onChange={handleChange}
-        style={{
-          width: '100%',
-          height: '100%',
-          fontFamily: 'monospace',
-          fontSize: '14px',
-          padding: '10px',
-          border: '1px solid #ccc',
-          borderRadius: '4px',
-          resize: 'none',
-          backgroundColor: '#f5f5f5',
-          lineHeight: '1.5',
-          whiteSpace: 'pre-wrap',
-          overflowWrap: 'break-word',
-        }}
-        placeholder="Enter markdown description here..."
-      />
-    </Box>
-  );
-};
-
-// Debounce function
-function debounce<T extends (...args: any[]) => any>(
-  func: T,
-  wait: number
-): (...args: Parameters<T>) => void {
-  let timeout: NodeJS.Timeout | null = null;
-  return (...args: Parameters<T>) => {
-    if (timeout) {
-      clearTimeout(timeout);
-    }
-    timeout = setTimeout(() => func(...args), wait);
-  };
-}
-
-const SourceEvaluationsConfig: React.FC<{
-  entity: string;
-  project: string;
-  sourceEvaluations: FilterAndGroupSourceEvaluationSpec[] | undefined;
-  updateConfig: (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => void;
-}> = ({sourceEvaluations, updateConfig, entity, project}) => {
-  const getTraceServerClient = useGetTraceServerClientContext();
-  const [evaluationNames, setEvaluationNames] = useState<string[]>([]);
-
-  useEffect(() => {
-    fetchEvaluationNames(getTraceServerClient(), entity, project).then(
-      setEvaluationNames
-    );
-  }, [entity, getTraceServerClient, project]);
-
-  const handleAddSourceEvaluation = () => {
-    updateConfig(spec => ({
-      ...spec,
-      sourceEvaluations: [
-        ...(spec.sourceEvaluations || []),
-        {name: ALL_VALUE, version: ALL_VALUE},
-      ],
-    }));
-  };
-
-  return (
-    <Box sx={{mb: 3}}>
-      <Typography variant="h6" gutterBottom>
-        Source Evaluations
-      </Typography>
-      {sourceEvaluations?.map((evaluation, index) => (
-        <SourceEvaluationItem
-          key={index}
-          entity={entity}
-          project={project}
-          evaluation={evaluation}
-          evaluationNames={evaluationNames}
-          updateConfig={updateConfig}
-          index={index}
-        />
-      ))}
-      <Button variant="outlined" onClick={handleAddSourceEvaluation}>
-        Add Source Evaluation
-      </Button>
-    </Box>
-  );
-};
-
-const SourceEvaluationItem: React.FC<{
-  entity: string;
-  project: string;
-  evaluation: FilterAndGroupSourceEvaluationSpec;
-  evaluationNames: string[];
-  updateConfig: (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => void;
-  index: number;
-}> = ({entity, project, evaluation, evaluationNames, updateConfig, index}) => {
-  const getTraceServerClient = useGetTraceServerClientContext();
-  const [versions, setVersions] = useState<VersionDetails[]>([]);
-
-  useEffect(() => {
-    if (evaluation.name && evaluation.name !== ALL_VALUE) {
-      fetchEvaluationVersionsForName(
-        getTraceServerClient(),
-        entity,
-        project,
-        evaluation.name
-      ).then(setVersions);
-    }
-  }, [entity, evaluation.name, getTraceServerClient, project]);
-
-  const handleNameChange = (event: SelectChangeEvent<string>) => {
-    const newName = event.target.value as string;
-    updateConfig(spec => ({
-      ...spec,
-      sourceEvaluations: spec.sourceEvaluations?.map((e, i) =>
-        i === index ? {...e, name: newName, version: ALL_VALUE} : e
-      ),
-    }));
-  };
-
-  const handleVersionChange = (event: SelectChangeEvent<string>) => {
-    const newVersion = event.target.value as string;
-    updateConfig(spec => ({
-      ...spec,
-      sourceEvaluations: spec.sourceEvaluations?.map((e, i) =>
-        i === index ? {...e, version: newVersion} : e
-      ),
-    }));
-  };
-
-  const handleDelete = () => {
-    updateConfig(spec => ({
-      ...spec,
-      sourceEvaluations: spec.sourceEvaluations?.filter((_, i) => i !== index),
-    }));
-  };
-
-  return (
-    <Box sx={{display: 'flex', gap: 2, mb: 2, alignItems: 'center'}}>
-      <FormControl fullWidth>
-        <InputLabel>Name</InputLabel>
-        <Select
-          value={evaluation.name || ALL_VALUE}
-          onChange={handleNameChange}>
-          <MenuItem value={ALL_VALUE}>All</MenuItem>
-          {evaluationNames.map(name => (
-            <MenuItem key={name} value={name}>
-              {name}
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-      <FormControl fullWidth disabled={evaluation.name === ALL_VALUE}>
-        <InputLabel>Version</InputLabel>
-        <Select
-          value={evaluation.version}
-          onChange={handleVersionChange}
-          disabled={evaluation.name === ALL_VALUE}>
-          <MenuItem value={ALL_VALUE}>All</MenuItem>
-          {versions.map(version => (
-            <MenuItem key={version.digest} value={version.digest}>
-              <VersionItem version={version} />
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-      <IconButton onClick={handleDelete} size="small">
-        <DeleteIcon />
-      </IconButton>
-    </Box>
-  );
-};
-
-const DatasetsConfig: React.FC<{
-  entity: string;
-  project: string;
-  parentSpec: FilterAndGroupSpec;
-  datasets: FilterAndGroupDatasetSpec[] | undefined;
-  updateConfig: (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => void;
-}> = ({datasets, updateConfig, entity, project, parentSpec}) => {
-  const getTraceServerClient = useGetTraceServerClientContext();
-  const [datasetNames, setDatasetNames] = useState<string[]>([]);
-
-  useEffect(() => {
-    fetchDatasetNamesForSpec(
-      getTraceServerClient(),
-      entity,
-      project,
-      parentSpec
-    ).then(setDatasetNames);
-  }, [entity, getTraceServerClient, project, parentSpec]);
-
-  const handleAddDataset = () => {
-    updateConfig(spec => ({
-      ...spec,
-      datasets: [
-        ...(spec.datasets || []),
-        {name: ALL_VALUE, version: ALL_VALUE, groupAllVersions: false},
-      ],
-    }));
-  };
-
-  return (
-    <Box sx={{mb: 3}}>
-      <Typography variant="h6" gutterBottom>
-        Datasets
-      </Typography>
-      {datasets?.map((dataset, index) => (
-        <DatasetItem
-          key={index}
-          entity={entity}
-          project={project}
-          parentSpec={{
-            sourceEvaluations: parentSpec.sourceEvaluations,
-            datasets: [
-              {
-                name: dataset.name,
-                version: ALL_VALUE,
-              },
-            ],
-          }}
-          dataset={dataset}
-          datasetNames={datasetNames}
-          updateConfig={updateConfig}
-          index={index}
-        />
-      ))}
-      <Button variant="outlined" onClick={handleAddDataset}>
-        Add Dataset
-      </Button>
-    </Box>
-  );
-};
-
-const DatasetItem: React.FC<{
-  entity: string;
-  project: string;
-  parentSpec: FilterAndGroupSpec;
-  dataset: FilterAndGroupDatasetSpec;
-  datasetNames: string[];
-  updateConfig: (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => void;
-  index: number;
-}> = ({
-  entity,
-  project,
-  parentSpec,
-  dataset,
-  datasetNames,
-  updateConfig,
-  index,
-}) => {
-  const getTraceServerClient = useGetTraceServerClientContext();
-  const [versions, setVersions] = useState<VersionDetails[]>([]);
-  const [scorerNames, setScorerNames] = useState<string[]>([]);
-
-  useEffect(() => {
-    if (dataset.name && dataset.name !== ALL_VALUE) {
-      fetchDatasetVersionsForSpecAndName(
-        getTraceServerClient(),
-        entity,
-        project,
-        parentSpec,
-        dataset.name
-      ).then(setVersions);
-    }
-    fetchScorerNamesForSpec(getTraceServerClient(), entity, project, {
-      ...parentSpec,
-      datasets: [dataset],
-    }).then(setScorerNames);
-  }, [
-    dataset.name,
-    entity,
-    getTraceServerClient,
-    project,
-    parentSpec,
-    dataset,
-  ]);
-
-  const handleNameChange = (event: SelectChangeEvent<string>) => {
-    const newName = event.target.value as string;
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === index ? {...d, name: newName, version: ALL_VALUE} : d
-      ),
-    }));
-  };
-
-  const handleVersionChange = (event: SelectChangeEvent<string>) => {
-    const newVersion = event.target.value as string;
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === index ? {...d, version: newVersion} : d
-      ),
-    }));
-  };
-
-  const handleGroupAllVersionsChange = (
-    event: React.ChangeEvent<HTMLInputElement>
-  ) => {
-    const newGroupAllVersions = event.target.checked;
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === index ? {...d, groupAllVersions: newGroupAllVersions} : d
-      ),
-    }));
-  };
-
-  const handleAddScorer = () => {
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === index
-          ? {
-              ...d,
-              scorers: [
-                ...(d.scorers || []),
-                {
-                  name: ALL_VALUE,
-                  version: ALL_VALUE,
-                  groupAllVersions: false,
-                  metrics: [],
-                },
-              ],
-            }
-          : d
-      ),
-    }));
-  };
-
-  const handleDelete = () => {
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.filter((_, i) => i !== index),
-    }));
-  };
-
-  return (
-    <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, mb: 2}}>
-      <Box sx={{display: 'flex', gap: 2, alignItems: 'center'}}>
-        <FormControl fullWidth>
-          <InputLabel>Name</InputLabel>
-          <Select value={dataset.name || ALL_VALUE} onChange={handleNameChange}>
-            <MenuItem value={ALL_VALUE}>All</MenuItem>
-            {datasetNames.map(name => (
-              <MenuItem key={name} value={name}>
-                {name}
-              </MenuItem>
-            ))}
-          </Select>
-        </FormControl>
-        <FormControl fullWidth disabled={dataset.name === ALL_VALUE}>
-          <InputLabel>Version</InputLabel>
-          <Select
-            value={dataset.version}
-            onChange={handleVersionChange}
-            disabled={dataset.name === ALL_VALUE}>
-            <MenuItem value={ALL_VALUE}>All</MenuItem>
-            {versions.map(version => (
-              <MenuItem key={version.digest} value={version.digest}>
-                <VersionItem version={version} />
-              </MenuItem>
-            ))}
-          </Select>
-        </FormControl>
-        <IconButton onClick={handleDelete} size="small">
-          <DeleteIcon />
-        </IconButton>
-      </Box>
-      <FormControlLabel
-        control={
-          <Checkbox
-            checked={dataset.groupAllVersions}
-            onChange={handleGroupAllVersionsChange}
-          />
-        }
-        label="Group All Versions"
-      />
-      <Typography variant="subtitle1">Scorers</Typography>
-      {dataset.scorers?.map((scorer, scorerIndex) => (
-        <ScorerItem
-          key={scorerIndex}
-          entity={entity}
-          project={project}
-          parentSpec={{
-            sourceEvaluations: parentSpec.sourceEvaluations,
-            datasets: [
-              {
-                ...(parentSpec.datasets?.[0] ?? {
-                  name: ALL_VALUE,
-                  version: ALL_VALUE,
-                }),
-              },
-            ],
-          }}
-          scorer={scorer}
-          scorerNames={scorerNames}
-          updateConfig={updateConfig}
-          datasetIndex={index}
-          scorerIndex={scorerIndex}
-        />
-      ))}
-      <Button variant="outlined" onClick={handleAddScorer}>
-        Add Scorer
-      </Button>
-    </Box>
-  );
-};
-
-const ScorerItem: React.FC<{
-  entity: string;
-  project: string;
-  parentSpec: FilterAndGroupSpec;
-  scorer: FilterAndGroupDatasetScorerSpec;
-  scorerNames: string[];
-  updateConfig: (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => void;
-  datasetIndex: number;
-  scorerIndex: number;
-}> = ({
-  entity,
-  project,
-  parentSpec,
-  scorer,
-  scorerNames,
-  updateConfig,
-  datasetIndex,
-  scorerIndex,
-}) => {
-  const getTraceServerClient = useGetTraceServerClientContext();
-  const [versions, setVersions] = useState<VersionDetails[]>([]);
-  const [metricPaths, setMetricPaths] = useState<string[]>([]);
-
-  useEffect(() => {
-    if (scorer.name && scorer.name !== ALL_VALUE) {
-      fetchScorerVersionsForSpecAndName(
-        getTraceServerClient(),
-        entity,
-        project,
-        parentSpec,
-        scorer.name
-      ).then(setVersions);
-    }
-    fetchMetricPathsForSpec(getTraceServerClient(), entity, project, {
-      ...parentSpec,
-      datasets: [
-        {
-          ...(parentSpec.datasets?.[0] ?? {
-            name: ALL_VALUE,
-            version: ALL_VALUE,
-          }),
-          scorers: [
-            {
-              ...scorer,
-              metrics: [],
-            },
-          ],
-        },
-      ],
-    }).then(setMetricPaths);
-  }, [entity, getTraceServerClient, parentSpec, project, scorer, scorer.name]);
-
-  const handleNameChange = (event: SelectChangeEvent<string>) => {
-    const newName = event.target.value as string;
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === datasetIndex
-          ? {
-              ...d,
-              scorers: d.scorers?.map((s, j) =>
-                j === scorerIndex
-                  ? {...s, name: newName, version: ALL_VALUE}
-                  : s
-              ),
-            }
-          : d
-      ),
-    }));
-  };
-
-  const handleVersionChange = (event: SelectChangeEvent<string>) => {
-    const newVersion = event.target.value as string;
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === datasetIndex
-          ? {
-              ...d,
-              scorers: d.scorers?.map((s, j) =>
-                j === scorerIndex ? {...s, version: newVersion} : s
-              ),
-            }
-          : d
-      ),
-    }));
-  };
-
-  const handleGroupAllVersionsChange = (
-    event: React.ChangeEvent<HTMLInputElement>
-  ) => {
-    const newGroupAllVersions = event.target.checked;
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === datasetIndex
-          ? {
-              ...d,
-              scorers: d.scorers?.map((s, j) =>
-                j === scorerIndex
-                  ? {...s, groupAllVersions: newGroupAllVersions}
-                  : s
-              ),
-            }
-          : d
-      ),
-    }));
-  };
-
-  const handleAddMetric = () => {
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === datasetIndex
-          ? {
-              ...d,
-              scorers: d.scorers?.map((s, j) =>
-                j === scorerIndex
-                  ? {
-                      ...s,
-                      metrics: [
-                        ...(s.metrics || []),
-                        {path: ALL_VALUE, shouldMinimize: false},
-                      ],
-                    }
-                  : s
-              ),
-            }
-          : d
-      ),
-    }));
-  };
-
-  const handleDelete = () => {
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === datasetIndex
-          ? {
-              ...d,
-              scorers: d.scorers?.filter((_, j) => j !== scorerIndex),
-            }
-          : d
-      ),
-    }));
-  };
-
-  return (
-    <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, ml: 2, mb: 2}}>
-      <Box sx={{display: 'flex', gap: 2, alignItems: 'center'}}>
-        <FormControl fullWidth>
-          <InputLabel>Name</InputLabel>
-          <Select value={scorer.name || ALL_VALUE} onChange={handleNameChange}>
-            <MenuItem value={ALL_VALUE}>All</MenuItem>
-            {scorerNames.map(name => (
-              <MenuItem key={name} value={name}>
-                {name}
-              </MenuItem>
-            ))}
-          </Select>
-        </FormControl>
-        <FormControl fullWidth disabled={scorer.name === ALL_VALUE}>
-          <InputLabel>Version</InputLabel>
-          <Select
-            value={scorer.version}
-            onChange={handleVersionChange}
-            disabled={scorer.name === ALL_VALUE}>
-            <MenuItem value={ALL_VALUE}>All</MenuItem>
-            {versions.map(version => (
-              <MenuItem key={version.digest} value={version.digest}>
-                <VersionItem version={version} />
-              </MenuItem>
-            ))}
-          </Select>
-        </FormControl>
-        <IconButton onClick={handleDelete} size="small">
-          <DeleteIcon />
-        </IconButton>
-      </Box>
-      <FormControlLabel
-        control={
-          <Checkbox
-            checked={scorer.groupAllVersions}
-            onChange={handleGroupAllVersionsChange}
-          />
-        }
-        label="Group All Versions"
-      />
-      <Typography variant="subtitle2">Metrics</Typography>
-      {scorer.metrics?.map((metric, metricIndex) => (
-        <MetricItem
-          key={metricIndex}
-          metric={metric}
-          metricPaths={metricPaths}
-          updateConfig={updateConfig}
-          datasetIndex={datasetIndex}
-          scorerIndex={scorerIndex}
-          metricIndex={metricIndex}
-        />
-      ))}
-      <Button variant="outlined" onClick={handleAddMetric}>
-        Add Metric
-      </Button>
-    </Box>
-  );
-};
-
-const MetricItem: React.FC<{
-  metric: FilterAndGroupDatasetScorerMetricSpec;
-  metricPaths: string[];
-  updateConfig: (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => void;
-  datasetIndex: number;
-  scorerIndex: number;
-  metricIndex: number;
-}> = ({
-  metric,
-  metricPaths,
-  updateConfig,
-  datasetIndex,
-  scorerIndex,
-  metricIndex,
-}) => {
-  const handlePathChange = (event: SelectChangeEvent<string>) => {
-    const newPath = event.target.value as string;
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === datasetIndex
-          ? {
-              ...d,
-              scorers: d.scorers?.map((s, j) =>
-                j === scorerIndex
-                  ? {
-                      ...s,
-                      metrics: s.metrics?.map((m, k) =>
-                        k === metricIndex ? {...m, path: newPath} : m
-                      ),
-                    }
-                  : s
-              ),
-            }
-          : d
-      ),
-    }));
-  };
-
-  const handleShouldMinimizeChange = (
-    event: React.ChangeEvent<HTMLInputElement>
-  ) => {
-    const newShouldMinimize = event.target.checked;
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === datasetIndex
-          ? {
-              ...d,
-              scorers: d.scorers?.map((s, j) =>
-                j === scorerIndex
-                  ? {
-                      ...s,
-                      metrics: s.metrics?.map((m, k) =>
-                        k === metricIndex
-                          ? {...m, shouldMinimize: newShouldMinimize}
-                          : m
-                      ),
-                    }
-                  : s
-              ),
-            }
-          : d
-      ),
-    }));
-  };
-
-  const handleDelete = () => {
-    updateConfig(spec => ({
-      ...spec,
-      datasets: spec.datasets?.map((d, i) =>
-        i === datasetIndex
-          ? {
-              ...d,
-              scorers: d.scorers?.map((s, j) =>
-                j === scorerIndex
-                  ? {
-                      ...s,
-                      metrics: s.metrics?.filter((_, k) => k !== metricIndex),
-                    }
-                  : s
-              ),
-            }
-          : d
-      ),
-    }));
-  };
-
-  return (
-    <Box sx={{display: 'flex', gap: 2, ml: 2, mb: 2, alignItems: 'center'}}>
-      <FormControl fullWidth>
-        <InputLabel>Metric Path</InputLabel>
-        <Select value={metric.path} onChange={handlePathChange}>
-          <MenuItem value={ALL_VALUE}>All</MenuItem>
-          {metricPaths.map(path => (
-            <MenuItem key={path} value={path}>
-              {path}
-            </MenuItem>
-          ))}
-        </Select>
-      </FormControl>
-      <FormControlLabel
-        control={
-          <Checkbox
-            checked={metric.shouldMinimize}
-            onChange={handleShouldMinimizeChange}
-          />
-        }
-        label="Should Minimize"
-      />
-      <IconButton onClick={handleDelete} size="small">
-        <DeleteIcon />
-      </IconButton>
-    </Box>
-  );
-};
-
-const ModelsConfig: React.FC<{
-  entity: string;
-  project: string;
-  parentSpec: FilterAndGroupSpec;
-  models: FilterAndGroupDatasetSpec[] | undefined;
-  updateConfig: (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => void;
-}> = ({entity, project, parentSpec, models, updateConfig}) => {
-  const getTraceServerClient = useGetTraceServerClientContext();
-  const [modelNames, setModelNames] = useState<string[]>([]);
-
-  useEffect(() => {
-    fetchModelNamesForSpec(
-      getTraceServerClient(),
-      entity,
-      project,
-      parentSpec
-    ).then(setModelNames);
-  }, [entity, project, getTraceServerClient, parentSpec]);
-
-  const handleAddModel = () => {
-    updateConfig(spec => ({
-      ...spec,
-      models: [
-        ...(spec.models || []),
-        {name: ALL_VALUE, version: ALL_VALUE, groupAllVersions: false},
-      ],
-    }));
-  };
-
-  return (
-    <Box sx={{mb: 3}}>
-      <Typography variant="h6" gutterBottom>
-        Models
-      </Typography>
-      {models?.map((model, index) => (
-        <ModelItem
-          key={index}
-          entity={entity}
-          project={project}
-          parentSpec={{
-            sourceEvaluations: parentSpec.sourceEvaluations,
-            datasets: parentSpec.datasets,
-            models: [
-              {
-                name: model.name,
-                version: ALL_VALUE,
-              },
-            ],
-          }}
-          model={model}
-          modelNames={modelNames}
-          updateConfig={updateConfig}
-          index={index}
-        />
-      ))}
-      <Button variant="outlined" onClick={handleAddModel}>
-        Add Model
-      </Button>
-    </Box>
-  );
-};
-
-const ModelItem: React.FC<{
-  entity: string;
-  project: string;
-  parentSpec: FilterAndGroupSpec;
-  model: FilterAndGroupDatasetSpec;
-  modelNames: string[];
-  updateConfig: (
-    updater: (spec: FilterAndGroupSpec) => FilterAndGroupSpec
-  ) => void;
-  index: number;
-}> = ({
-  entity,
-  project,
-  parentSpec,
-  model,
-  modelNames,
-  updateConfig,
-  index,
-}) => {
-  const getTraceServerClient = useGetTraceServerClientContext();
-
-  const [versions, setVersions] = useState<VersionDetails[]>([]);
-
-  useEffect(() => {
-    if (model.name && model.name !== ALL_VALUE) {
-      fetchModelVersionsForSpecAndName(
-        getTraceServerClient(),
-        entity,
-        project,
-        parentSpec,
-        model.name
-      ).then(setVersions);
-    }
-  }, [model.name, entity, project, getTraceServerClient, parentSpec]);
-
-  const handleNameChange = (event: SelectChangeEvent<string>) => {
-    const newName = event.target.value as string;
-    updateConfig(spec => ({
-      ...spec,
-      models: spec.models?.map((m, i) =>
-        i === index ? {...m, name: newName, version: ALL_VALUE} : m
-      ),
-    }));
-  };
-
-  const handleVersionChange = (event: SelectChangeEvent<string>) => {
-    const newVersion = event.target.value as string;
-    updateConfig(spec => ({
-      ...spec,
-      models: spec.models?.map((m, i) =>
-        i === index ? {...m, version: newVersion} : m
-      ),
-    }));
-  };
-
-  const handleGroupAllVersionsChange = (
-    event: React.ChangeEvent<HTMLInputElement>
-  ) => {
-    const newGroupAllVersions = event.target.checked;
-    updateConfig(spec => ({
-      ...spec,
-      models: spec.models?.map((m, i) =>
-        i === index ? {...m, groupAllVersions: newGroupAllVersions} : m
-      ),
-    }));
-  };
-
-  const handleDelete = () => {
-    updateConfig(spec => ({
-      ...spec,
-      models: spec.models?.filter((_, i) => i !== index),
-    }));
-  };
-
-  return (
-    <Box sx={{display: 'flex', flexDirection: 'column', gap: 2, mb: 2}}>
-      <Box sx={{display: 'flex', gap: 2, alignItems: 'center'}}>
-        <FormControl fullWidth>
-          <InputLabel>Name</InputLabel>
-          <Select value={model.name || ALL_VALUE} onChange={handleNameChange}>
-            <MenuItem value={ALL_VALUE}>All</MenuItem>
-            {modelNames.map(name => (
-              <MenuItem key={name} value={name}>
-                {name}
-              </MenuItem>
-            ))}
-          </Select>
-        </FormControl>
-        <FormControl fullWidth disabled={model.name === ALL_VALUE}>
-          <InputLabel>Version</InputLabel>
-          <Select
-            value={model.version}
-            onChange={handleVersionChange}
-            disabled={model.name === ALL_VALUE}>
-            <MenuItem value={ALL_VALUE}>All</MenuItem>
-            {versions.map(version => (
-              <MenuItem key={version.digest} value={version.digest}>
-                <VersionItem version={version} />
-              </MenuItem>
-            ))}
-          </Select>
-        </FormControl>
-        <IconButton onClick={handleDelete} size="small">
-          <DeleteIcon />
-        </IconButton>
-      </Box>
-      <FormControlLabel
-        control={
-          <Checkbox
-            checked={model.groupAllVersions}
-            onChange={handleGroupAllVersionsChange}
-          />
-        }
-        label="Group All Versions"
-      />
-    </Box>
-  );
-};
-
-const VersionItem: React.FC<{
-  version: VersionDetails;
-}> = ({version}) => {
-  return (
-    <div>
-      v{version.index} ({version.digest.slice(0, 6)})
-    </div>
-  );
+  leaderboardVal: PythonLeaderboardObjectVal;
+  saving: boolean;
+  setWorkingCopy: (leaderboardVal: PythonLeaderboardObjectVal) => void;
+  discardChanges: () => void;
+  commitChanges: () => void;
+}> = ({leaderboardVal, setWorkingCopy, discardChanges, commitChanges}) => {
+  return <Box>Config Editor</Box>;
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index cde7c56c08d..646e9b3ad01 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -15,6 +15,7 @@ import {useWFHooks} from '../wfReactInterface/context';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
 import {StyledReactMarkdown} from './EditableMarkdown';
+import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
 
 type LeaderboardPageProps = {
   entity: string;
@@ -247,16 +248,6 @@ export const LeaderboardPageContentInner: React.FC<
   );
 };
 
-const LeaderboardConfigEditor: React.FC<{
-  leaderboardVal: PythonLeaderboardObjectVal;
-  saving: boolean;
-  setWorkingCopy: (leaderboardVal: PythonLeaderboardObjectVal) => void;
-  discardChanges: () => void;
-  commitChanges: () => void;
-}> = ({leaderboardVal, setWorkingCopy, discardChanges, commitChanges}) => {
-  return <Box>Config Editor</Box>;
-};
-
 export const ToggleLeaderboardConfig: React.FC<{
   isOpen: boolean;
   onClick: () => void;

From dc307b58f255fb8c5de9bb2e2dc4e9ce9b054825 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Sun, 20 Oct 2024 19:46:15 -0700
Subject: [PATCH 088/169] TODO: Make the editor pull live data, Add Create new,
 code clean

---
 weave-js/package.json                         |   2 +
 .../LeaderboardConfigEditor.tsx               | 259 +++++++++++++++++-
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  60 +++-
 weave-js/yarn.lock                            |  48 +++-
 4 files changed, 352 insertions(+), 17 deletions(-)

diff --git a/weave-js/package.json b/weave-js/package.json
index a112610f807..b9d3afe8ff8 100644
--- a/weave-js/package.json
+++ b/weave-js/package.json
@@ -34,6 +34,7 @@
     "@emotion/styled": "^11.11.0",
     "@floating-ui/react": "^0.24.5",
     "@material-ui/core": "^4.12.4",
+    "@material-ui/icons": "^4.11.3",
     "@monaco-editor/react": "^4.3.1",
     "@mui/icons-material": "^5.14.12",
     "@mui/material": "^6.0.2",
@@ -94,6 +95,7 @@
     "react": "^17.0.2",
     "react-app-polyfill": "^3.0.0",
     "react-base-table": "^1.12.0",
+    "react-beautiful-dnd": "^13.1.1",
     "react-cytoscapejs": "^1.2.1",
     "react-datetime": "^3",
     "react-diff-viewer": "^3.1.1",
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index c777cc88044..960b7ce375f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -1,14 +1,267 @@
-import {Box} from '@material-ui/core';
+import {
+  Box,
+  Button,
+  Checkbox,
+  FormControlLabel,
+  Grid,
+  IconButton,
+  MenuItem,
+  Select,
+  TextField,
+  Typography,
+} from '@material-ui/core';
+import {
+  Add as AddIcon,
+  ArrowDownward as MoveDownIcon,
+  ArrowUpward as MoveUpIcon,
+  Delete as DeleteIcon,
+  FileCopy as CloneIcon,
+} from '@material-ui/icons';
 import React from 'react';
 
 import {PythonLeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
 
+// Placeholder data for dropdowns
+const evaluationObjects = ['Evaluation1', 'Evaluation2', 'Evaluation3'];
+const scorerNames = ['Scorer1', 'Scorer2', 'Scorer3'];
+const metricPaths = ['Metric1', 'Metric2', 'Metric3'];
+
 export const LeaderboardConfigEditor: React.FC<{
   leaderboardVal: PythonLeaderboardObjectVal;
   saving: boolean;
+  isDirty: boolean;
   setWorkingCopy: (leaderboardVal: PythonLeaderboardObjectVal) => void;
   discardChanges: () => void;
   commitChanges: () => void;
-}> = ({leaderboardVal, setWorkingCopy, discardChanges, commitChanges}) => {
-  return <Box>Config Editor</Box>;
+}> = ({
+  leaderboardVal,
+  saving,
+  isDirty,
+  setWorkingCopy,
+  discardChanges,
+  commitChanges,
+}) => {
+  const handleNameChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    setWorkingCopy({...leaderboardVal, name: event.target.value});
+  };
+
+  const handleDescriptionChange = (
+    event: React.ChangeEvent<HTMLInputElement>
+  ) => {
+    setWorkingCopy({...leaderboardVal, description: event.target.value});
+  };
+
+  const handleColumnChange = (index: number, field: string, value: any) => {
+    const newColumns = [...leaderboardVal.columns];
+    newColumns[index] = {...newColumns[index], [field]: value};
+    setWorkingCopy({...leaderboardVal, columns: newColumns});
+  };
+
+  const addColumn = () => {
+    setWorkingCopy({
+      ...leaderboardVal,
+      columns: [
+        ...leaderboardVal.columns,
+        {
+          evaluation_object_ref: '',
+          scorer_name: '',
+          should_minimize: false,
+          summary_metric_path_parts: [],
+        },
+      ],
+    });
+  };
+
+  const removeColumn = (index: number) => {
+    const newColumns = leaderboardVal.columns.filter((_, i) => i !== index);
+    setWorkingCopy({...leaderboardVal, columns: newColumns});
+  };
+
+  const cloneColumn = (index: number) => {
+    const newColumns = [...leaderboardVal.columns];
+    newColumns.splice(index + 1, 0, {...newColumns[index]});
+    setWorkingCopy({...leaderboardVal, columns: newColumns});
+  };
+
+  const moveColumn = (fromIndex: number, toIndex: number) => {
+    const newColumns = [...leaderboardVal.columns];
+    const [removed] = newColumns.splice(fromIndex, 1);
+    newColumns.splice(toIndex, 0, removed);
+    setWorkingCopy({...leaderboardVal, columns: newColumns});
+  };
+
+  return (
+    <Box display="flex" flexDirection="column" height="100%" width="100%">
+      <Box flexGrow={1} overflow="auto" p={2}>
+        <Typography variant="h6" gutterBottom>
+          Display Name
+        </Typography>
+        <TextField
+          fullWidth
+          value={leaderboardVal.name}
+          onChange={handleNameChange}
+          margin="normal"
+        />
+        <Typography variant="h6" gutterBottom style={{marginTop: 16}}>
+          Description
+        </Typography>
+        <TextField
+          fullWidth
+          value={leaderboardVal.description}
+          onChange={handleDescriptionChange}
+          margin="normal"
+          multiline
+          rows={6}
+          InputProps={{style: {fontFamily: 'monospace', minHeight: '120px'}}}
+        />
+        <Typography variant="h6" gutterBottom style={{marginTop: 16}}>
+          Columns
+        </Typography>
+        {leaderboardVal.columns.map((column, index) => (
+          <Grid
+            container
+            spacing={2}
+            key={index}
+            alignItems="center"
+            style={{marginBottom: 8}}>
+            <Grid item xs={12} sm={3}>
+              <Select
+                fullWidth
+                value={column.evaluation_object_ref}
+                onChange={e =>
+                  handleColumnChange(
+                    index,
+                    'evaluation_object_ref',
+                    e.target.value
+                  )
+                }
+                displayEmpty
+                margin="dense">
+                <MenuItem value="">
+                  <em>Select Evaluation Object</em>
+                </MenuItem>
+                {evaluationObjects.map(obj => (
+                  <MenuItem key={obj} value={obj}>
+                    {obj}
+                  </MenuItem>
+                ))}
+              </Select>
+            </Grid>
+            <Grid item xs={12} sm={3}>
+              <Select
+                fullWidth
+                value={column.scorer_name}
+                onChange={e =>
+                  handleColumnChange(index, 'scorer_name', e.target.value)
+                }
+                displayEmpty
+                margin="dense">
+                <MenuItem value="">
+                  <em>Select Scorer</em>
+                </MenuItem>
+                {scorerNames.map(scorer => (
+                  <MenuItem key={scorer} value={scorer}>
+                    {scorer}
+                  </MenuItem>
+                ))}
+              </Select>
+            </Grid>
+            <Grid item xs={12} sm={3}>
+              <Select
+                fullWidth
+                multiple
+                value={column.summary_metric_path_parts}
+                onChange={e =>
+                  handleColumnChange(
+                    index,
+                    'summary_metric_path_parts',
+                    e.target.value
+                  )
+                }
+                renderValue={selected => (selected as string[]).join(' > ')}
+                displayEmpty
+                margin="dense">
+                <MenuItem value="">
+                  <em>Select Metric Path</em>
+                </MenuItem>
+                {metricPaths.map(path => (
+                  <MenuItem key={path} value={path}>
+                    {path}
+                  </MenuItem>
+                ))}
+              </Select>
+            </Grid>
+            <Grid item xs={6} sm={2}>
+              <FormControlLabel
+                control={
+                  <Checkbox
+                    checked={column.should_minimize}
+                    onChange={e =>
+                      handleColumnChange(
+                        index,
+                        'should_minimize',
+                        e.target.checked
+                      )
+                    }
+                  />
+                }
+                label="Minimize"
+              />
+            </Grid>
+            <Grid item xs={6} sm={1}>
+              <Box display="flex" justifyContent="flex-end">
+                <IconButton
+                  size="small"
+                  onClick={() => moveColumn(index, index - 1)}
+                  disabled={index === 0}>
+                  <MoveUpIcon />
+                </IconButton>
+                <IconButton
+                  size="small"
+                  onClick={() => moveColumn(index, index + 1)}
+                  disabled={index === leaderboardVal.columns.length - 1}>
+                  <MoveDownIcon />
+                </IconButton>
+                <IconButton size="small" onClick={() => cloneColumn(index)}>
+                  <CloneIcon />
+                </IconButton>
+                <IconButton size="small" onClick={() => removeColumn(index)}>
+                  <DeleteIcon />
+                </IconButton>
+              </Box>
+            </Grid>
+          </Grid>
+        ))}
+        <Button startIcon={<AddIcon />} onClick={addColumn}>
+          Add Column
+        </Button>
+      </Box>
+      <Box
+        flexShrink={0}
+        p={2}
+        borderTop={1}
+        borderColor="divider"
+        height="52px"
+        display="flex"
+        alignItems="center"
+        justifyContent="flex-end">
+        <Button
+          variant="outlined"
+          onClick={discardChanges}
+          disabled={saving}
+          style={{marginRight: 8}}>
+          {isDirty ? 'Discard' : 'Close'}
+        </Button>
+        {isDirty && (
+          <Button
+            variant="contained"
+            color="primary"
+            onClick={commitChanges}
+            disabled={saving}>
+            Save
+          </Button>
+        )}
+      </Box>
+    </Box>
+  );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 646e9b3ad01..64c0a90c39d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -1,9 +1,18 @@
 import {Box} from '@mui/material';
 import {MOON_250} from '@wandb/weave/common/css/color.styles';
+import {useViewerInfo} from '@wandb/weave/common/hooks/useViewerInfo';
 import {Button} from '@wandb/weave/components/Button';
 import {ErrorPanel} from '@wandb/weave/components/ErrorPanel';
 import {Loading} from '@wandb/weave/components/Loading';
-import React, {FC, useCallback, useContext, useEffect, useState} from 'react';
+import _ from 'lodash';
+import React, {
+  FC,
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useState,
+} from 'react';
 
 import {WeaveflowPeekContext} from '../../context';
 import {NotFoundPanel} from '../../NotFoundPanel';
@@ -27,6 +36,7 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
   const [name, setName] = useState(props.leaderboardName);
   const {isPeeking} = useContext(WeaveflowPeekContext);
   const [isEditing, setIsEditing] = useState(false);
+  const {isEditor} = useIsEditor(props.entity);
   return (
     <SimplePageLayout
       title={name}
@@ -46,7 +56,8 @@ export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
       ]}
       headerExtra={
         !isPeeking &&
-        !isEditing && (
+        !isEditing &&
+        isEditor && (
           <EditLeaderboardButton
             isEditing={isEditing}
             setIsEditing={setIsEditing}
@@ -127,7 +138,12 @@ const useUpdateLeaderboard = (
       obj: {
         project_id: projectIdFromParts({entity, project}),
         object_id: objectId,
-        val: leaderboardVal,
+        val: {
+          _type: 'Leaderboard',
+          _class_name: 'Leaderboard',
+          _bases: ['Object', 'BaseModel'],
+          ...leaderboardVal,
+        },
       },
     });
   };
@@ -144,11 +160,6 @@ export const LeaderboardPageContentInner: React.FC<
     leaderboardVal: PythonLeaderboardObjectVal;
   }
 > = props => {
-  useEffect(() => {
-    props.setName(props.leaderboardVal.name);
-  }, [props]);
-
-  const description = props.leaderboardVal.description;
   const updateLeaderboard = useUpdateLeaderboard(
     props.entity,
     props.project,
@@ -157,6 +168,9 @@ export const LeaderboardPageContentInner: React.FC<
   const [workingLeaderboardValCopy, setWorkingLeaderboardValCopy] = useState(
     props.leaderboardVal
   );
+  useEffect(() => {
+    props.setName(workingLeaderboardValCopy.name);
+  }, [props, workingLeaderboardValCopy.name]);
   const {loading, data} = usePythonLeaderboardData(
     props.entity,
     props.project,
@@ -185,6 +199,9 @@ export const LeaderboardPageContentInner: React.FC<
         }
       });
   }, [props, updateLeaderboard, workingLeaderboardValCopy]);
+  const isDirty = useMemo(() => {
+    return !_.isEqual(props.leaderboardVal, workingLeaderboardValCopy);
+  }, [props.leaderboardVal, workingLeaderboardValCopy]);
 
   return (
     <Box display="flex" flexDirection="row" height="100%" flexGrow={1}>
@@ -206,8 +223,10 @@ export const LeaderboardPageContentInner: React.FC<
             gap: '12px',
             overflowY: 'auto',
           }}>
-          {description && (
-            <StyledReactMarkdown>{description}</StyledReactMarkdown>
+          {workingLeaderboardValCopy.description && (
+            <StyledReactMarkdown>
+              {workingLeaderboardValCopy.description}
+            </StyledReactMarkdown>
           )}
         </Box>
         <Box
@@ -237,7 +256,8 @@ export const LeaderboardPageContentInner: React.FC<
           }}>
           <LeaderboardConfigEditor
             saving={saving}
-            leaderboardVal={props.leaderboardVal}
+            isDirty={isDirty}
+            leaderboardVal={workingLeaderboardValCopy}
             setWorkingCopy={setWorkingLeaderboardValCopy}
             discardChanges={discardChanges}
             commitChanges={commitChanges}
@@ -357,3 +377,21 @@ const EditLeaderboardButton: FC<{
     </Box>
   );
 };
+
+const useIsEditor = (entity: string) => {
+  const {loading: loadingUserInfo, userInfo} = useViewerInfo();
+  return useMemo(() => {
+    if (loadingUserInfo) {
+      return {
+        loading: true,
+        isEditor: false,
+      };
+    }
+    const viewer = userInfo ? userInfo.id : null;
+
+    return {
+      loading: false,
+      isEditor: viewer && userInfo?.teams.includes(entity),
+    };
+  }, [entity, loadingUserInfo, userInfo]);
+};
diff --git a/weave-js/yarn.lock b/weave-js/yarn.lock
index a47e318e3fe..7f5b0262f70 100644
--- a/weave-js/yarn.lock
+++ b/weave-js/yarn.lock
@@ -2792,6 +2792,13 @@
     react-is "^16.8.0 || ^17.0.0"
     react-transition-group "^4.4.0"
 
+"@material-ui/icons@^4.11.3":
+  version "4.11.3"
+  resolved "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz#b0693709f9b161ce9ccde276a770d968484ecff1"
+  integrity sha512-IKHlyx6LDh8n19vzwH5RtHIOHl9Tu90aAAxcbWME6kp4dmvODM3UvOHJeMIDzUbd4muuJKHmlNoBN+mDY4XkBA==
+  dependencies:
+    "@babel/runtime" "^7.4.4"
+
 "@material-ui/lab@^4.0.0-alpha.60":
   version "4.0.0-alpha.61"
   resolved "https://registry.yarnpkg.com/@material-ui/lab/-/lab-4.0.0-alpha.61.tgz#9bf8eb389c0c26c15e40933cc114d4ad85e3d978"
@@ -6448,6 +6455,13 @@ cryptiles@2.x.x:
   dependencies:
     boom "2.x.x"
 
+css-box-model@^1.2.0:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/css-box-model/-/css-box-model-1.2.1.tgz#59951d3b81fd6b2074a62d49444415b0d2b4d7c1"
+  integrity sha512-a7Vr4Q/kd/aw96bnJG332W9V9LkJO69JRcaCYDUqjp6/z0w6VcZjgAcTbgFxEPfBgdnAwlh3iwu+hLopa+flJw==
+  dependencies:
+    tiny-invariant "^1.0.6"
+
 css-color-keywords@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/css-color-keywords/-/css-color-keywords-1.0.0.tgz#fea2616dc676b2962686b3af8dbdbe180b244e05"
@@ -10378,7 +10392,7 @@ memoize-one@*, memoize-one@^6.0.0:
   resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-6.0.0.tgz#b2591b871ed82948aee4727dc6abceeeac8c1045"
   integrity sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw==
 
-"memoize-one@>=3.1.1 <6", memoize-one@^5.0.0, memoize-one@^5.0.4:
+"memoize-one@>=3.1.1 <6", memoize-one@^5.0.0, memoize-one@^5.0.4, memoize-one@^5.1.1:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e"
   integrity sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==
@@ -11960,6 +11974,11 @@ quickselect@^2.0.0:
   resolved "https://registry.yarnpkg.com/quickselect/-/quickselect-2.0.0.tgz#f19680a486a5eefb581303e023e98faaf25dd018"
   integrity sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw==
 
+raf-schd@^4.0.2:
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/raf-schd/-/raf-schd-4.0.3.tgz#5d6c34ef46f8b2a0e880a8fcdb743efc5bfdbc1a"
+  integrity sha512-tQkJl2GRWh83ui2DiPTJz9wEiMN20syf+5oKfB03yYP7ioZcJwsIK8FjrtLwH1m7C7e+Tt2yYBlrOpdT+dyeIQ==
+
 raf@^3.4.1:
   version "3.4.1"
   resolved "https://registry.yarnpkg.com/raf/-/raf-3.4.1.tgz#0742e99a4a6552f445d73e3ee0328af0ff1ede39"
@@ -11991,6 +12010,19 @@ react-base-table@^1.12.0:
     react-virtualized-auto-sizer "^1.0.2"
     react-window "^1.8.2"
 
+react-beautiful-dnd@^13.1.1:
+  version "13.1.1"
+  resolved "https://registry.yarnpkg.com/react-beautiful-dnd/-/react-beautiful-dnd-13.1.1.tgz#b0f3087a5840920abf8bb2325f1ffa46d8c4d0a2"
+  integrity sha512-0Lvs4tq2VcrEjEgDXHjT98r+63drkKEgqyxdA7qD3mvKwga6a5SscbdLPO2IExotU1jW8L0Ksdl0Cj2AF67nPQ==
+  dependencies:
+    "@babel/runtime" "^7.9.2"
+    css-box-model "^1.2.0"
+    memoize-one "^5.1.1"
+    raf-schd "^4.0.2"
+    react-redux "^7.2.0"
+    redux "^4.0.4"
+    use-memo-one "^1.1.1"
+
 react-cytoscapejs@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/react-cytoscapejs/-/react-cytoscapejs-1.2.1.tgz#ccce46acabf4f0c41dce9070743854f92b0dc050"
@@ -12116,7 +12148,7 @@ react-popper@^1.3.4:
     typed-styles "^0.0.7"
     warning "^4.0.2"
 
-react-redux@^7.1.1:
+react-redux@^7.1.1, react-redux@^7.2.0:
   version "7.2.9"
   resolved "https://registry.yarnpkg.com/react-redux/-/react-redux-7.2.9.tgz#09488fbb9416a4efe3735b7235055442b042481d"
   integrity sha512-Gx4L3uM182jEEayZfRbI/G11ZpYdNAnBs70lFVMNdHJI76XYtR+7m0MN+eAs7UHBPhWXcnFPaS+9owSCJQHNpQ==
@@ -12372,7 +12404,7 @@ redent@^3.0.0:
     indent-string "^4.0.0"
     strip-indent "^3.0.0"
 
-redux@^4.0.0, redux@^4.0.1, redux@^4.0.5:
+redux@^4.0.0, redux@^4.0.1, redux@^4.0.4, redux@^4.0.5:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/redux/-/redux-4.2.1.tgz#c08f4306826c49b5e9dc901dee0452ea8fce6197"
   integrity sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==
@@ -13738,6 +13770,11 @@ tiny-invariant@^1.0.2:
   resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.3.1.tgz#8560808c916ef02ecfd55e66090df23a4b7aa642"
   integrity sha512-AD5ih2NlSssTCwsMznbvwMZpJ1cbhkGd2uueNxzv2jDlEeZdU04JQfRnggJQ8DrcVBGjAsCKwFBbDlVNtEMlzw==
 
+tiny-invariant@^1.0.6:
+  version "1.3.3"
+  resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.3.3.tgz#46680b7a873a0d5d10005995eb90a70d74d60127"
+  integrity sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==
+
 tiny-warning@^1.0.0, tiny-warning@^1.0.2, tiny-warning@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/tiny-warning/-/tiny-warning-1.0.3.tgz#94a30db453df4c643d0fd566060d60a875d84754"
@@ -14451,6 +14488,11 @@ use-latest@^1.2.1:
   dependencies:
     use-isomorphic-layout-effect "^1.1.1"
 
+use-memo-one@^1.1.1:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/use-memo-one/-/use-memo-one-1.1.3.tgz#2fd2e43a2169eabc7496960ace8c79efef975e99"
+  integrity sha512-g66/K7ZQGYrI6dy8GLpVcMsBp4s17xNkYJVSMvTEevGy3nDxHOfE6z8BVE22+5G5x7t3+bhzrlTDB7ObrEE0cQ==
+
 use-sidecar@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/use-sidecar/-/use-sidecar-1.1.2.tgz#2f43126ba2d7d7e117aa5855e5d8f0276dfe73c2"

From d1034483470b1cd575a712e9c249c08a863ca21e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Sun, 20 Oct 2024 20:14:27 -0700
Subject: [PATCH 089/169] Lint and a few changes

---
 .../LeaderboardConfigEditor.tsx               |  84 ++++-
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  32 +-
 .../query/configEditorQuery.ts                | 352 +++++++++---------
 3 files changed, 270 insertions(+), 198 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 960b7ce375f..ddbbaca67bc 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -17,16 +17,22 @@ import {
   Delete as DeleteIcon,
   FileCopy as CloneIcon,
 } from '@material-ui/icons';
-import React from 'react';
+import {refUri} from '@wandb/weave/react';
+import React, {useEffect, useState} from 'react';
 
 import {PythonLeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
+import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
+import {TraceObjSchema} from '../wfReactInterface/traceServerClientTypes';
+import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
 
 // Placeholder data for dropdowns
-const evaluationObjects = ['Evaluation1', 'Evaluation2', 'Evaluation3'];
+// const evaluationNames = ['Evaluation1', 'Evaluation2', 'Evaluation3'];
 const scorerNames = ['Scorer1', 'Scorer2', 'Scorer3'];
 const metricPaths = ['Metric1', 'Metric2', 'Metric3'];
 
 export const LeaderboardConfigEditor: React.FC<{
+  entity: string;
+  project: string;
   leaderboardVal: PythonLeaderboardObjectVal;
   saving: boolean;
   isDirty: boolean;
@@ -34,6 +40,8 @@ export const LeaderboardConfigEditor: React.FC<{
   discardChanges: () => void;
   commitChanges: () => void;
 }> = ({
+  entity,
+  project,
   leaderboardVal,
   saving,
   isDirty,
@@ -90,6 +98,8 @@ export const LeaderboardConfigEditor: React.FC<{
     setWorkingCopy({...leaderboardVal, columns: newColumns});
   };
 
+  const evalObjs = useEvaluationObjects(entity, project);
+
   return (
     <Box display="flex" flexDirection="column" height="100%" width="100%">
       <Box flexGrow={1} overflow="auto" p={2}>
@@ -111,8 +121,9 @@ export const LeaderboardConfigEditor: React.FC<{
           onChange={handleDescriptionChange}
           margin="normal"
           multiline
-          rows={6}
-          InputProps={{style: {fontFamily: 'monospace', minHeight: '120px'}}}
+          minRows={1}
+          maxRows={10}
+          InputProps={{style: {fontFamily: 'monospace', fontSize: '14px'}}}
         />
         <Typography variant="h6" gutterBottom style={{marginTop: 16}}>
           Columns
@@ -140,9 +151,12 @@ export const LeaderboardConfigEditor: React.FC<{
                 <MenuItem value="">
                   <em>Select Evaluation Object</em>
                 </MenuItem>
-                {evaluationObjects.map(obj => (
-                  <MenuItem key={obj} value={obj}>
-                    {obj}
+                {evalObjs.map(obj => (
+                  <MenuItem key={obj.ref} value={obj.ref}>
+                    {`${obj.name}:v${obj.versionIndex} (${obj.digest.slice(
+                      0,
+                      6
+                    )})`}
                   </MenuItem>
                 ))}
               </Select>
@@ -265,3 +279,59 @@ export const LeaderboardConfigEditor: React.FC<{
     </Box>
   );
 };
+
+type EvaluationHelperObj = {
+  obj: TraceObjSchema;
+  ref: string;
+  name: string;
+  versionIndex: number;
+  digest: string;
+};
+
+const useEvaluationObjects = (
+  entity: string,
+  project: string
+): EvaluationHelperObj[] => {
+  const getClient = useGetTraceServerClientContext();
+  const client = getClient();
+  const [evalObjs, setEvalObjs] = useState<EvaluationHelperObj[]>([]);
+  useEffect(() => {
+    let mounted = true;
+    client
+      .objsQuery({
+        project_id: projectIdFromParts({entity, project}),
+        filter: {
+          base_object_classes: ['Evaluation'],
+          is_op: false,
+          // latest_only: true,
+        },
+        metadata_only: true,
+        sort_by: [{field: 'created_at', direction: 'desc'}],
+      })
+      .then(res => {
+        if (mounted) {
+          setEvalObjs(
+            res.objs.map(obj => ({
+              obj,
+              ref: refUri({
+                scheme: 'weave',
+                entityName: entity,
+                projectName: project,
+                weaveKind: 'object',
+                artifactName: obj.object_id,
+                artifactVersion: obj.digest,
+              }),
+              name: obj.object_id,
+              versionIndex: obj.version_index,
+              digest: obj.digest,
+            }))
+          );
+        }
+      });
+
+    return () => {
+      mounted = false;
+    };
+  }, [client, entity, project]);
+  return evalObjs;
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 64c0a90c39d..c21435728ed 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -211,24 +211,24 @@ export const LeaderboardPageContentInner: React.FC<
         flexDirection="column"
         height="100%"
         minWidth="50%">
-        <Box
-          display="flex"
-          flexDirection="row"
-          maxHeight="35%"
-          width="100%"
-          sx={{
-            flex: '1 1 auto',
-            alignItems: 'flex-start',
-            padding: '12px 16px',
-            gap: '12px',
-            overflowY: 'auto',
-          }}>
-          {workingLeaderboardValCopy.description && (
+        {workingLeaderboardValCopy.description && (
+          <Box
+            display="flex"
+            flexDirection="row"
+            maxHeight="35%"
+            width="100%"
+            sx={{
+              flex: '1 1 auto',
+              alignItems: 'flex-start',
+              padding: '12px 16px',
+              gap: '12px',
+              overflowY: 'auto',
+            }}>
             <StyledReactMarkdown>
               {workingLeaderboardValCopy.description}
             </StyledReactMarkdown>
-          )}
-        </Box>
+          </Box>
+        )}
         <Box
           display="flex"
           flexDirection="row"
@@ -255,6 +255,8 @@ export const LeaderboardPageContentInner: React.FC<
             borderLeft: `1px solid ${MOON_250}`,
           }}>
           <LeaderboardConfigEditor
+            entity={props.entity}
+            project={props.project}
             saving={saving}
             isDirty={isDirty}
             leaderboardVal={workingLeaderboardValCopy}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
index 31d65bd9eee..52a784328bd 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
@@ -2,186 +2,186 @@
  * This file implements a memoized version of getLeaderboardGroupableData for improved performance.
  */
 
-import _ from 'lodash';
+// import _ from 'lodash';
 
-import {getLeaderboardGroupableData} from '../../../views/Leaderboard/query/leaderboardQuery';
-import {FilterAndGroupSpec} from '../../../views/Leaderboard/types/leaderboardConfigType';
-import {TraceServerClient} from '../../wfReactInterface/traceServerClient';
-import {projectIdFromParts} from '../../wfReactInterface/tsDataModelHooks';
+// import {getLeaderboardGroupableData} from '../../../views/Leaderboard/query/leaderboardQuery';
+// import {FilterAndGroupSpec} from '../../../views/Leaderboard/types/leaderboardConfigType';
+// import {TraceServerClient} from '../../wfReactInterface/traceServerClient';
+// import {projectIdFromParts} from '../../wfReactInterface/tsDataModelHooks';
 
 export type VersionDetails = {digest: string; index: number};
 
-const memoedGetLeaderboardGroupableData = _.memoize(
-  async (
-    client: TraceServerClient,
-    entity: string,
-    project: string,
-    spec: FilterAndGroupSpec = {}
-  ) => {
-    return getLeaderboardGroupableData(client, entity, project, spec);
-  },
-  (client, entity, project, spec) => {
-    // Create a cache key based on the function arguments
-    return JSON.stringify({entity, project, spec});
-  }
-);
+// const memoedGetLeaderboardGroupableData = _.memoize(
+//   async (
+//     client: TraceServerClient,
+//     entity: string,
+//     project: string,
+//     spec: FilterAndGroupSpec = {}
+//   ) => {
+//     return getLeaderboardGroupableData(client, entity, project, spec);
+//   },
+//   (client, entity, project, spec) => {
+//     // Create a cache key based on the function arguments
+//     return JSON.stringify({entity, project, spec});
+//   }
+// );
 
-export const fetchEvaluationNames = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string
-): Promise<string[]> => {
-  return client
-    .objsQuery({
-      project_id: projectIdFromParts({entity, project}),
-      filter: {
-        base_object_classes: ['Evaluation'],
-        is_op: false,
-        latest_only: true,
-      },
-      metadata_only: true,
-      sort_by: [{field: 'created_at', direction: 'desc'}],
-    })
-    .then(res => {
-      return res.objs.map(obj => obj.object_id);
-    });
-};
-export const fetchEvaluationVersionsForName = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  name: string
-): Promise<VersionDetails[]> => {
-  return client
-    .objsQuery({
-      project_id: projectIdFromParts({entity, project}),
-      filter: {
-        base_object_classes: ['Evaluation'],
-        is_op: false,
-        object_ids: [name],
-      },
-      metadata_only: true,
-      sort_by: [{field: 'created_at', direction: 'desc'}],
-    })
-    .then(res => {
-      return res.objs.map(obj => ({
-        digest: obj.digest,
-        index: obj.version_index,
-      }));
-    });
-};
-export const fetchDatasetNamesForSpec = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec
-): Promise<string[]> => {
-  // This is def a hacky/slow solution, just getting it working.
-  const groupableData = await memoedGetLeaderboardGroupableData(
-    client,
-    entity,
-    project,
-    spec
-  );
-  return _.uniq(groupableData.map(g => g.row.datasetName));
-};
-export const fetchDatasetVersionsForSpecAndName = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec,
-  name: string
-): Promise<VersionDetails[]> => {
-  const groupableData = await memoedGetLeaderboardGroupableData(
-    client,
-    entity,
-    project,
-    spec
-  );
+// export const fetchEvaluationNames = async (
+//   client: TraceServerClient,
+//   entity: string,
+//   project: string
+// ): Promise<string[]> => {
+//   return client
+//     .objsQuery({
+//       project_id: projectIdFromParts({entity, project}),
+//       filter: {
+//         base_object_classes: ['Evaluation'],
+//         is_op: false,
+//         latest_only: true,
+//       },
+//       metadata_only: true,
+//       sort_by: [{field: 'created_at', direction: 'desc'}],
+//     })
+//     .then(res => {
+//       return res.objs.map(obj => obj.object_id);
+//     });
+// };
+// export const fetchEvaluationVersionsForName = async (
+//   client: TraceServerClient,
+//   entity: string,
+//   project: string,
+//   name: string
+// ): Promise<VersionDetails[]> => {
+//   return client
+//     .objsQuery({
+//       project_id: projectIdFromParts({entity, project}),
+//       filter: {
+//         base_object_classes: ['Evaluation'],
+//         is_op: false,
+//         object_ids: [name],
+//       },
+//       metadata_only: true,
+//       sort_by: [{field: 'created_at', direction: 'desc'}],
+//     })
+//     .then(res => {
+//       return res.objs.map(obj => ({
+//         digest: obj.digest,
+//         index: obj.version_index,
+//       }));
+//     });
+// };
+// export const fetchDatasetNamesForSpec = async (
+//   client: TraceServerClient,
+//   entity: string,
+//   project: string,
+//   spec: FilterAndGroupSpec
+// ): Promise<string[]> => {
+//   // This is def a hacky/slow solution, just getting it working.
+//   const groupableData = await memoedGetLeaderboardGroupableData(
+//     client,
+//     entity,
+//     project,
+//     spec
+//   );
+//   return _.uniq(groupableData.map(g => g.row.datasetName));
+// };
+// export const fetchDatasetVersionsForSpecAndName = async (
+//   client: TraceServerClient,
+//   entity: string,
+//   project: string,
+//   spec: FilterAndGroupSpec,
+//   name: string
+// ): Promise<VersionDetails[]> => {
+//   const groupableData = await memoedGetLeaderboardGroupableData(
+//     client,
+//     entity,
+//     project,
+//     spec
+//   );
 
-  return _.uniqBy(
-    groupableData.map(g => ({digest: g.row.datasetVersion, index: -1})),
-    o => `${o.digest}-${o.index}`
-  );
-};
+//   return _.uniqBy(
+//     groupableData.map(g => ({digest: g.row.datasetVersion, index: -1})),
+//     o => `${o.digest}-${o.index}`
+//   );
+// };
 
-export const fetchModelNamesForSpec = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec
-): Promise<string[]> => {
-  // This is def a hacky/slow solution, just getting it working.
-  const groupableData = await memoedGetLeaderboardGroupableData(
-    client,
-    entity,
-    project,
-    spec
-  );
-  return _.uniq(groupableData.map(g => g.row.modelName));
-};
-export const fetchModelVersionsForSpecAndName = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec,
-  name: string
-): Promise<VersionDetails[]> => {
-  // This is def a hacky/slow solution, just getting it working.
-  const groupableData = await memoedGetLeaderboardGroupableData(
-    client,
-    entity,
-    project,
-    spec
-  );
-  return _.uniqBy(
-    groupableData.map(g => ({digest: g.row.modelVersion, index: -1})),
-    o => `${o.digest}-${o.index}`
-  );
-};
-export const fetchScorerNamesForSpec = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec
-): Promise<string[]> => {
-  const groupableData = await memoedGetLeaderboardGroupableData(
-    client,
-    entity,
-    project,
-    spec
-  );
-  return _.uniq(groupableData.map(g => g.row.scorerName));
-};
-export const fetchScorerVersionsForSpecAndName = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec,
-  name: string
-): Promise<VersionDetails[]> => {
-  const groupableData = await memoedGetLeaderboardGroupableData(
-    client,
-    entity,
-    project,
-    spec
-  );
-  return _.uniqBy(
-    groupableData.map(g => ({digest: g.row.scorerVersion, index: -1})),
-    o => `${o.digest}-${o.index}`
-  );
-};
-export const fetchMetricPathsForSpec = async (
-  client: TraceServerClient,
-  entity: string,
-  project: string,
-  spec: FilterAndGroupSpec
-): Promise<string[]> => {
-  const groupableData = await memoedGetLeaderboardGroupableData(
-    client,
-    entity,
-    project,
-    spec
-  );
-  return _.uniq(groupableData.map(g => g.row.metricPath));
-};
+// export const fetchModelNamesForSpec = async (
+//   client: TraceServerClient,
+//   entity: string,
+//   project: string,
+//   spec: FilterAndGroupSpec
+// ): Promise<string[]> => {
+//   // This is def a hacky/slow solution, just getting it working.
+//   const groupableData = await memoedGetLeaderboardGroupableData(
+//     client,
+//     entity,
+//     project,
+//     spec
+//   );
+//   return _.uniq(groupableData.map(g => g.row.modelName));
+// };
+// export const fetchModelVersionsForSpecAndName = async (
+//   client: TraceServerClient,
+//   entity: string,
+//   project: string,
+//   spec: FilterAndGroupSpec,
+//   name: string
+// ): Promise<VersionDetails[]> => {
+//   // This is def a hacky/slow solution, just getting it working.
+//   const groupableData = await memoedGetLeaderboardGroupableData(
+//     client,
+//     entity,
+//     project,
+//     spec
+//   );
+//   return _.uniqBy(
+//     groupableData.map(g => ({digest: g.row.modelVersion, index: -1})),
+//     o => `${o.digest}-${o.index}`
+//   );
+// };
+// export const fetchScorerNamesForSpec = async (
+//   client: TraceServerClient,
+//   entity: string,
+//   project: string,
+//   spec: FilterAndGroupSpec
+// ): Promise<string[]> => {
+//   const groupableData = await memoedGetLeaderboardGroupableData(
+//     client,
+//     entity,
+//     project,
+//     spec
+//   );
+//   return _.uniq(groupableData.map(g => g.row.scorerName));
+// };
+// export const fetchScorerVersionsForSpecAndName = async (
+//   client: TraceServerClient,
+//   entity: string,
+//   project: string,
+//   spec: FilterAndGroupSpec,
+//   name: string
+// ): Promise<VersionDetails[]> => {
+//   const groupableData = await memoedGetLeaderboardGroupableData(
+//     client,
+//     entity,
+//     project,
+//     spec
+//   );
+//   return _.uniqBy(
+//     groupableData.map(g => ({digest: g.row.scorerVersion, index: -1})),
+//     o => `${o.digest}-${o.index}`
+//   );
+// };
+// export const fetchMetricPathsForSpec = async (
+//   client: TraceServerClient,
+//   entity: string,
+//   project: string,
+//   spec: FilterAndGroupSpec
+// ): Promise<string[]> => {
+//   const groupableData = await memoedGetLeaderboardGroupableData(
+//     client,
+//     entity,
+//     project,
+//     spec
+//   );
+//   return _.uniq(groupableData.map(g => g.row.metricPath));
+// };

From 973dead51902c7c13e648760aeba68648658b5c0 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Sun, 20 Oct 2024 20:30:04 -0700
Subject: [PATCH 090/169] almost there

---
 .../LeaderboardConfigEditor.tsx               | 349 ++++++++++++------
 1 file changed, 228 insertions(+), 121 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index ddbbaca67bc..5703b24447e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -24,11 +24,7 @@ import {PythonLeaderboardObjectVal} from '../../views/Leaderboard/types/leaderbo
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {TraceObjSchema} from '../wfReactInterface/traceServerClientTypes';
 import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
-
-// Placeholder data for dropdowns
-// const evaluationNames = ['Evaluation1', 'Evaluation2', 'Evaluation3'];
-const scorerNames = ['Scorer1', 'Scorer2', 'Scorer3'];
-const metricPaths = ['Metric1', 'Metric2', 'Metric3'];
+import { parseRefMaybe } from '../../../Browse2/SmallRef';
 
 export const LeaderboardConfigEditor: React.FC<{
   entity: string;
@@ -62,6 +58,15 @@ export const LeaderboardConfigEditor: React.FC<{
   const handleColumnChange = (index: number, field: string, value: any) => {
     const newColumns = [...leaderboardVal.columns];
     newColumns[index] = {...newColumns[index], [field]: value};
+    
+    // Reset dependent fields when changing evaluation_object_ref or scorer_name
+    if (field === 'evaluation_object_ref') {
+      newColumns[index].scorer_name = '';
+      newColumns[index].summary_metric_path_parts = [];
+    } else if (field === 'scorer_name') {
+      newColumns[index].summary_metric_path_parts = [];
+    }
+    
     setWorkingCopy({...leaderboardVal, columns: newColumns});
   };
 
@@ -129,122 +134,19 @@ export const LeaderboardConfigEditor: React.FC<{
           Columns
         </Typography>
         {leaderboardVal.columns.map((column, index) => (
-          <Grid
-            container
-            spacing={2}
+          <ColumnEditor
             key={index}
-            alignItems="center"
-            style={{marginBottom: 8}}>
-            <Grid item xs={12} sm={3}>
-              <Select
-                fullWidth
-                value={column.evaluation_object_ref}
-                onChange={e =>
-                  handleColumnChange(
-                    index,
-                    'evaluation_object_ref',
-                    e.target.value
-                  )
-                }
-                displayEmpty
-                margin="dense">
-                <MenuItem value="">
-                  <em>Select Evaluation Object</em>
-                </MenuItem>
-                {evalObjs.map(obj => (
-                  <MenuItem key={obj.ref} value={obj.ref}>
-                    {`${obj.name}:v${obj.versionIndex} (${obj.digest.slice(
-                      0,
-                      6
-                    )})`}
-                  </MenuItem>
-                ))}
-              </Select>
-            </Grid>
-            <Grid item xs={12} sm={3}>
-              <Select
-                fullWidth
-                value={column.scorer_name}
-                onChange={e =>
-                  handleColumnChange(index, 'scorer_name', e.target.value)
-                }
-                displayEmpty
-                margin="dense">
-                <MenuItem value="">
-                  <em>Select Scorer</em>
-                </MenuItem>
-                {scorerNames.map(scorer => (
-                  <MenuItem key={scorer} value={scorer}>
-                    {scorer}
-                  </MenuItem>
-                ))}
-              </Select>
-            </Grid>
-            <Grid item xs={12} sm={3}>
-              <Select
-                fullWidth
-                multiple
-                value={column.summary_metric_path_parts}
-                onChange={e =>
-                  handleColumnChange(
-                    index,
-                    'summary_metric_path_parts',
-                    e.target.value
-                  )
-                }
-                renderValue={selected => (selected as string[]).join(' > ')}
-                displayEmpty
-                margin="dense">
-                <MenuItem value="">
-                  <em>Select Metric Path</em>
-                </MenuItem>
-                {metricPaths.map(path => (
-                  <MenuItem key={path} value={path}>
-                    {path}
-                  </MenuItem>
-                ))}
-              </Select>
-            </Grid>
-            <Grid item xs={6} sm={2}>
-              <FormControlLabel
-                control={
-                  <Checkbox
-                    checked={column.should_minimize}
-                    onChange={e =>
-                      handleColumnChange(
-                        index,
-                        'should_minimize',
-                        e.target.checked
-                      )
-                    }
-                  />
-                }
-                label="Minimize"
-              />
-            </Grid>
-            <Grid item xs={6} sm={1}>
-              <Box display="flex" justifyContent="flex-end">
-                <IconButton
-                  size="small"
-                  onClick={() => moveColumn(index, index - 1)}
-                  disabled={index === 0}>
-                  <MoveUpIcon />
-                </IconButton>
-                <IconButton
-                  size="small"
-                  onClick={() => moveColumn(index, index + 1)}
-                  disabled={index === leaderboardVal.columns.length - 1}>
-                  <MoveDownIcon />
-                </IconButton>
-                <IconButton size="small" onClick={() => cloneColumn(index)}>
-                  <CloneIcon />
-                </IconButton>
-                <IconButton size="small" onClick={() => removeColumn(index)}>
-                  <DeleteIcon />
-                </IconButton>
-              </Box>
-            </Grid>
-          </Grid>
+            column={column}
+            index={index}
+            evalObjs={evalObjs}
+            entity={entity}
+            project={project}
+            handleColumnChange={handleColumnChange}
+            moveColumn={moveColumn}
+            cloneColumn={cloneColumn}
+            removeColumn={removeColumn}
+            totalColumns={leaderboardVal.columns.length}
+          />
         ))}
         <Button startIcon={<AddIcon />} onClick={addColumn}>
           Add Column
@@ -280,6 +182,134 @@ export const LeaderboardConfigEditor: React.FC<{
   );
 };
 
+const ColumnEditor: React.FC<{
+  column: PythonLeaderboardObjectVal['columns'][0];
+  index: number;
+  evalObjs: EvaluationHelperObj[];
+  entity: string;
+  project: string;
+  handleColumnChange: (index: number, field: string, value: any) => void;
+  moveColumn: (fromIndex: number, toIndex: number) => void;
+  cloneColumn: (index: number) => void;
+  removeColumn: (index: number) => void;
+  totalColumns: number;
+}> = ({
+  column,
+  index,
+  evalObjs,
+  entity,
+  project,
+  handleColumnChange,
+  moveColumn,
+  cloneColumn,
+  removeColumn,
+  totalColumns,
+}) => {
+  const scorers = useScorers(entity, project, column.evaluation_object_ref);
+  const metrics = useMetrics(entity, project, column.evaluation_object_ref, column.scorer_name);
+
+  return (
+    <Grid container spacing={2} alignItems="center" style={{marginBottom: 8}}>
+      <Grid item xs={12} sm={3}>
+        <Select
+          fullWidth
+          value={column.evaluation_object_ref}
+          onChange={e =>
+            handleColumnChange(index, 'evaluation_object_ref', e.target.value)
+          }
+          displayEmpty
+          margin="dense">
+          <MenuItem value="">
+            <em>Select Evaluation Object</em>
+          </MenuItem>
+          {evalObjs.map(obj => (
+            <MenuItem key={obj.ref} value={obj.ref}>
+              {`${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`}
+            </MenuItem>
+          ))}
+        </Select>
+      </Grid>
+      <Grid item xs={12} sm={3}>
+        <Select
+          fullWidth
+          value={column.scorer_name}
+          onChange={e =>
+            handleColumnChange(index, 'scorer_name', e.target.value)
+          }
+          displayEmpty
+          margin="dense"
+          disabled={!column.evaluation_object_ref}>
+          <MenuItem value="">
+            <em>Select Scorer</em>
+          </MenuItem>
+          {scorers.map(scorer => (
+            <MenuItem key={scorer} value={scorer}>
+              {scorer}
+            </MenuItem>
+          ))}
+        </Select>
+      </Grid>
+      <Grid item xs={12} sm={3}>
+        <Select
+          fullWidth
+          multiple
+          value={column.summary_metric_path_parts}
+          onChange={e =>
+            handleColumnChange(index, 'summary_metric_path_parts', e.target.value)
+          }
+          renderValue={selected => (selected as string[]).join(' > ')}
+          displayEmpty
+          margin="dense"
+          disabled={!column.evaluation_object_ref || !column.scorer_name}>
+          <MenuItem value="">
+            <em>Select Metric Path</em>
+          </MenuItem>
+          {metrics.map(path => (
+            <MenuItem key={path} value={path}>
+              {path}
+            </MenuItem>
+          ))}
+        </Select>
+      </Grid>
+      <Grid item xs={6} sm={2}>
+        <FormControlLabel
+          control={
+            <Checkbox
+              checked={column.should_minimize}
+              onChange={e =>
+                handleColumnChange(index, 'should_minimize', e.target.checked)
+              }
+            />
+          }
+          label="Minimize"
+        />
+      </Grid>
+      <Grid item xs={6} sm={1}>
+        <Box display="flex" justifyContent="flex-end">
+          <IconButton
+            size="small"
+            onClick={() => moveColumn(index, index - 1)}
+            disabled={index === 0}>
+            <MoveUpIcon />
+          </IconButton>
+          <IconButton
+            size="small"
+            onClick={() => moveColumn(index, index + 1)}
+            disabled={index === totalColumns - 1}>
+            <MoveDownIcon />
+          </IconButton>
+          <IconButton size="small" onClick={() => cloneColumn(index)}>
+            <CloneIcon />
+          </IconButton>
+          <IconButton size="small" onClick={() => removeColumn(index)}>
+            <DeleteIcon />
+          </IconButton>
+        </Box>
+      </Grid>
+    </Grid>
+  );
+};
+
 type EvaluationHelperObj = {
   obj: TraceObjSchema;
   ref: string;
@@ -303,7 +333,6 @@ const useEvaluationObjects = (
         filter: {
           base_object_classes: ['Evaluation'],
           is_op: false,
-          // latest_only: true,
         },
         metadata_only: true,
         sort_by: [{field: 'created_at', direction: 'desc'}],
@@ -335,3 +364,81 @@ const useEvaluationObjects = (
   }, [client, entity, project]);
   return evalObjs;
 };
+
+const useScorers = (
+  entity: string,
+  project: string,
+  evaluationObjectRef: string
+): string[] => {
+  const getClient = useGetTraceServerClientContext();
+  const client = getClient();
+  const [scorers, setScorers] = useState<string[]>([]);
+
+  useEffect(() => {
+    if (!evaluationObjectRef) {
+      setScorers([]);
+      return;
+    }
+
+    let mounted = true;
+    client
+      .readBatch({refs: [evaluationObjectRef]})
+      .then(res => {
+        console.log(res);
+        if (mounted) {
+          setScorers((res.vals[0].scorers ?? []).map((scorer: string) => parseRefMaybe(scorer)?.artifactName).filter(Boolean) as string[]);
+        }
+      });
+
+    return () => {
+      mounted = false;
+    };
+  }, [client, entity, project, evaluationObjectRef]);
+
+  return scorers;
+};
+
+const useMetrics = (
+  entity: string,
+  project: string,
+  evaluationObjectRef: string,
+  scorerName: string
+): string[] => {
+  const getClient = useGetTraceServerClientContext();
+  const client = getClient();
+  const [metrics, setMetrics] = useState<string[]>([]);
+
+  useEffect(() => {
+    if (!evaluationObjectRef || !scorerName) {
+      setMetrics([]);
+      return;
+    }
+
+    let mounted = true;
+    // TODO: Implement the actual API call to fetch metrics for the given evaluation object and scorer
+    // This is a placeholder implementation
+    // client
+    //   .someOtherApiCall(evaluationObjectRef, scorerName)
+    //   .then(res => {
+    //     if (mounted) {
+    //       setMetrics(res.metrics);
+    //     }
+    //   });
+
+    return () => {
+      mounted = false;
+    };
+  }, [client, entity, project, evaluationObjectRef, scorerName]);
+
+  return metrics;
+};
+
+
+
+// TODO:
+// - [ ] Create new leaderboard
+// - [ ] Wire up metric path dropdowns
+// - [ ] flip the default sort for minimization if it is first
+// - [ ] Ordering of the columns should respect the columns in config (due to grouping) - maybe we should allow splitting of groups?
+// - [ ] Handle non-leaf metric names?
+

From c68d3e520c94ca38f0877aa0ed4dee976316df4d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Sun, 20 Oct 2024 23:12:14 -0700
Subject: [PATCH 091/169] working

---
 .../LeaderboardConfigEditor.tsx               | 102 +++++++++++++-----
 1 file changed, 73 insertions(+), 29 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 5703b24447e..29b6d74f699 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -20,11 +20,17 @@ import {
 import {refUri} from '@wandb/weave/react';
 import React, {useEffect, useState} from 'react';
 
-import {PythonLeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
+import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
+import {parseRefMaybe} from '../../../Browse2/SmallRef';
+import {
+  ALL_VALUE,
+  PythonLeaderboardObjectVal,
+} from '../../views/Leaderboard/types/leaderboardConfigType';
+import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {TraceObjSchema} from '../wfReactInterface/traceServerClientTypes';
 import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
-import { parseRefMaybe } from '../../../Browse2/SmallRef';
+import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
 
 export const LeaderboardConfigEditor: React.FC<{
   entity: string;
@@ -58,7 +64,7 @@ export const LeaderboardConfigEditor: React.FC<{
   const handleColumnChange = (index: number, field: string, value: any) => {
     const newColumns = [...leaderboardVal.columns];
     newColumns[index] = {...newColumns[index], [field]: value};
-    
+
     // Reset dependent fields when changing evaluation_object_ref or scorer_name
     if (field === 'evaluation_object_ref') {
       newColumns[index].scorer_name = '';
@@ -66,7 +72,7 @@ export const LeaderboardConfigEditor: React.FC<{
     } else if (field === 'scorer_name') {
       newColumns[index].summary_metric_path_parts = [];
     }
-    
+
     setWorkingCopy({...leaderboardVal, columns: newColumns});
   };
 
@@ -206,7 +212,12 @@ const ColumnEditor: React.FC<{
   totalColumns,
 }) => {
   const scorers = useScorers(entity, project, column.evaluation_object_ref);
-  const metrics = useMetrics(entity, project, column.evaluation_object_ref, column.scorer_name);
+  const metrics = useMetrics(
+    entity,
+    project,
+    column.evaluation_object_ref,
+    column.scorer_name
+  );
 
   return (
     <Grid container spacing={2} alignItems="center" style={{marginBottom: 8}}>
@@ -252,12 +263,14 @@ const ColumnEditor: React.FC<{
       <Grid item xs={12} sm={3}>
         <Select
           fullWidth
-          multiple
-          value={column.summary_metric_path_parts}
+          value={column.summary_metric_path_parts.join('.')}
           onChange={e =>
-            handleColumnChange(index, 'summary_metric_path_parts', e.target.value)
+            handleColumnChange(
+              index,
+              'summary_metric_path_parts',
+              (e.target.value as string).split('.')
+            )
           }
-          renderValue={selected => (selected as string[]).join(' > ')}
           displayEmpty
           margin="dense"
           disabled={!column.evaluation_object_ref || !column.scorer_name}>
@@ -381,14 +394,16 @@ const useScorers = (
     }
 
     let mounted = true;
-    client
-      .readBatch({refs: [evaluationObjectRef]})
-      .then(res => {
-        console.log(res);
-        if (mounted) {
-          setScorers((res.vals[0].scorers ?? []).map((scorer: string) => parseRefMaybe(scorer)?.artifactName).filter(Boolean) as string[]);
-        }
-      });
+    client.readBatch({refs: [evaluationObjectRef]}).then(res => {
+      console.log(res);
+      if (mounted) {
+        setScorers(
+          (res.vals[0].scorers ?? [])
+            .map((scorer: string) => parseRefMaybe(scorer)?.artifactName)
+            .filter(Boolean) as string[]
+        );
+      }
+    });
 
     return () => {
       mounted = false;
@@ -417,13 +432,47 @@ const useMetrics = (
     let mounted = true;
     // TODO: Implement the actual API call to fetch metrics for the given evaluation object and scorer
     // This is a placeholder implementation
-    // client
-    //   .someOtherApiCall(evaluationObjectRef, scorerName)
-    //   .then(res => {
-    //     if (mounted) {
-    //       setMetrics(res.metrics);
-    //     }
-    //   });
+    client
+      .callsStreamQuery({
+        project_id: projectIdFromParts({entity, project}),
+        filter: {
+          op_names: [
+            opVersionKeyToRefUri({
+              entity,
+              project,
+              opId: EVALUATE_OP_NAME_POST_PYDANTIC,
+              versionHash: ALL_VALUE,
+            }),
+          ],
+          input_refs: [evaluationObjectRef],
+        },
+        limit: 1,
+      })
+      .then(res => {
+        if (mounted) {
+          const calls = res.calls;
+          if (calls.length === 0) {
+            setMetrics([]);
+          } else {
+            const output = calls[0].output ?? {};
+            if (
+              typeof output === 'object' &&
+              output != null &&
+              scorerName in output
+            ) {
+              setMetrics(
+                Object.keys(
+                  flattenObjectPreservingWeaveTypes(
+                    (output as Record<string, any>)[scorerName]
+                  )
+                )
+              );
+            } else {
+              setMetrics([]);
+            }
+          }
+        }
+      });
 
     return () => {
       mounted = false;
@@ -433,12 +482,7 @@ const useMetrics = (
   return metrics;
 };
 
-
-
 // TODO:
 // - [ ] Create new leaderboard
-// - [ ] Wire up metric path dropdowns
 // - [ ] flip the default sort for minimization if it is first
 // - [ ] Ordering of the columns should respect the columns in config (due to grouping) - maybe we should allow splitting of groups?
-// - [ ] Handle non-leaf metric names?
-

From f1b6b727672014e4bea0134bc38654d69e3b3ea5 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Sun, 20 Oct 2024 23:43:12 -0700
Subject: [PATCH 092/169] nearly complete

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 24 +++++++++-
 .../views/Leaderboard/LeaderboardGrid.tsx     | 29 +++++++++---
 .../views/Leaderboard/query/hookAdapters.ts   | 14 ++++--
 .../Leaderboard/query/leaderboardQuery.ts     | 45 +++++++++++++++----
 4 files changed, 94 insertions(+), 18 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index c21435728ed..f42abb33b64 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -171,7 +171,7 @@ export const LeaderboardPageContentInner: React.FC<
   useEffect(() => {
     props.setName(workingLeaderboardValCopy.name);
   }, [props, workingLeaderboardValCopy.name]);
-  const {loading, data} = usePythonLeaderboardData(
+  const {loading, data, evalData} = usePythonLeaderboardData(
     props.entity,
     props.project,
     workingLeaderboardValCopy
@@ -202,6 +202,27 @@ export const LeaderboardPageContentInner: React.FC<
   const isDirty = useMemo(() => {
     return !_.isEqual(props.leaderboardVal, workingLeaderboardValCopy);
   }, [props.leaderboardVal, workingLeaderboardValCopy]);
+  const columnOrder = useMemo(() => {
+    return workingLeaderboardValCopy.columns
+      .map(col => {
+        const datasetGroup = evalData[col.evaluation_object_ref]?.datasetGroup;
+        const scorerGroup =
+          evalData[col.evaluation_object_ref]?.scorers[col.scorer_name];
+        const metricGroup = col.summary_metric_path_parts.join('.');
+        console.log({datasetGroup, scorerGroup, metricGroup});
+        if (datasetGroup && scorerGroup && metricGroup) {
+          return {
+            datasetGroup,
+            scorerGroup,
+            metricGroup,
+          };
+        }
+        return null;
+      })
+      .filter(c => c != null);
+  }, [workingLeaderboardValCopy, evalData]);
+
+  console.log({columnOrder});
 
   return (
     <Box display="flex" flexDirection="row" height="100%" flexGrow={1}>
@@ -241,6 +262,7 @@ export const LeaderboardPageContentInner: React.FC<
             project={props.project}
             loading={loading}
             data={data}
+            columnOrder={columnOrder}
           />
         </Box>
       </Box>
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index 8e032736b63..02dc4463ba2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -28,6 +28,11 @@ interface LeaderboardGridProps {
   entity: string;
   project: string;
   data: GroupedLeaderboardData;
+  columnOrder?: Array<{
+    datasetGroup: string;
+    scorerGroup: string;
+    metricGroup: string;
+  }>;
   loading: boolean;
 }
 
@@ -42,6 +47,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
   project,
   data,
   loading,
+  columnOrder,
 }) => {
   const {peekingRouter} = useWeaveflowRouteContext();
   const history = useHistory();
@@ -251,6 +257,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         const datasetColGroup: GridColumnGroup = {
           groupId: datasetGroupName,
           headerName: datasetGroupName,
+          freeReordering: true,
           children: [],
           renderHeaderGroup: params => {
             const ref = parseRefMaybe(
@@ -268,6 +275,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
             const scorerColGroup: GridColumnGroup = {
               groupId: `${datasetGroupName}--${scorerGroupName}`,
               headerName: scorerGroupName,
+              freeReordering: true,
               children: [],
               renderHeaderGroup: params => {
                 const ref = parseRefMaybe(
@@ -306,6 +314,20 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     }
   }, [columns, loading, sortModel]);
 
+  const orderedColumns = useMemo(() => {
+    if (!columnOrder) {
+      return columns;
+    }
+    const columnOrderKeys = columnOrder.map(
+      c => `${c.datasetGroup}--${c.scorerGroup}--${c.metricGroup}`
+    );
+    return columns.sort((a, b) => {
+      return (
+        columnOrderKeys.indexOf(a.field) - columnOrderKeys.indexOf(b.field)
+      );
+    });
+  }, [columns, columnOrder]);
+
   return (
     <Box
       sx={{
@@ -317,9 +339,10 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
       }}>
       <StyledDataGrid
         rows={rows}
-        columns={columns}
+        columns={orderedColumns}
         columnGroupingModel={groupingModel}
         disableRowSelectionOnClick
+        disableColumnReorder
         hideFooterSelectedRowCount
         disableMultipleColumnsSorting={false}
         columnHeaderHeight={40}
@@ -523,7 +546,3 @@ const defaultGetSortComparator =
     return aValue.localeCompare(bValue);
   };
 
-/**
-TODO:
-* [ ] When models are Ops, their link is not quite right
- */
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
index 37aaf6daf9c..34a492f356a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
@@ -10,6 +10,7 @@ import {
   getLeaderboardData,
   getPythonLeaderboardData,
   GroupedLeaderboardData,
+  PythonLeaderboardEvalData,
 } from './leaderboardQuery';
 
 type LeaderboardDataState = {
@@ -48,11 +49,14 @@ export const usePythonLeaderboardData = (
   entity: string,
   project: string,
   val: PythonLeaderboardObjectVal
-): LeaderboardDataState => {
+): LeaderboardDataState & {evalData: PythonLeaderboardEvalData} => {
   const getTraceServerClient = useGetTraceServerClientContext();
-  const [state, setState] = useState<LeaderboardDataState>({
+  const [state, setState] = useState<
+    LeaderboardDataState & {evalData: PythonLeaderboardEvalData}
+  >({
     loading: true,
     data: {modelGroups: {}},
+    evalData: {},
   });
   const deepVal = useDeepMemo(val);
   useEffect(() => {
@@ -64,7 +68,11 @@ export const usePythonLeaderboardData = (
       deepVal
     ).then(data => {
       if (mounted) {
-        setState({loading: false, data});
+        setState({
+          loading: false,
+          data: data.finalData,
+          evalData: data.evalData,
+        });
       }
     });
     return () => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
index ace9958c25c..7c6eaff0427 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
@@ -493,8 +493,11 @@ export const getPythonLeaderboardData = async (
   entity: string,
   project: string,
   val: PythonLeaderboardObjectVal
-): Promise<GroupedLeaderboardData> => {
-  const groupableData = await getPythonLeaderboardGroupableData(
+): Promise<{
+  finalData: GroupedLeaderboardData;
+  evalData: PythonLeaderboardEvalData;
+}> => {
+  const {groupableData, evalData} = await getPythonLeaderboardGroupableData(
     client,
     entity,
     project,
@@ -531,7 +534,16 @@ export const getPythonLeaderboardData = async (
     ),
   };
 
-  return finalData;
+  return {finalData, evalData};
+};
+
+export type PythonLeaderboardEvalData = {
+  [evalRefUri: string]: {
+    datasetGroup: string;
+    scorers: {
+      [scorerName: string]: string;
+    };
+  };
 };
 
 const getPythonLeaderboardGroupableData = async (
@@ -539,13 +551,16 @@ const getPythonLeaderboardGroupableData = async (
   entity: string,
   project: string,
   val: PythonLeaderboardObjectVal
-): Promise<GroupableLeaderboardValueRecord[]> => {
+): Promise<{
+  groupableData: GroupableLeaderboardValueRecord[];
+  evalData: PythonLeaderboardEvalData;
+}> => {
   const evalObjectRefs = _.uniq(
     val.columns.map(col => col.evaluation_object_ref)
   ).filter(ref => parseRefMaybe(ref)?.scheme === 'weave');
 
   if (evalObjectRefs.length === 0) {
-    return [];
+    return {groupableData: [], evalData: {}};
   }
 
   const evalObjectRefsValsProm = client.readBatch({refs: evalObjectRefs});
@@ -572,6 +587,7 @@ const getPythonLeaderboardGroupableData = async (
   const allEvaluationCallsRes = await allEvaluationCallsProm;
 
   const data: GroupableLeaderboardValueRecord[] = [];
+  const evalData: PythonLeaderboardEvalData = {};
   allEvaluationCallsRes.calls.forEach(call => {
     val.columns.forEach(col => {
       const evalObjRefUri = call.inputs.self;
@@ -617,10 +633,13 @@ const getPythonLeaderboardGroupableData = async (
             value = (value as any)[part];
           }
         });
+        const modelGroup = `${modelRef.artifactName}:${modelRef.artifactVersion}`;
+        const datasetGroup = `${datasetRef.artifactName}:${datasetRef.artifactVersion}`;
+        const scorerGroup = `${scorerRef.artifactName}:${scorerRef.artifactVersion}`;
         const row: GroupableLeaderboardValueRecord = {
-          modelGroup: `${modelRef.artifactName}:${modelRef.artifactVersion}`,
-          datasetGroup: `${datasetRef.artifactName}:${datasetRef.artifactVersion}`,
-          scorerGroup: `${scorerRef.artifactName}:${scorerRef.artifactVersion}`,
+          modelGroup,
+          datasetGroup,
+          scorerGroup,
           metricPathGroup: col.summary_metric_path_parts.join('.'),
           sortKey: -convertISOToDate(call.started_at).getTime(),
           row: {
@@ -642,9 +661,17 @@ const getPythonLeaderboardGroupableData = async (
           },
         };
         data.push(row);
+        if (!(col.evaluation_object_ref in evalData)) {
+          evalData[col.evaluation_object_ref] = {
+            datasetGroup,
+            scorers: {},
+          };
+        }
+        evalData[col.evaluation_object_ref].scorers[scorerRef.artifactName] =
+          scorerGroup;
       }
     });
   });
 
-  return data;
+  return {groupableData: data, evalData};
 };

From 3f7669c89d465752cdffa1eca4e0f96ef73396b5 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 21 Oct 2024 00:07:31 -0700
Subject: [PATCH 093/169] everything done except make now

---
 .../LeaderboardConfigEditor.tsx               | 203 +++++++++---------
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  12 +-
 .../views/Leaderboard/LeaderboardGrid.tsx     |  37 ++--
 3 files changed, 135 insertions(+), 117 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 29b6d74f699..8b753c6e510 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -3,7 +3,6 @@ import {
   Button,
   Checkbox,
   FormControlLabel,
-  Grid,
   IconButton,
   MenuItem,
   Select,
@@ -220,106 +219,114 @@ const ColumnEditor: React.FC<{
   );
 
   return (
-    <Grid container spacing={2} alignItems="center" style={{marginBottom: 8}}>
-      <Grid item xs={12} sm={3}>
-        <Select
-          fullWidth
-          value={column.evaluation_object_ref}
-          onChange={e =>
-            handleColumnChange(index, 'evaluation_object_ref', e.target.value)
-          }
-          displayEmpty
-          margin="dense">
-          <MenuItem value="">
-            <em>Select Evaluation Object</em>
-          </MenuItem>
-          {evalObjs.map(obj => (
-            <MenuItem key={obj.ref} value={obj.ref}>
-              {`${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`}
+    <Box
+      display="flex"
+      flexWrap="wrap"
+      alignItems="center"
+      mb={2}
+      p={2}
+      border={1}
+      borderColor="divider"
+      borderRadius={4}>
+      <Box flexGrow={1} display="flex" flexWrap="wrap" alignItems="center">
+        <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
+          <Select
+            fullWidth
+            value={column.evaluation_object_ref}
+            onChange={e =>
+              handleColumnChange(index, 'evaluation_object_ref', e.target.value)
+            }
+            displayEmpty
+            margin="dense">
+            <MenuItem value="">
+              <em>Select Evaluation Object</em>
             </MenuItem>
-          ))}
-        </Select>
-      </Grid>
-      <Grid item xs={12} sm={3}>
-        <Select
-          fullWidth
-          value={column.scorer_name}
-          onChange={e =>
-            handleColumnChange(index, 'scorer_name', e.target.value)
-          }
-          displayEmpty
-          margin="dense"
-          disabled={!column.evaluation_object_ref}>
-          <MenuItem value="">
-            <em>Select Scorer</em>
-          </MenuItem>
-          {scorers.map(scorer => (
-            <MenuItem key={scorer} value={scorer}>
-              {scorer}
+            {evalObjs.map(obj => (
+              <MenuItem key={obj.ref} value={obj.ref}>
+                {`${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`}
+              </MenuItem>
+            ))}
+          </Select>
+        </Box>
+        <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
+          <Select
+            fullWidth
+            value={column.scorer_name}
+            onChange={e =>
+              handleColumnChange(index, 'scorer_name', e.target.value)
+            }
+            displayEmpty
+            margin="dense"
+            disabled={!column.evaluation_object_ref}>
+            <MenuItem value="">
+              <em>Select Scorer</em>
             </MenuItem>
-          ))}
-        </Select>
-      </Grid>
-      <Grid item xs={12} sm={3}>
-        <Select
-          fullWidth
-          value={column.summary_metric_path_parts.join('.')}
-          onChange={e =>
-            handleColumnChange(
-              index,
-              'summary_metric_path_parts',
-              (e.target.value as string).split('.')
-            )
-          }
-          displayEmpty
-          margin="dense"
-          disabled={!column.evaluation_object_ref || !column.scorer_name}>
-          <MenuItem value="">
-            <em>Select Metric Path</em>
-          </MenuItem>
-          {metrics.map(path => (
-            <MenuItem key={path} value={path}>
-              {path}
+            {scorers.map(scorer => (
+              <MenuItem key={scorer} value={scorer}>
+                {scorer}
+              </MenuItem>
+            ))}
+          </Select>
+        </Box>
+        <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
+          <Select
+            fullWidth
+            value={column.summary_metric_path_parts.join('.')}
+            onChange={e =>
+              handleColumnChange(
+                index,
+                'summary_metric_path_parts',
+                (e.target.value as string).split('.')
+              )
+            }
+            displayEmpty
+            margin="dense"
+            disabled={!column.evaluation_object_ref || !column.scorer_name}>
+            <MenuItem value="">
+              <em>Select Metric Path</em>
             </MenuItem>
-          ))}
-        </Select>
-      </Grid>
-      <Grid item xs={6} sm={2}>
-        <FormControlLabel
-          control={
-            <Checkbox
-              checked={column.should_minimize}
-              onChange={e =>
-                handleColumnChange(index, 'should_minimize', e.target.checked)
-              }
-            />
-          }
-          label="Minimize"
-        />
-      </Grid>
-      <Grid item xs={6} sm={1}>
-        <Box display="flex" justifyContent="flex-end">
-          <IconButton
-            size="small"
-            onClick={() => moveColumn(index, index - 1)}
-            disabled={index === 0}>
-            <MoveUpIcon />
-          </IconButton>
-          <IconButton
-            size="small"
-            onClick={() => moveColumn(index, index + 1)}
-            disabled={index === totalColumns - 1}>
-            <MoveDownIcon />
-          </IconButton>
-          <IconButton size="small" onClick={() => cloneColumn(index)}>
-            <CloneIcon />
-          </IconButton>
-          <IconButton size="small" onClick={() => removeColumn(index)}>
-            <DeleteIcon />
-          </IconButton>
+            {metrics.map(path => (
+              <MenuItem key={path} value={path}>
+                {path}
+              </MenuItem>
+            ))}
+          </Select>
         </Box>
-      </Grid>
-    </Grid>
+        <Box display="flex" alignItems="center" mb={2} mr={2}>
+          <FormControlLabel
+            control={
+              <Checkbox
+                checked={column.should_minimize}
+                onChange={e =>
+                  handleColumnChange(index, 'should_minimize', e.target.checked)
+                }
+              />
+            }
+            label="Minimize"
+          />
+        </Box>
+      </Box>
+      <Box display="flex" justifyContent="flex-end" mb={2}>
+        <IconButton
+          size="small"
+          onClick={() => moveColumn(index, index - 1)}
+          disabled={index === 0}>
+          <MoveUpIcon />
+        </IconButton>
+        <IconButton
+          size="small"
+          onClick={() => moveColumn(index, index + 1)}
+          disabled={index === totalColumns - 1}>
+          <MoveDownIcon />
+        </IconButton>
+        <IconButton size="small" onClick={() => cloneColumn(index)}>
+          <CloneIcon />
+        </IconButton>
+        <IconButton size="small" onClick={() => removeColumn(index)}>
+          <DeleteIcon />
+        </IconButton>
+      </Box>
+    </Box>
   );
 };
 
@@ -484,5 +491,3 @@ const useMetrics = (
 
 // TODO:
 // - [ ] Create new leaderboard
-// - [ ] flip the default sort for minimization if it is first
-// - [ ] Ordering of the columns should respect the columns in config (due to grouping) - maybe we should allow splitting of groups?
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index f42abb33b64..8c3c149a2ec 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -16,7 +16,10 @@ import React, {
 
 import {WeaveflowPeekContext} from '../../context';
 import {NotFoundPanel} from '../../NotFoundPanel';
-import {LeaderboardGrid} from '../../views/Leaderboard/LeaderboardGrid';
+import {
+  LeaderboardColumnOrderType,
+  LeaderboardGrid,
+} from '../../views/Leaderboard/LeaderboardGrid';
 import {usePythonLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
 import {PythonLeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {SimplePageLayout} from '../common/SimplePageLayout';
@@ -209,21 +212,20 @@ export const LeaderboardPageContentInner: React.FC<
         const scorerGroup =
           evalData[col.evaluation_object_ref]?.scorers[col.scorer_name];
         const metricGroup = col.summary_metric_path_parts.join('.');
-        console.log({datasetGroup, scorerGroup, metricGroup});
+
         if (datasetGroup && scorerGroup && metricGroup) {
           return {
             datasetGroup,
             scorerGroup,
             metricGroup,
+            minimize: col.should_minimize ?? false,
           };
         }
         return null;
       })
-      .filter(c => c != null);
+      .filter(c => c != null) as LeaderboardColumnOrderType;
   }, [workingLeaderboardValCopy, evalData]);
 
-  console.log({columnOrder});
-
   return (
     <Box display="flex" flexDirection="row" height="100%" flexGrow={1}>
       <Box
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index 02dc4463ba2..97e95189edd 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -23,16 +23,17 @@ import {
 } from './query/leaderboardQuery';
 
 const USE_COMPARE_EVALUATIONS_PAGE = true;
-
+export type LeaderboardColumnOrderType = Array<{
+  datasetGroup: string;
+  scorerGroup: string;
+  metricGroup: string;
+  minimize: boolean;
+}>;
 interface LeaderboardGridProps {
   entity: string;
   project: string;
   data: GroupedLeaderboardData;
-  columnOrder?: Array<{
-    datasetGroup: string;
-    scorerGroup: string;
-    metricGroup: string;
-  }>;
+  columnOrder?: LeaderboardColumnOrderType;
   loading: boolean;
 }
 
@@ -308,12 +309,6 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
 
   const [sortModel, setSortModel] = useState<GridSortItem[]>([]);
 
-  useEffect(() => {
-    if (sortModel.length === 0 && columns.length > 1 && !loading) {
-      setSortModel([{field: columns[1].field, sort: 'desc'}]);
-    }
-  }, [columns, loading, sortModel]);
-
   const orderedColumns = useMemo(() => {
     if (!columnOrder) {
       return columns;
@@ -328,6 +323,23 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     });
   }, [columns, columnOrder]);
 
+  const defaultSortModel: GridSortItem[] = useMemo(() => {
+    if (!columnOrder) {
+      return columns.map(c => ({field: c.field, sort: 'desc'}));
+    } else {
+      return columnOrder.map(c => ({
+        field: `${c.datasetGroup}--${c.scorerGroup}--${c.metricGroup}`,
+        sort: c.minimize ? 'asc' : 'desc',
+      }));
+    }
+  }, [columnOrder, columns]);
+
+  useEffect(() => {
+    if (columns.length > 1 && !loading) {
+      setSortModel(defaultSortModel);
+    }
+  }, [columns, defaultSortModel, loading]);
+
   return (
     <Box
       sx={{
@@ -545,4 +557,3 @@ const defaultGetSortComparator =
     }
     return aValue.localeCompare(bValue);
   };
-

From f210380fad4cb2206a706b5ddf333a59d42285d5 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 21 Oct 2024 00:25:18 -0700
Subject: [PATCH 094/169] Ok, all the building blocks are now there

---
 .../PagePanelComponents/Home/Browse3.tsx      |  3 +
 .../Home/Browse3/context.tsx                  | 11 ++--
 .../LeaderboardConfigEditor.tsx               |  3 -
 .../LeaderboardListingPage.tsx                | 64 ++++++++++++++++++-
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  8 ++-
 .../Home/Browse3/pages/ObjectVersionsPage.tsx |  7 +-
 6 files changed, 84 insertions(+), 12 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
index 49adab6af2e..f23fbf97c58 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3.tsx
@@ -986,6 +986,8 @@ const CompareEvaluationsBinding = () => {
 const LeaderboardPageBinding = () => {
   const params = useParamsDecoded<Browse3TabItemParams>();
   const {entity, project, itemName: leaderboardName} = params;
+  const query = useURLSearchParamsDict();
+  const edit = query.edit === 'true';
   if (!leaderboardName) {
     return <LeaderboardListingPage entity={entity} project={project} />;
   }
@@ -994,6 +996,7 @@ const LeaderboardPageBinding = () => {
       entity={entity}
       project={project}
       leaderboardName={leaderboardName}
+      openEditorOnMount={edit}
     />
   );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx
index 44f207bbb5e..f453e583c54 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx
@@ -196,7 +196,8 @@ export const browse2Context = {
   leaderboardsUIUrl: (
     entityName: string,
     projectName: string,
-    leaderboardName?: string
+    leaderboardName?: string,
+    edit?: boolean
   ) => {
     throw new Error('Not implemented');
   },
@@ -432,11 +433,12 @@ export const browse3ContextGen = (
     leaderboardsUIUrl: (
       entityName: string,
       projectName: string,
-      leaderboardName?: string
+      leaderboardName?: string,
+      edit?: boolean
     ) => {
       return `${projectRoot(entityName, projectName)}/leaderboards${
         leaderboardName ? `/${leaderboardName}` : ''
-      }`;
+      }${edit ? '?edit=true' : ''}`;
     },
   };
   return browse3Context;
@@ -525,7 +527,8 @@ type RouteType = {
   leaderboardsUIUrl: (
     entityName: string,
     projectName: string,
-    leaderboardName?: string
+    leaderboardName?: string,
+    edit?: boolean
   ) => string;
 };
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 8b753c6e510..be598d0fb37 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -488,6 +488,3 @@ const useMetrics = (
 
   return metrics;
 };
-
-// TODO:
-// - [ ] Create new leaderboard
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index de42cf9bfd8..d225baf2ba4 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -12,6 +12,8 @@ import {EMPTY_PROPS_LEADERBOARDS} from '../common/EmptyContent';
 import {SimplePageLayout} from '../common/SimplePageLayout';
 import {ObjectVersionsTable} from '../ObjectVersionsPage';
 import {useWFHooks} from '../wfReactInterface/context';
+import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
+import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
 import {ObjectVersionSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 
 const Container = styled.div`
@@ -34,7 +36,12 @@ export const LeaderboardListingPage: React.FC<{
           content: <LeaderboardListingPageInner {...props} />,
         },
       ]}
-      headerExtra={<CreateLeaderboardButton />}
+      headerExtra={
+        <CreateLeaderboardButton
+          entity={props.entity}
+          project={props.project}
+        />
+      }
     />
   );
 };
@@ -50,7 +57,15 @@ export const LeaderboardListingPageInner: React.FC<{
   );
 };
 
-const CreateLeaderboardButton: FC = () => {
+const CreateLeaderboardButton: FC<{
+  entity: string;
+  project: string;
+}> = props => {
+  const createLeaderboard = useCreateLeaderboard(props.entity, props.project);
+  const navigateToLeaderboard = useNavigateToLeaderboard(
+    props.entity,
+    props.project
+  );
   return (
     <Box
       sx={{
@@ -65,7 +80,9 @@ const CreateLeaderboardButton: FC = () => {
         }}
         size="medium"
         variant="secondary"
-        onClick={() => console.log('create leaderboard')}
+        onClick={() => {
+          createLeaderboard().then(navigateToLeaderboard);
+        }}
         icon="add-new">
         Create Leaderboard
       </Button>
@@ -125,3 +142,44 @@ const LeaderboardTable: React.FC<{
     />
   );
 };
+
+const useCreateLeaderboard = (entity: string, project: string) => {
+  const getTsClient = useGetTraceServerClientContext();
+  const client = getTsClient();
+
+  const createLeaderboard = async () => {
+    const objectId = `leaderboard-${new Date()
+      .toISOString()
+      .replace(/[:.]/g, '-')}`;
+    await client.objCreate({
+      obj: {
+        project_id: projectIdFromParts({entity, project}),
+        object_id: objectId,
+        val: {
+          _type: 'Leaderboard',
+          _class_name: 'Leaderboard',
+          _bases: ['Object', 'BaseModel'],
+          name: objectId,
+          description: '',
+          columns: [],
+        },
+      },
+    });
+    return objectId;
+  };
+
+  return createLeaderboard;
+};
+
+const useNavigateToLeaderboard = (entity: string, project: string) => {
+  const history = useHistory();
+  const {baseRouter} = useWeaveflowRouteContext();
+  const navigateToLeaderboard = useCallback(
+    (objectId: string) => {
+      const to = baseRouter.leaderboardsUIUrl(entity, project, objectId, true);
+      history.push(to);
+    },
+    [history, baseRouter, entity, project]
+  );
+  return navigateToLeaderboard;
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 8c3c149a2ec..3b016c5b114 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -33,13 +33,19 @@ type LeaderboardPageProps = {
   entity: string;
   project: string;
   leaderboardName: string;
+  openEditorOnMount?: boolean;
 };
 
 export const LeaderboardPage: React.FC<LeaderboardPageProps> = props => {
   const [name, setName] = useState(props.leaderboardName);
   const {isPeeking} = useContext(WeaveflowPeekContext);
-  const [isEditing, setIsEditing] = useState(false);
   const {isEditor} = useIsEditor(props.entity);
+  const [isEditing, setIsEditing] = useState(false);
+  useEffect(() => {
+    if (isEditor && props.openEditorOnMount) {
+      setIsEditing(true);
+    }
+  }, [isEditor, props.openEditorOnMount]);
   return (
     <SimplePageLayout
       title={name}
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
index 101d887ccf5..c8a64276604 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
@@ -234,10 +234,15 @@ export const ObjectVersionsTable: React.FC<{
           // Icon to indicate navigation to the object version
           const obj: ObjectVersionSchema = cellParams.row.obj;
           if (props.onRowClick) {
-            const text = props.hideVersionSuffix
+            let text = props.hideVersionSuffix
               ? obj.objectId
               : objectVersionText(obj.objectId, obj.versionIndex);
 
+            // Maybe this should be conditional?
+            if (obj.val.name) {
+              text = obj.val.name;
+            }
+
             return (
               <CustomLink text={text} onClick={() => props.onRowClick?.(obj)} />
             );

From 71ae5a3d81ab755bdf96b7c7885dbaa31573abb0 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 21 Oct 2024 00:27:42 -0700
Subject: [PATCH 095/169] Ok, all the building blocks are now there

---
 .../pages/LeaderboardPage/LeaderboardListingPage.tsx | 12 ++++++++----
 .../pages/LeaderboardPage/LeaderboardPage.tsx        |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index d225baf2ba4..eb3708b9ea4 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -15,6 +15,7 @@ import {useWFHooks} from '../wfReactInterface/context';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
 import {ObjectVersionSchema} from '../wfReactInterface/wfDataModelHooksInterface';
+import {useIsEditor} from './LeaderboardPage';
 
 const Container = styled.div`
   width: 100%;
@@ -26,6 +27,7 @@ export const LeaderboardListingPage: React.FC<{
   entity: string;
   project: string;
 }> = props => {
+  const {isEditor} = useIsEditor(props.entity);
   return (
     <SimplePageLayout
       title={`Leaderboards`}
@@ -37,10 +39,12 @@ export const LeaderboardListingPage: React.FC<{
         },
       ]}
       headerExtra={
-        <CreateLeaderboardButton
-          entity={props.entity}
-          project={props.project}
-        />
+        isEditor && (
+          <CreateLeaderboardButton
+            entity={props.entity}
+            project={props.project}
+          />
+        )
       }
     />
   );
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 3b016c5b114..623b3be3724 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -410,7 +410,7 @@ const EditLeaderboardButton: FC<{
   );
 };
 
-const useIsEditor = (entity: string) => {
+export const useIsEditor = (entity: string) => {
   const {loading: loadingUserInfo, userInfo} = useViewerInfo();
   return useMemo(() => {
     if (loadingUserInfo) {

From b738739e1856eb7ac4c21fff9e8bf653b73f5e4f Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 23 Oct 2024 00:42:29 -0700
Subject: [PATCH 096/169] Keep working

---
 demos/leaderboard_demo.ipynb | 1759 ++++++++++++++++++++++++++++++++++
 1 file changed, 1759 insertions(+)
 create mode 100644 demos/leaderboard_demo.ipynb

diff --git a/demos/leaderboard_demo.ipynb b/demos/leaderboard_demo.ipynb
new file mode 100644
index 00000000000..67e773b3948
--- /dev/null
+++ b/demos/leaderboard_demo.ipynb
@@ -0,0 +1,1759 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from openai import OpenAI\n",
+    "from pydantic import BaseModel\n",
+    "\n",
+    "\n",
+    "class Row(BaseModel):\n",
+    "    zip_code: str\n",
+    "    city: str\n",
+    "    state: str\n",
+    "    avg_temp_f: float\n",
+    "    population: int\n",
+    "    median_income: int\n",
+    "    known_for: str\n",
+    "\n",
+    "class Rows(BaseModel):\n",
+    "    rows: list[Row]\n",
+    "\n",
+    "def generate_dataset_rows(location: str = \"United States\", count: int = 5, year: int = 2022):\n",
+    "    client = OpenAI()\n",
+    "\n",
+    "    completion = client.beta.chat.completions.parse(\n",
+    "        model=\"gpt-4o-mini\",\n",
+    "        messages=[\n",
+    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
+    "            {\"role\": \"user\", \"content\": f\"Please generate {count} rows of data for random zip codes in {location} for the year {year}.\"},\n",
+    "        ],\n",
+    "        response_format=Rows\n",
+    "        \n",
+    "    )\n",
+    "\n",
+    "    return completion.choices[0].message.parsed.model_dump()['rows']\n",
+    "\n",
+    "rows = generate_dataset_rows()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import weave\n",
+    "\n",
+    "\n",
+    "@weave.op\n",
+    "def check_concrete_fields(city: str, state: str, model_output: dict):\n",
+    "    return {\n",
+    "        \"city_match\": city == model_output[\"city\"],\n",
+    "        \"state_match\": state == model_output[\"state\"],\n",
+    "    }\n",
+    "\n",
+    "@weave.op\n",
+    "def check_value_fields(avg_temp_f: float, population: int, median_income: int, model_output: dict):\n",
+    "    return {\n",
+    "        \"avg_temp_f_err\": abs(avg_temp_f - model_output[\"avg_temp_f\"]) / avg_temp_f,\n",
+    "        \"population_err\": abs(population - model_output[\"population\"]) / population,\n",
+    "        \"median_income_err\": abs(median_income - model_output[\"median_income\"]) / median_income,\n",
+    "    }\n",
+    "\n",
+    "@weave.op\n",
+    "def check_subjective_fields(zip_code: str, known_for: str, model_output: dict):\n",
+    "    client = OpenAI()\n",
+    "\n",
+    "    class Response(BaseModel):\n",
+    "        correct_known_for: bool\n",
+    "\n",
+    "    completion = client.beta.chat.completions.parse(\n",
+    "        model=\"gpt-4o-mini\",\n",
+    "        messages=[\n",
+    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
+    "            {\"role\": \"user\", \"content\": f\"My student was asked what the zip code {zip_code} is best known best for. The right answer is '{known_for}', and they said '{model_output['known_for']}'. Is their answer correct?\"},\n",
+    "        ],\n",
+    "        response_format=Response\n",
+    "    )\n",
+    "\n",
+    "    return completion.choices[0].message.parsed.model_dump()\n",
+    "\n",
+    "\n",
+    "evaluation = weave.Evaluation(\n",
+    "    name=\"United States - 2022 - Concrete Fields\",\n",
+    "    dataset=rows,\n",
+    "    scorers=[\n",
+    "        check_concrete_fields,\n",
+    "        check_value_fields,\n",
+    "        check_subjective_fields,\n",
+    "    ]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6228438228438228</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.16338317484024056</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6866822079727841</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.005936479568481446</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6228438228438228\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.16338317484024056\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6866822079727841\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.005936479568481446\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b84c-550a-7701-8ca5-47609c8a5892\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'model_output': {'avg_temp_f': {'mean': 50.0},\n",
+       "  'population': {'mean': 1000000.0},\n",
+       "  'median_income': {'mean': 100000.0}},\n",
+       " 'check_concrete_fields': {'city_match': {'true_count': 1,\n",
+       "   'true_fraction': 0.2},\n",
+       "  'state_match': {'true_count': 1, 'true_fraction': 0.2}},\n",
+       " 'check_value_fields': {'median_income_err': {'mean': 0.6228438228438228},\n",
+       "  'avg_temp_f_err': {'mean': 0.16338317484024056},\n",
+       "  'population_err': {'mean': 0.6866822079727841}},\n",
+       " 'check_subjective_fields': {'correct_known_for': {'true_count': 0,\n",
+       "   'true_fraction': 0.0}},\n",
+       " 'model_latency': {'mean': 0.005936479568481446}}"
+      ]
+     },
+     "execution_count": 41,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "weave.init(\"leaderboard-demo-1\")\n",
+    "\n",
+    "@weave.op\n",
+    "def baseline_model(zip_code: str):\n",
+    "    return {\n",
+    "        \"city\": \"New York\",\n",
+    "        \"state\": \"NY\",\n",
+    "        \"avg_temp_f\": 50.0,\n",
+    "        \"population\": 1000000,\n",
+    "        \"median_income\": 100000,\n",
+    "        \"known_for\": \"The Big Apple\",\n",
+    "    }\n",
+    "\n",
+    "await evaluation.evaluate(baseline_model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6228438228438228</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.16338317484024056</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6866822079727841</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.00630335807800293</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6228438228438228\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.16338317484024056\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6866822079727841\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.00630335807800293\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b84c-6a1b-7651-9a71-f86d8eb6d4da\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'model_output': {'avg_temp_f': {'mean': 50.0},\n",
+       "  'population': {'mean': 1000000.0},\n",
+       "  'median_income': {'mean': 100000.0}},\n",
+       " 'check_concrete_fields': {'city_match': {'true_count': 1,\n",
+       "   'true_fraction': 0.2},\n",
+       "  'state_match': {'true_count': 1, 'true_fraction': 0.2}},\n",
+       " 'check_value_fields': {'median_income_err': {'mean': 0.6228438228438228},\n",
+       "  'avg_temp_f_err': {'mean': 0.16338317484024056},\n",
+       "  'population_err': {'mean': 0.6866822079727841}},\n",
+       " 'check_subjective_fields': {'correct_known_for': {'true_count': 0,\n",
+       "   'true_fraction': 0.0}},\n",
+       " 'model_latency': {'mean': 0.00630335807800293}}"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "weave.init(\"leaderboard-demo-1\")\n",
+    "\n",
+    "@weave.op\n",
+    "def gpt_4o_mini_no_context(zip_code: str):\n",
+    "    client = OpenAI()\n",
+    "\n",
+    "    completion = client.beta.chat.completions.parse(\n",
+    "        model=\"gpt-4o-mini\",\n",
+    "        messages=[{\"role\": \"user\", \"content\": f\"\"\"Zip code {zip_code}\"\"\"}],\n",
+    "        response_format=Row\n",
+    "    )\n",
+    "\n",
+    "    return completion.choices[0].message.parsed.model_dump()\n",
+    "\n",
+    "await evaluation.evaluate(baseline_model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6228438228438228</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.16338317484024056</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6866822079727841</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.009626293182373047</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6228438228438228\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.16338317484024056\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6866822079727841\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.009626293182373047\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b84c-87c0-7d92-b470-75a25d3a172d\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'model_output': {'avg_temp_f': {'mean': 50.0},\n",
+       "  'population': {'mean': 1000000.0},\n",
+       "  'median_income': {'mean': 100000.0}},\n",
+       " 'check_concrete_fields': {'city_match': {'true_count': 1,\n",
+       "   'true_fraction': 0.2},\n",
+       "  'state_match': {'true_count': 1, 'true_fraction': 0.2}},\n",
+       " 'check_value_fields': {'median_income_err': {'mean': 0.6228438228438228},\n",
+       "  'avg_temp_f_err': {'mean': 0.16338317484024056},\n",
+       "  'population_err': {'mean': 0.6866822079727841}},\n",
+       " 'check_subjective_fields': {'correct_known_for': {'true_count': 0,\n",
+       "   'true_fraction': 0.0}},\n",
+       " 'model_latency': {'mean': 0.009626293182373047}}"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "weave.init(\"leaderboard-demo-1\")\n",
+    "\n",
+    "@weave.op\n",
+    "def gpt_4o_mini_with_context(zip_code: str):\n",
+    "    client = OpenAI()\n",
+    "\n",
+    "    completion = client.beta.chat.completions.parse(\n",
+    "        model=\"gpt-4o-mini\",\n",
+    "        messages=[{\"role\": \"user\", \"content\": f\"\"\"Please answer the following questions about the zip code {zip_code}:\n",
+    "                   1. What is the city?\n",
+    "                   2. What is the state?\n",
+    "                   3. What is the average temperature in Fahrenheit?\n",
+    "                   4. What is the population?\n",
+    "                   5. What is the median income?\n",
+    "                   6. What is the most well known thing about this zip code?\n",
+    "                   \"\"\"}],\n",
+    "        response_format=Row\n",
+    "    )\n",
+    "\n",
+    "    return completion.choices[0].message.parsed.model_dump()\n",
+    "\n",
+    "await evaluation.evaluate(baseline_model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4050141490502101</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.21821794303416694</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.957273935662758</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.00478057861328125</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.4050141490502101\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.21821794303416694\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m6.957273935662758\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.00478057861328125\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-b035-7103-825f-a8730fb15283\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">62.35999999999999</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">842848.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">69597.8</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.18255178974054148</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.05120532743037322</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.03469806588734624</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.2620424747467043</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m62.35999999999999\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m842848.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m69597.8\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m4\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.8\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.18255178974054148\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.05120532743037322\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.03469806588734624\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m2\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.4\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.2620424747467043\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-bb6f-7901-9c2b-a28d13408460\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">62.94</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">256081.4</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">90850.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3581675170068027</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.042052216442918955</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.24075816760087537</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.1589102268218996</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m62.94\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m256081.4\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m90850.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.3581675170068027\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.042052216442918955\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.24075816760087537\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m2\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.4\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.1589102268218996\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-d643-7562-a5f6-3e6cf0dfaabc\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.34373485022464034</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.22868226800995295</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.097803457657646</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.004935073852539063</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.34373485022464034\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.22868226800995295\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m6.097803457657646\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.004935073852539063\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-ef25-7121-a41d-30b7ace44725\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">63.94</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">149693.2</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">84428.6</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2400336561684155</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.017941257079061138</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5826742124226545</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.1302245616912843</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m63.94\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m149693.2\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m84428.6\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.6\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.2400336561684155\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.017941257079061138\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.5826742124226545\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.6\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.1302245616912843\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-fa08-7da1-803c-117001bc1dd2\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">63.94</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">82250.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">76740.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1313042970645883</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.03346736052382718</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7226121363777034</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.7572145462036133</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m63.94\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m82250.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m76740.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.1313042970645883\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.03346736052382718\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.7226121363777034\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.7572145462036133\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b853-113e-73d1-87ee-e648a4604096\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7179344154463176</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.15751090428978923</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1639.524473320823</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.005164194107055664</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.7179344154463176\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.15751090428978923\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1639.524473320823\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.005164194107055664\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b853-2fe5-7603-a2d9-2fcd025b516a\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">60.760000000000005</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">217712.2</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">72990.8</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.36520391602103663</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.021658375735112832</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">48.83519885100851</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.703943061828613</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m60.760000000000005\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m217712.2\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m72990.8\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.6\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.36520391602103663\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.021658375735112832\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m48.83519885100851\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.703943061828613\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b853-384b-72a2-9429-e381d072ee6e\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
+       "<span style=\"font-weight: bold\">{</span>\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">60.9</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">75024.8</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">84372.6</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3543283982836134</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.015047371269703083</span><span style=\"font-weight: bold\">}</span>,\n",
+       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">70.28742213711891</span><span style=\"font-weight: bold\">}</span>\n",
+       "    <span style=\"font-weight: bold\">}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}}</span>,\n",
+       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.426439666748047</span><span style=\"font-weight: bold\">}</span>\n",
+       "<span style=\"font-weight: bold\">}</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "Evaluation summary\n",
+       "\u001b[1m{\u001b[0m\n",
+       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m60.9\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m75024.8\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m84372.6\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m2\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.4\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
+       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.3543283982836134\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.015047371269703083\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m70.28742213711891\u001b[0m\u001b[1m}\u001b[0m\n",
+       "    \u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.426439666748047\u001b[0m\u001b[1m}\u001b[0m\n",
+       "\u001b[1m}\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b853-57cf-7cd0-a363-d42b22f5ec9c\n"
+     ]
+    }
+   ],
+   "source": [
+    "scorers = [\n",
+    "    check_concrete_fields,\n",
+    "    check_value_fields,\n",
+    "    check_subjective_fields,\n",
+    "]\n",
+    "evaluations = [\n",
+    "    weave.Evaluation(\n",
+    "        name=\"United States - 2022\",\n",
+    "        dataset=weave.Dataset(name=\"United States - 2022\", rows=generate_dataset_rows(\"United States\", 5, 2022)),\n",
+    "        scorers=scorers,\n",
+    "    ),\n",
+    "    weave.Evaluation(\n",
+    "        name=\"California - 2022\",\n",
+    "        dataset=weave.Dataset(name=\"California - 2022\", rows=generate_dataset_rows(\"California\", 5, 2022)),\n",
+    "        scorers=scorers,\n",
+    "    ),\n",
+    "    weave.Evaluation(\n",
+    "        name=\"United States - 2000\",\n",
+    "        dataset=weave.Dataset(name=\"United States - 2000\", rows=generate_dataset_rows(\"United States\", 5, 2000)),\n",
+    "        scorers=scorers,\n",
+    "    ),\n",
+    "]\n",
+    "models = [\n",
+    "    baseline_model,\n",
+    "    gpt_4o_mini_no_context,\n",
+    "    gpt_4o_mini_with_context,\n",
+    "]\n",
+    "\n",
+    "for evaluation in evaluations:\n",
+    "    for model in models:\n",
+    "        await evaluation.evaluate(model, __weave={\"display_name\": evaluation.name + \":\" + model.__name__})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "wandb-weave",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

From ed702cb560ae921dd6f9dbf16eb9510c88235924 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 23 Oct 2024 00:47:12 -0700
Subject: [PATCH 097/169] Keep working

---
 demos/leaderboard_demo.ipynb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/demos/leaderboard_demo.ipynb b/demos/leaderboard_demo.ipynb
index 67e773b3948..4b929007762 100644
--- a/demos/leaderboard_demo.ipynb
+++ b/demos/leaderboard_demo.ipynb
@@ -1710,11 +1710,11 @@
     "        dataset=weave.Dataset(name=\"California - 2022\", rows=generate_dataset_rows(\"California\", 5, 2022)),\n",
     "        scorers=scorers,\n",
     "    ),\n",
-    "    weave.Evaluation(\n",
-    "        name=\"United States - 2000\",\n",
-    "        dataset=weave.Dataset(name=\"United States - 2000\", rows=generate_dataset_rows(\"United States\", 5, 2000)),\n",
-    "        scorers=scorers,\n",
-    "    ),\n",
+    "    # weave.Evaluation(\n",
+    "    #     name=\"United States - 2000\",\n",
+    "    #     dataset=weave.Dataset(name=\"United States - 2000\", rows=generate_dataset_rows(\"United States\", 5, 2000)),\n",
+    "    #     scorers=scorers,\n",
+    "    # ),\n",
     "]\n",
     "models = [\n",
     "    baseline_model,\n",

From f44cc5876a1707ce141b6c883a7c7c2412d32acb Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 23 Oct 2024 00:47:33 -0700
Subject: [PATCH 098/169] Keep working

---
 demos/leaderboard_demo.ipynb | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/demos/leaderboard_demo.ipynb b/demos/leaderboard_demo.ipynb
index 4b929007762..89d76079f69 100644
--- a/demos/leaderboard_demo.ipynb
+++ b/demos/leaderboard_demo.ipynb
@@ -1733,6 +1733,13 @@
    "metadata": {},
    "outputs": [],
    "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "https://beta.wandb.ai/timssweeney/leaderboard-demo-1/weave/leaderboards/leaderboard-2024-10-23T07-43-07-604Z?edit=true"
+   ]
   }
  ],
  "metadata": {

From 05dc0d3ecdf31631e6c9dfebbbbc2c03c4f2ec00 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 23 Oct 2024 00:49:18 -0700
Subject: [PATCH 099/169] Keep working

---
 demos/leaderboard_demo.ipynb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/demos/leaderboard_demo.ipynb b/demos/leaderboard_demo.ipynb
index 89d76079f69..6622601749f 100644
--- a/demos/leaderboard_demo.ipynb
+++ b/demos/leaderboard_demo.ipynb
@@ -1738,6 +1738,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "https://beta.wandb.ai/?betaVersion=234f24492fa0a1999a084aeb93d4d2128944512c\n",
     "https://beta.wandb.ai/timssweeney/leaderboard-demo-1/weave/leaderboards/leaderboard-2024-10-23T07-43-07-604Z?edit=true"
    ]
   }

From 92bedfd7d8cf3fd0653a17a8066208dc885774a2 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 23 Oct 2024 10:44:31 -0700
Subject: [PATCH 100/169] Keep working

---
 demos/leaderboard_demo.ipynb | 1562 +---------------------------------
 1 file changed, 27 insertions(+), 1535 deletions(-)

diff --git a/demos/leaderboard_demo.ipynb b/demos/leaderboard_demo.ipynb
index 6622601749f..ceb6795b70e 100644
--- a/demos/leaderboard_demo.ipynb
+++ b/demos/leaderboard_demo.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -42,7 +42,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "project_name = \"leaderboard-demo-3\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -96,159 +105,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 41,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6228438228438228</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.16338317484024056</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6866822079727841</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.005936479568481446</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6228438228438228\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.16338317484024056\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6866822079727841\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.005936479568481446\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b84c-550a-7701-8ca5-47609c8a5892\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'model_output': {'avg_temp_f': {'mean': 50.0},\n",
-       "  'population': {'mean': 1000000.0},\n",
-       "  'median_income': {'mean': 100000.0}},\n",
-       " 'check_concrete_fields': {'city_match': {'true_count': 1,\n",
-       "   'true_fraction': 0.2},\n",
-       "  'state_match': {'true_count': 1, 'true_fraction': 0.2}},\n",
-       " 'check_value_fields': {'median_income_err': {'mean': 0.6228438228438228},\n",
-       "  'avg_temp_f_err': {'mean': 0.16338317484024056},\n",
-       "  'population_err': {'mean': 0.6866822079727841}},\n",
-       " 'check_subjective_fields': {'correct_known_for': {'true_count': 0,\n",
-       "   'true_fraction': 0.0}},\n",
-       " 'model_latency': {'mean': 0.005936479568481446}}"
-      ]
-     },
-     "execution_count": 41,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "weave.init(\"leaderboard-demo-1\")\n",
+    "weave.init(project_name)\n",
     "\n",
     "@weave.op\n",
     "def baseline_model(zip_code: str):\n",
     "    return {\n",
-    "        \"city\": \"New York\",\n",
+    "        \"city\": \"California\",\n",
     "        \"state\": \"NY\",\n",
     "        \"avg_temp_f\": 50.0,\n",
     "        \"population\": 1000000,\n",
@@ -261,154 +127,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6228438228438228</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.16338317484024056</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6866822079727841</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.00630335807800293</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6228438228438228\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.16338317484024056\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6866822079727841\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.00630335807800293\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b84c-6a1b-7651-9a71-f86d8eb6d4da\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'model_output': {'avg_temp_f': {'mean': 50.0},\n",
-       "  'population': {'mean': 1000000.0},\n",
-       "  'median_income': {'mean': 100000.0}},\n",
-       " 'check_concrete_fields': {'city_match': {'true_count': 1,\n",
-       "   'true_fraction': 0.2},\n",
-       "  'state_match': {'true_count': 1, 'true_fraction': 0.2}},\n",
-       " 'check_value_fields': {'median_income_err': {'mean': 0.6228438228438228},\n",
-       "  'avg_temp_f_err': {'mean': 0.16338317484024056},\n",
-       "  'population_err': {'mean': 0.6866822079727841}},\n",
-       " 'check_subjective_fields': {'correct_known_for': {'true_count': 0,\n",
-       "   'true_fraction': 0.0}},\n",
-       " 'model_latency': {'mean': 0.00630335807800293}}"
-      ]
-     },
-     "execution_count": 42,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "weave.init(\"leaderboard-demo-1\")\n",
+    "weave.init(project_name)\n",
     "\n",
     "@weave.op\n",
     "def gpt_4o_mini_no_context(zip_code: str):\n",
@@ -422,159 +145,16 @@
     "\n",
     "    return completion.choices[0].message.parsed.model_dump()\n",
     "\n",
-    "await evaluation.evaluate(baseline_model)"
+    "await evaluation.evaluate(gpt_4o_mini_no_context)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 43,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6228438228438228</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.16338317484024056</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6866822079727841</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.009626293182373047</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6228438228438228\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.16338317484024056\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6866822079727841\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.009626293182373047\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b84c-87c0-7d92-b470-75a25d3a172d\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'model_output': {'avg_temp_f': {'mean': 50.0},\n",
-       "  'population': {'mean': 1000000.0},\n",
-       "  'median_income': {'mean': 100000.0}},\n",
-       " 'check_concrete_fields': {'city_match': {'true_count': 1,\n",
-       "   'true_fraction': 0.2},\n",
-       "  'state_match': {'true_count': 1, 'true_fraction': 0.2}},\n",
-       " 'check_value_fields': {'median_income_err': {'mean': 0.6228438228438228},\n",
-       "  'avg_temp_f_err': {'mean': 0.16338317484024056},\n",
-       "  'population_err': {'mean': 0.6866822079727841}},\n",
-       " 'check_subjective_fields': {'correct_known_for': {'true_count': 0,\n",
-       "   'true_fraction': 0.0}},\n",
-       " 'model_latency': {'mean': 0.009626293182373047}}"
-      ]
-     },
-     "execution_count": 43,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "weave.init(\"leaderboard-demo-1\")\n",
+    "weave.init(project_name)\n",
     "\n",
     "@weave.op\n",
     "def gpt_4o_mini_with_context(zip_code: str):\n",
@@ -600,1099 +180,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 49,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4050141490502101</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.21821794303416694</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.957273935662758</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.00478057861328125</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.4050141490502101\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.21821794303416694\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m6.957273935662758\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.00478057861328125\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-b035-7103-825f-a8730fb15283\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">62.35999999999999</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">842848.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">69597.8</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.18255178974054148</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.05120532743037322</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.03469806588734624</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.2620424747467043</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m62.35999999999999\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m842848.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m69597.8\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m4\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.8\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.18255178974054148\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.05120532743037322\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.03469806588734624\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m2\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.4\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.2620424747467043\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-bb6f-7901-9c2b-a28d13408460\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">62.94</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">256081.4</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">90850.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3581675170068027</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.042052216442918955</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.24075816760087537</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.1589102268218996</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m62.94\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m256081.4\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m90850.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.3581675170068027\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.042052216442918955\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.24075816760087537\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m2\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.4\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.1589102268218996\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-d643-7562-a5f6-3e6cf0dfaabc\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.34373485022464034</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.22868226800995295</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6.097803457657646</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.004935073852539063</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.34373485022464034\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.22868226800995295\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m6.097803457657646\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.004935073852539063\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-ef25-7121-a41d-30b7ace44725\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">63.94</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">149693.2</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">84428.6</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2400336561684155</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.017941257079061138</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5826742124226545</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.1302245616912843</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m63.94\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m149693.2\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m84428.6\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.6\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.2400336561684155\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.017941257079061138\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.5826742124226545\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.6\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.1302245616912843\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b852-fa08-7da1-803c-117001bc1dd2\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">63.94</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">82250.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">76740.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1313042970645883</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.03346736052382718</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7226121363777034</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.7572145462036133</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m63.94\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m82250.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m76740.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.1313042970645883\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.03346736052382718\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.7226121363777034\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.7572145462036133\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b853-113e-73d1-87ee-e648a4604096\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1000000.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100000.0</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7179344154463176</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.15751090428978923</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1639.524473320823</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.005164194107055664</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m50.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1000000.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m100000.0\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.2\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.7179344154463176\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.15751090428978923\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m1639.524473320823\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.005164194107055664\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b853-2fe5-7603-a2d9-2fcd025b516a\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">60.760000000000005</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">217712.2</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">72990.8</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.36520391602103663</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.021658375735112832</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">48.83519885100851</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.703943061828613</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m60.760000000000005\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m217712.2\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m72990.8\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.6\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.36520391602103663\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.021658375735112832\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m48.83519885100851\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.703943061828613\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b853-384b-72a2-9429-e381d072ee6e\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> examples\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m5\u001b[0m examples\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
-       "<span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_output'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">60.9</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">75024.8</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">84372.6</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_concrete_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'city_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'state_match'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_value_fields'</span>: <span style=\"font-weight: bold\">{</span>\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'median_income_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3543283982836134</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'avg_temp_f_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.015047371269703083</span><span style=\"font-weight: bold\">}</span>,\n",
-       "        <span style=\"color: #008000; text-decoration-color: #008000\">'population_err'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">70.28742213711891</span><span style=\"font-weight: bold\">}</span>\n",
-       "    <span style=\"font-weight: bold\">}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'check_subjective_fields'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'correct_known_for'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'true_count'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'true_fraction'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span><span style=\"font-weight: bold\">}}</span>,\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.426439666748047</span><span style=\"font-weight: bold\">}</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Evaluation summary\n",
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'model_output'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'avg_temp_f'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m60.9\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m75024.8\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'median_income'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m84372.6\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_concrete_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'city_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'state_match'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m2\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m0.4\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_value_fields'\u001b[0m: \u001b[1m{\u001b[0m\n",
-       "        \u001b[32m'median_income_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.3543283982836134\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'avg_temp_f_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.015047371269703083\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[32m'population_err'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m70.28742213711891\u001b[0m\u001b[1m}\u001b[0m\n",
-       "    \u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'check_subjective_fields'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'correct_known_for'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.426439666748047\u001b[0m\u001b[1m}\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "🍩 https://wandb.ai/timssweeney/leaderboard-demo-1/r/call/0192b853-57cf-7cd0-a363-d42b22f5ec9c\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "scorers = [\n",
     "    check_concrete_fields,\n",
@@ -1732,7 +222,9 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "\n"
+   ]
   },
   {
    "cell_type": "markdown",

From baf6a03c4d097311d3058abd361923c4de50470b Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 14:03:45 -0700
Subject: [PATCH 101/169] finished merge

---
 .../Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx   | 2 +-
 .../Home/Browse3/pages/ObjectVersionsPage.tsx                   | 1 -
 .../Home/Browse3/pages/common/EmptyContent.tsx                  | 1 -
 3 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index be598d0fb37..1a4296f5c6b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -15,7 +15,7 @@ import {
   ArrowUpward as MoveUpIcon,
   Delete as DeleteIcon,
   FileCopy as CloneIcon,
-} from '@material-ui/icons';
+} from '@mui/icons-material';
 import {refUri} from '@wandb/weave/react';
 import React, {useEffect, useState} from 'react';
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
index 78da6ee8e1d..f9ccd0283db 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
@@ -331,7 +331,6 @@ export const ObjectVersionsTable: React.FC<{
         })
       );
     }
-    
 
     if (!props.hideCreatedAtColumn) {
       cols.push(
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx
index 5ba00263ab7..78ef9cd7f46 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx
@@ -43,7 +43,6 @@ export const EMPTY_PROPS_EVALUATIONS: EmptyProps = {
   ),
 };
 
-
 export const EMPTY_PROPS_LEADERBOARDS: EmptyProps = {
   icon: 'benchmark-square' as const,
   heading: 'No leaderboards yet',

From 5c47b8d28dedd1883427e07f24de5efc4d33ee3d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 14:44:57 -0700
Subject: [PATCH 102/169] code clean

---
 weave-js/package.json                         |   2 -
 .../LeaderboardPage/EditableMarkdown.tsx      | 163 ------------------
 .../LeaderboardConfigEditor.tsx               |   2 -
 .../pages/LeaderboardPage/LeaderboardPage.tsx |  33 +++-
 weave-js/yarn.lock                            |  48 +-----
 5 files changed, 35 insertions(+), 213 deletions(-)
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx

diff --git a/weave-js/package.json b/weave-js/package.json
index 36c83613161..fd4ae47bf23 100644
--- a/weave-js/package.json
+++ b/weave-js/package.json
@@ -34,7 +34,6 @@
     "@emotion/styled": "^11.11.0",
     "@floating-ui/react": "^0.24.5",
     "@material-ui/core": "^4.12.4",
-    "@material-ui/icons": "^4.11.3",
     "@monaco-editor/react": "^4.3.1",
     "@mui/icons-material": "^5.14.12",
     "@mui/material": "^6.0.2",
@@ -95,7 +94,6 @@
     "react": "^17.0.2",
     "react-app-polyfill": "^3.0.0",
     "react-base-table": "^1.12.0",
-    "react-beautiful-dnd": "^13.1.1",
     "react-cytoscapejs": "^1.2.1",
     "react-datetime": "^3",
     "react-diff-viewer": "^3.1.1",
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
deleted file mode 100644
index bb55ca863e2..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/EditableMarkdown.tsx
+++ /dev/null
@@ -1,163 +0,0 @@
-import {Box, TextField, Typography} from '@mui/material';
-import React, {
-  useCallback,
-  useEffect,
-  useLayoutEffect,
-  useRef,
-  useState,
-} from 'react';
-import ReactMarkdown from 'react-markdown';
-import styled from 'styled-components';
-
-interface EditableMarkdownProps {
-  value: string;
-  onChange: (value: string) => void;
-  placeholder: string;
-}
-
-export const StyledReactMarkdown = styled(ReactMarkdown)`
-  > *:first-child {
-    margin-top: 0;
-  }
-  h1 {
-    font-weight: 600;
-    font-size: 1.2rem;
-  }
-  h2 {
-    font-weight: 600;
-    font-size: 1.15rem;
-  }
-  h3 {
-    font-weight: 600;
-    font-size: 1.1rem;
-  }
-  h4 {
-    font-weight: 600;
-    font-size: 1.05rem;
-  }
-  h5 {
-    font-weight: 600;
-    font-size: 1rem;
-  }
-  h6 {
-    font-weight: 600;
-    font-size: 1rem;
-  }
-`;
-
-export const EditableMarkdown: React.FC<EditableMarkdownProps> = ({
-  value,
-  onChange,
-  placeholder,
-}) => {
-  const [isEditing, setIsEditing] = useState(false);
-  const containerRef = useRef<HTMLDivElement>(null);
-  const contentRef = useRef<HTMLDivElement>(null);
-  const textFieldRef = useRef<HTMLTextAreaElement>(null);
-
-  const updateHeight = useCallback(() => {
-    if (containerRef.current) {
-      let newHeight: number;
-
-      if (isEditing && textFieldRef.current) {
-        // For editing mode, use scrollHeight of the textarea
-        newHeight = textFieldRef.current.scrollHeight;
-      } else if (!isEditing && contentRef.current) {
-        // For viewing mode, get the rendered content height
-        newHeight = contentRef.current.offsetHeight;
-      } else {
-        return; // Exit if we can't determine the height
-      }
-
-      newHeight -= 3;
-
-      // Set a minimum height to prevent collapsing
-      newHeight = Math.max(newHeight, 24);
-
-      containerRef.current.style.height = `${newHeight}px`;
-    }
-  }, [isEditing]);
-
-  useLayoutEffect(() => {
-    // Use RAF to ensure DOM is fully updated before measuring
-    requestAnimationFrame(() => {
-      updateHeight();
-    });
-
-    const resizeObserver = new ResizeObserver(() => {
-      requestAnimationFrame(() => {
-        updateHeight();
-      });
-    });
-
-    if (containerRef.current) {
-      resizeObserver.observe(containerRef.current);
-    }
-
-    return () => resizeObserver.disconnect();
-  }, [updateHeight]);
-
-  const handleBlur = () => {
-    setIsEditing(false);
-    requestAnimationFrame(() => {
-      updateHeight();
-    });
-  };
-
-  const handleEdit = () => {
-    setIsEditing(true);
-    requestAnimationFrame(() => {
-      updateHeight();
-    });
-  };
-
-  useEffect(() => {
-    requestAnimationFrame(() => {
-      updateHeight();
-    });
-  }, [updateHeight, value]);
-
-  return (
-    <Box
-      ref={containerRef}
-      sx={{padding: '0px', transition: 'height 0.3s', flex: 1}}>
-      {isEditing ? (
-        <TextField
-          fullWidth
-          multiline
-          value={value}
-          onChange={e => {
-            onChange(e.target.value);
-          }}
-          onBlur={handleBlur}
-          placeholder={placeholder}
-          autoFocus
-          variant="outlined"
-          inputRef={textFieldRef}
-          sx={{
-            '& .MuiOutlinedInput-root': {
-              padding: 0,
-              '& textarea': {
-                padding: '0',
-                overflow: 'hidden',
-              },
-            },
-          }}
-        />
-      ) : (
-        <Typography
-          component="div"
-          ref={contentRef}
-          sx={{
-            cursor: 'pointer',
-            '&:hover': {
-              backgroundColor: 'action.hover',
-            },
-          }}
-          onClick={handleEdit}>
-          <StyledReactMarkdown>{value || placeholder}</StyledReactMarkdown>
-        </Typography>
-      )}
-    </Box>
-  );
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 1a4296f5c6b..a2e1865aafb 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -437,8 +437,6 @@ const useMetrics = (
     }
 
     let mounted = true;
-    // TODO: Implement the actual API call to fetch metrics for the given evaluation object and scorer
-    // This is a placeholder implementation
     client
       .callsStreamQuery({
         project_id: projectIdFromParts({entity, project}),
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 623b3be3724..9977d81e93e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -13,6 +13,8 @@ import React, {
   useMemo,
   useState,
 } from 'react';
+import ReactMarkdown from 'react-markdown';
+import styled from 'styled-components';
 
 import {WeaveflowPeekContext} from '../../context';
 import {NotFoundPanel} from '../../NotFoundPanel';
@@ -26,7 +28,6 @@ import {SimplePageLayout} from '../common/SimplePageLayout';
 import {useWFHooks} from '../wfReactInterface/context';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
-import {StyledReactMarkdown} from './EditableMarkdown';
 import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
 
 type LeaderboardPageProps = {
@@ -427,3 +428,33 @@ export const useIsEditor = (entity: string) => {
     };
   }, [entity, loadingUserInfo, userInfo]);
 };
+
+export const StyledReactMarkdown = styled(ReactMarkdown)`
+  > *:first-child {
+    margin-top: 0;
+  }
+  h1 {
+    font-weight: 600;
+    font-size: 1.2rem;
+  }
+  h2 {
+    font-weight: 600;
+    font-size: 1.15rem;
+  }
+  h3 {
+    font-weight: 600;
+    font-size: 1.1rem;
+  }
+  h4 {
+    font-weight: 600;
+    font-size: 1.05rem;
+  }
+  h5 {
+    font-weight: 600;
+    font-size: 1rem;
+  }
+  h6 {
+    font-weight: 600;
+    font-size: 1rem;
+  }
+`;
diff --git a/weave-js/yarn.lock b/weave-js/yarn.lock
index 2cf59620df0..3315ac14ade 100644
--- a/weave-js/yarn.lock
+++ b/weave-js/yarn.lock
@@ -2792,13 +2792,6 @@
     react-is "^16.8.0 || ^17.0.0"
     react-transition-group "^4.4.0"
 
-"@material-ui/icons@^4.11.3":
-  version "4.11.3"
-  resolved "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz#b0693709f9b161ce9ccde276a770d968484ecff1"
-  integrity sha512-IKHlyx6LDh8n19vzwH5RtHIOHl9Tu90aAAxcbWME6kp4dmvODM3UvOHJeMIDzUbd4muuJKHmlNoBN+mDY4XkBA==
-  dependencies:
-    "@babel/runtime" "^7.4.4"
-
 "@material-ui/lab@^4.0.0-alpha.60":
   version "4.0.0-alpha.61"
   resolved "https://registry.yarnpkg.com/@material-ui/lab/-/lab-4.0.0-alpha.61.tgz#9bf8eb389c0c26c15e40933cc114d4ad85e3d978"
@@ -6455,13 +6448,6 @@ cryptiles@2.x.x:
   dependencies:
     boom "2.x.x"
 
-css-box-model@^1.2.0:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/css-box-model/-/css-box-model-1.2.1.tgz#59951d3b81fd6b2074a62d49444415b0d2b4d7c1"
-  integrity sha512-a7Vr4Q/kd/aw96bnJG332W9V9LkJO69JRcaCYDUqjp6/z0w6VcZjgAcTbgFxEPfBgdnAwlh3iwu+hLopa+flJw==
-  dependencies:
-    tiny-invariant "^1.0.6"
-
 css-color-keywords@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/css-color-keywords/-/css-color-keywords-1.0.0.tgz#fea2616dc676b2962686b3af8dbdbe180b244e05"
@@ -10392,7 +10378,7 @@ memoize-one@*, memoize-one@^6.0.0:
   resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-6.0.0.tgz#b2591b871ed82948aee4727dc6abceeeac8c1045"
   integrity sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw==
 
-"memoize-one@>=3.1.1 <6", memoize-one@^5.0.0, memoize-one@^5.0.4, memoize-one@^5.1.1:
+"memoize-one@>=3.1.1 <6", memoize-one@^5.0.0, memoize-one@^5.0.4:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e"
   integrity sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==
@@ -11974,11 +11960,6 @@ quickselect@^2.0.0:
   resolved "https://registry.yarnpkg.com/quickselect/-/quickselect-2.0.0.tgz#f19680a486a5eefb581303e023e98faaf25dd018"
   integrity sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw==
 
-raf-schd@^4.0.2:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/raf-schd/-/raf-schd-4.0.3.tgz#5d6c34ef46f8b2a0e880a8fcdb743efc5bfdbc1a"
-  integrity sha512-tQkJl2GRWh83ui2DiPTJz9wEiMN20syf+5oKfB03yYP7ioZcJwsIK8FjrtLwH1m7C7e+Tt2yYBlrOpdT+dyeIQ==
-
 raf@^3.4.1:
   version "3.4.1"
   resolved "https://registry.yarnpkg.com/raf/-/raf-3.4.1.tgz#0742e99a4a6552f445d73e3ee0328af0ff1ede39"
@@ -12010,19 +11991,6 @@ react-base-table@^1.12.0:
     react-virtualized-auto-sizer "^1.0.2"
     react-window "^1.8.2"
 
-react-beautiful-dnd@^13.1.1:
-  version "13.1.1"
-  resolved "https://registry.yarnpkg.com/react-beautiful-dnd/-/react-beautiful-dnd-13.1.1.tgz#b0f3087a5840920abf8bb2325f1ffa46d8c4d0a2"
-  integrity sha512-0Lvs4tq2VcrEjEgDXHjT98r+63drkKEgqyxdA7qD3mvKwga6a5SscbdLPO2IExotU1jW8L0Ksdl0Cj2AF67nPQ==
-  dependencies:
-    "@babel/runtime" "^7.9.2"
-    css-box-model "^1.2.0"
-    memoize-one "^5.1.1"
-    raf-schd "^4.0.2"
-    react-redux "^7.2.0"
-    redux "^4.0.4"
-    use-memo-one "^1.1.1"
-
 react-cytoscapejs@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/react-cytoscapejs/-/react-cytoscapejs-1.2.1.tgz#ccce46acabf4f0c41dce9070743854f92b0dc050"
@@ -12148,7 +12116,7 @@ react-popper@^1.3.4:
     typed-styles "^0.0.7"
     warning "^4.0.2"
 
-react-redux@^7.1.1, react-redux@^7.2.0:
+react-redux@^7.1.1:
   version "7.2.9"
   resolved "https://registry.yarnpkg.com/react-redux/-/react-redux-7.2.9.tgz#09488fbb9416a4efe3735b7235055442b042481d"
   integrity sha512-Gx4L3uM182jEEayZfRbI/G11ZpYdNAnBs70lFVMNdHJI76XYtR+7m0MN+eAs7UHBPhWXcnFPaS+9owSCJQHNpQ==
@@ -12404,7 +12372,7 @@ redent@^3.0.0:
     indent-string "^4.0.0"
     strip-indent "^3.0.0"
 
-redux@^4.0.0, redux@^4.0.1, redux@^4.0.4, redux@^4.0.5:
+redux@^4.0.0, redux@^4.0.1, redux@^4.0.5:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/redux/-/redux-4.2.1.tgz#c08f4306826c49b5e9dc901dee0452ea8fce6197"
   integrity sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==
@@ -13770,11 +13738,6 @@ tiny-invariant@^1.0.2:
   resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.3.1.tgz#8560808c916ef02ecfd55e66090df23a4b7aa642"
   integrity sha512-AD5ih2NlSssTCwsMznbvwMZpJ1cbhkGd2uueNxzv2jDlEeZdU04JQfRnggJQ8DrcVBGjAsCKwFBbDlVNtEMlzw==
 
-tiny-invariant@^1.0.6:
-  version "1.3.3"
-  resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.3.3.tgz#46680b7a873a0d5d10005995eb90a70d74d60127"
-  integrity sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==
-
 tiny-warning@^1.0.0, tiny-warning@^1.0.2, tiny-warning@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/tiny-warning/-/tiny-warning-1.0.3.tgz#94a30db453df4c643d0fd566060d60a875d84754"
@@ -14505,11 +14468,6 @@ use-latest@^1.2.1:
   dependencies:
     use-isomorphic-layout-effect "^1.1.1"
 
-use-memo-one@^1.1.1:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/use-memo-one/-/use-memo-one-1.1.3.tgz#2fd2e43a2169eabc7496960ace8c79efef975e99"
-  integrity sha512-g66/K7ZQGYrI6dy8GLpVcMsBp4s17xNkYJVSMvTEevGy3nDxHOfE6z8BVE22+5G5x7t3+bhzrlTDB7ObrEE0cQ==
-
 use-sidecar@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/use-sidecar/-/use-sidecar-1.1.2.tgz#2f43126ba2d7d7e117aa5855e5d8f0276dfe73c2"

From b9544a2ce4454de8122be2f330e993a58253fa30 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 15:00:30 -0700
Subject: [PATCH 103/169] more cleaning

---
 .../LeaderboardListingPage.tsx                |  15 +-
 .../query/configEditorQuery.ts                | 187 ------------------
 2 files changed, 12 insertions(+), 190 deletions(-)
 delete mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index eb3708b9ea4..eb79a665317 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -101,6 +101,9 @@ const LeaderboardTable: React.FC<{
   const history = useHistory();
   const {peekingRouter} = useWeaveflowRouteContext();
   const {useRootObjectVersions} = useWFHooks();
+
+  // TODO: One `useCollectionObjects` lands from the online
+  // evals project, switch to that (much more type safe)
   const leaderboardObjectVersions = useRootObjectVersions(
     props.entity,
     props.project,
@@ -147,14 +150,20 @@ const LeaderboardTable: React.FC<{
   );
 };
 
+const generateLeaderboardId = () => {
+  const timestamp = new Date().getTime();
+  const timestampHex = timestamp.toString(36);
+  return `leaderboard-${timestampHex}`;
+};
+
 const useCreateLeaderboard = (entity: string, project: string) => {
   const getTsClient = useGetTraceServerClientContext();
   const client = getTsClient();
 
+  // TODO: One `useCreateCollectionObject` lands from the online
+  // evals project, switch to that (much more type safe)
   const createLeaderboard = async () => {
-    const objectId = `leaderboard-${new Date()
-      .toISOString()
-      .replace(/[:.]/g, '-')}`;
+    const objectId = generateLeaderboardId();
     await client.objCreate({
       obj: {
         project_id: projectIdFromParts({entity, project}),
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
deleted file mode 100644
index 52a784328bd..00000000000
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/query/configEditorQuery.ts
+++ /dev/null
@@ -1,187 +0,0 @@
-/**
- * This file implements a memoized version of getLeaderboardGroupableData for improved performance.
- */
-
-// import _ from 'lodash';
-
-// import {getLeaderboardGroupableData} from '../../../views/Leaderboard/query/leaderboardQuery';
-// import {FilterAndGroupSpec} from '../../../views/Leaderboard/types/leaderboardConfigType';
-// import {TraceServerClient} from '../../wfReactInterface/traceServerClient';
-// import {projectIdFromParts} from '../../wfReactInterface/tsDataModelHooks';
-
-export type VersionDetails = {digest: string; index: number};
-
-// const memoedGetLeaderboardGroupableData = _.memoize(
-//   async (
-//     client: TraceServerClient,
-//     entity: string,
-//     project: string,
-//     spec: FilterAndGroupSpec = {}
-//   ) => {
-//     return getLeaderboardGroupableData(client, entity, project, spec);
-//   },
-//   (client, entity, project, spec) => {
-//     // Create a cache key based on the function arguments
-//     return JSON.stringify({entity, project, spec});
-//   }
-// );
-
-// export const fetchEvaluationNames = async (
-//   client: TraceServerClient,
-//   entity: string,
-//   project: string
-// ): Promise<string[]> => {
-//   return client
-//     .objsQuery({
-//       project_id: projectIdFromParts({entity, project}),
-//       filter: {
-//         base_object_classes: ['Evaluation'],
-//         is_op: false,
-//         latest_only: true,
-//       },
-//       metadata_only: true,
-//       sort_by: [{field: 'created_at', direction: 'desc'}],
-//     })
-//     .then(res => {
-//       return res.objs.map(obj => obj.object_id);
-//     });
-// };
-// export const fetchEvaluationVersionsForName = async (
-//   client: TraceServerClient,
-//   entity: string,
-//   project: string,
-//   name: string
-// ): Promise<VersionDetails[]> => {
-//   return client
-//     .objsQuery({
-//       project_id: projectIdFromParts({entity, project}),
-//       filter: {
-//         base_object_classes: ['Evaluation'],
-//         is_op: false,
-//         object_ids: [name],
-//       },
-//       metadata_only: true,
-//       sort_by: [{field: 'created_at', direction: 'desc'}],
-//     })
-//     .then(res => {
-//       return res.objs.map(obj => ({
-//         digest: obj.digest,
-//         index: obj.version_index,
-//       }));
-//     });
-// };
-// export const fetchDatasetNamesForSpec = async (
-//   client: TraceServerClient,
-//   entity: string,
-//   project: string,
-//   spec: FilterAndGroupSpec
-// ): Promise<string[]> => {
-//   // This is def a hacky/slow solution, just getting it working.
-//   const groupableData = await memoedGetLeaderboardGroupableData(
-//     client,
-//     entity,
-//     project,
-//     spec
-//   );
-//   return _.uniq(groupableData.map(g => g.row.datasetName));
-// };
-// export const fetchDatasetVersionsForSpecAndName = async (
-//   client: TraceServerClient,
-//   entity: string,
-//   project: string,
-//   spec: FilterAndGroupSpec,
-//   name: string
-// ): Promise<VersionDetails[]> => {
-//   const groupableData = await memoedGetLeaderboardGroupableData(
-//     client,
-//     entity,
-//     project,
-//     spec
-//   );
-
-//   return _.uniqBy(
-//     groupableData.map(g => ({digest: g.row.datasetVersion, index: -1})),
-//     o => `${o.digest}-${o.index}`
-//   );
-// };
-
-// export const fetchModelNamesForSpec = async (
-//   client: TraceServerClient,
-//   entity: string,
-//   project: string,
-//   spec: FilterAndGroupSpec
-// ): Promise<string[]> => {
-//   // This is def a hacky/slow solution, just getting it working.
-//   const groupableData = await memoedGetLeaderboardGroupableData(
-//     client,
-//     entity,
-//     project,
-//     spec
-//   );
-//   return _.uniq(groupableData.map(g => g.row.modelName));
-// };
-// export const fetchModelVersionsForSpecAndName = async (
-//   client: TraceServerClient,
-//   entity: string,
-//   project: string,
-//   spec: FilterAndGroupSpec,
-//   name: string
-// ): Promise<VersionDetails[]> => {
-//   // This is def a hacky/slow solution, just getting it working.
-//   const groupableData = await memoedGetLeaderboardGroupableData(
-//     client,
-//     entity,
-//     project,
-//     spec
-//   );
-//   return _.uniqBy(
-//     groupableData.map(g => ({digest: g.row.modelVersion, index: -1})),
-//     o => `${o.digest}-${o.index}`
-//   );
-// };
-// export const fetchScorerNamesForSpec = async (
-//   client: TraceServerClient,
-//   entity: string,
-//   project: string,
-//   spec: FilterAndGroupSpec
-// ): Promise<string[]> => {
-//   const groupableData = await memoedGetLeaderboardGroupableData(
-//     client,
-//     entity,
-//     project,
-//     spec
-//   );
-//   return _.uniq(groupableData.map(g => g.row.scorerName));
-// };
-// export const fetchScorerVersionsForSpecAndName = async (
-//   client: TraceServerClient,
-//   entity: string,
-//   project: string,
-//   spec: FilterAndGroupSpec,
-//   name: string
-// ): Promise<VersionDetails[]> => {
-//   const groupableData = await memoedGetLeaderboardGroupableData(
-//     client,
-//     entity,
-//     project,
-//     spec
-//   );
-//   return _.uniqBy(
-//     groupableData.map(g => ({digest: g.row.scorerVersion, index: -1})),
-//     o => `${o.digest}-${o.index}`
-//   );
-// };
-// export const fetchMetricPathsForSpec = async (
-//   client: TraceServerClient,
-//   entity: string,
-//   project: string,
-//   spec: FilterAndGroupSpec
-// ): Promise<string[]> => {
-//   const groupableData = await memoedGetLeaderboardGroupableData(
-//     client,
-//     entity,
-//     project,
-//     spec
-//   );
-//   return _.uniq(groupableData.map(g => g.row.metricPath));
-// };

From 300be67fdc04103e1e1477b918d66bf7e90f4348 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 15:06:49 -0700
Subject: [PATCH 104/169] more cleaning

---
 .../LeaderboardListingPage.tsx                |  3 +-
 .../traceServerDirectClient.ts                | 34 +++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index eb79a665317..a87a9cfb7d6 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -13,6 +13,7 @@ import {SimplePageLayout} from '../common/SimplePageLayout';
 import {ObjectVersionsTable} from '../ObjectVersionsPage';
 import {useWFHooks} from '../wfReactInterface/context';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
+import {sanitizeObjectId} from '../wfReactInterface/traceServerDirectClient';
 import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
 import {ObjectVersionSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 import {useIsEditor} from './LeaderboardPage';
@@ -163,7 +164,7 @@ const useCreateLeaderboard = (entity: string, project: string) => {
   // TODO: One `useCreateCollectionObject` lands from the online
   // evals project, switch to that (much more type safe)
   const createLeaderboard = async () => {
-    const objectId = generateLeaderboardId();
+    const objectId = sanitizeObjectId(generateLeaderboardId());
     await client.objCreate({
       obj: {
         project_id: projectIdFromParts({entity, project}),
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
index a8e505a895b..eae71cfeb68 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
@@ -48,6 +48,33 @@ import {
   TraceTableQueryStatsRes,
 } from './traceServerClientTypes';
 
+/**
+ * Sanitizes an object name by replacing non-alphanumeric characters with dashes and enforcing length limits.
+ * This matches the Python implementation in weave_client.py.
+ *
+ * @param name The name to sanitize
+ * @returns The sanitized name
+ * @throws Error if the resulting name would be empty
+ */
+export function sanitizeObjectId(name: string): string {
+  // Replace any non-word chars (except dots and underscores) with dashes
+  let res = name.replace(/[^\w._]+/g, '-');
+  // Replace multiple consecutive dashes/dots/underscores with a single dash
+  res = res.replace(/([._-]{2,})+/g, '-');
+  // Remove leading/trailing dashes and underscores
+  res = res.replace(/^[-_]+|[-_]+$/g, '');
+
+  if (!res) {
+    throw new Error(`Invalid object name: ${name}`);
+  }
+
+  if (res.length > 128) {
+    res = res.slice(0, 128);
+  }
+
+  return res;
+}
+
 export class DirectTraceServerClient {
   private baseUrl: string;
   private inFlightFetchesRequests: Record<
@@ -227,6 +254,13 @@ export class DirectTraceServerClient {
   }
 
   public objCreate(req: TraceObjCreateReq): Promise<TraceObjCreateRes> {
+    const initialObjectId = req.obj.object_id;
+    const sanitizedObjectId = sanitizeObjectId(initialObjectId);
+    if (sanitizedObjectId !== initialObjectId) {
+      throw new Error(
+        `Invalid object name: ${initialObjectId}, sanitized to ${sanitizedObjectId}`
+      );
+    }
     return this.makeRequest<TraceObjCreateReq, TraceObjCreateRes>(
       '/obj/create',
       req

From 2d9522fd8a0c82fb2f2dfbb3a59b0d0e56cd7b97 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 15:07:10 -0700
Subject: [PATCH 105/169] more cleaning

---
 .../Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 9977d81e93e..3d544f921f3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -429,7 +429,7 @@ export const useIsEditor = (entity: string) => {
   }, [entity, loadingUserInfo, userInfo]);
 };
 
-export const StyledReactMarkdown = styled(ReactMarkdown)`
+const StyledReactMarkdown = styled(ReactMarkdown)`
   > *:first-child {
     margin-top: 0;
   }

From 08e6687e2638f585f485a1a1fd511d6a87b2507e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 15:11:21 -0700
Subject: [PATCH 106/169] more cleaning

---
 weave/flow/leaderboard.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/weave/flow/leaderboard.py b/weave/flow/leaderboard.py
index ff0f513e823..48c7cb5f009 100644
--- a/weave/flow/leaderboard.py
+++ b/weave/flow/leaderboard.py
@@ -15,7 +15,8 @@ class LeaderboardColumn(BaseModel):
     summary_metric_path_parts: list[str] = field(default_factory=list)
     should_minimize: Optional[bool] = None
 
-
+# TODO: Merge `Leaderboard` into weave/trace_server/interface/base_models/base_model_registry.py after
+# Online evals lands
 class Leaderboard(weave.Object):
     columns: list[LeaderboardColumn]
 
@@ -54,7 +55,9 @@ def get_leaderboard_results(
 
     res_map: dict[str, LeaderboardModelResult] = {}
     for call in calls:
-        # Frustrating that we have to get the ref like this (such a waste of network calls)
+        # Frustrating that we have to get the ref like this. Since the 
+        # `Call` object auto-derefs the inputs (making a network request),
+        # we have to manually get the ref here... waste of network calls.
         call_ref = get_ref(call)
         if call_ref is None:
             continue

From 000f50a6b30b6a516098b92957b8530b85f96fa4 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 15:11:31 -0700
Subject: [PATCH 107/169] more cleaning

---
 weave/flow/leaderboard.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/weave/flow/leaderboard.py b/weave/flow/leaderboard.py
index 48c7cb5f009..26b8191d417 100644
--- a/weave/flow/leaderboard.py
+++ b/weave/flow/leaderboard.py
@@ -15,6 +15,7 @@ class LeaderboardColumn(BaseModel):
     summary_metric_path_parts: list[str] = field(default_factory=list)
     should_minimize: Optional[bool] = None
 
+
 # TODO: Merge `Leaderboard` into weave/trace_server/interface/base_models/base_model_registry.py after
 # Online evals lands
 class Leaderboard(weave.Object):
@@ -55,7 +56,7 @@ def get_leaderboard_results(
 
     res_map: dict[str, LeaderboardModelResult] = {}
     for call in calls:
-        # Frustrating that we have to get the ref like this. Since the 
+        # Frustrating that we have to get the ref like this. Since the
         # `Call` object auto-derefs the inputs (making a network request),
         # we have to manually get the ref here... waste of network calls.
         call_ref = get_ref(call)

From d9996d8c32b5cdcef0e7a12a66eeed505c2c82a4 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 15:20:53 -0700
Subject: [PATCH 108/169] Cleanup and Refactoring of Names

---
 .../LeaderboardConfigEditor.tsx               |  8 +++---
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 16 +++++------
 .../Browse3/pages/common/SimplePageLayout.tsx |  2 +-
 .../traceServerDirectClient.ts                |  2 ++
 .../views/Leaderboard/query/hookAdapters.ts   | 12 ++++----
 .../Leaderboard/query/leaderboardQuery.ts     | 28 ++++++++++---------
 .../types/leaderboardConfigType.ts            |  4 ++-
 7 files changed, 38 insertions(+), 34 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index a2e1865aafb..065b2522aee 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -23,7 +23,7 @@ import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
 import {parseRefMaybe} from '../../../Browse2/SmallRef';
 import {
   ALL_VALUE,
-  PythonLeaderboardObjectVal,
+  LeaderboardObjectVal,
 } from '../../views/Leaderboard/types/leaderboardConfigType';
 import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
@@ -34,10 +34,10 @@ import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
 export const LeaderboardConfigEditor: React.FC<{
   entity: string;
   project: string;
-  leaderboardVal: PythonLeaderboardObjectVal;
+  leaderboardVal: LeaderboardObjectVal;
   saving: boolean;
   isDirty: boolean;
-  setWorkingCopy: (leaderboardVal: PythonLeaderboardObjectVal) => void;
+  setWorkingCopy: (leaderboardVal: LeaderboardObjectVal) => void;
   discardChanges: () => void;
   commitChanges: () => void;
 }> = ({
@@ -188,7 +188,7 @@ export const LeaderboardConfigEditor: React.FC<{
 };
 
 const ColumnEditor: React.FC<{
-  column: PythonLeaderboardObjectVal['columns'][0];
+  column: LeaderboardObjectVal['columns'][0];
   index: number;
   evalObjs: EvaluationHelperObj[];
   entity: string;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 3d544f921f3..34e7b1b1e3a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -22,8 +22,8 @@ import {
   LeaderboardColumnOrderType,
   LeaderboardGrid,
 } from '../../views/Leaderboard/LeaderboardGrid';
-import {usePythonLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
-import {PythonLeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
+import {useSavedLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
+import {LeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {SimplePageLayout} from '../common/SimplePageLayout';
 import {useWFHooks} from '../wfReactInterface/context';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
@@ -141,9 +141,7 @@ const useUpdateLeaderboard = (
   const getTsClient = useGetTraceServerClientContext();
   const client = getTsClient();
 
-  const updateLeaderboard = async (
-    leaderboardVal: PythonLeaderboardObjectVal
-  ) => {
+  const updateLeaderboard = async (leaderboardVal: LeaderboardObjectVal) => {
     return await client.objCreate({
       obj: {
         project_id: projectIdFromParts({entity, project}),
@@ -167,7 +165,7 @@ export const LeaderboardPageContentInner: React.FC<
     isEditing: boolean;
     setIsEditing: (isEditing: boolean) => void;
   } & {
-    leaderboardVal: PythonLeaderboardObjectVal;
+    leaderboardVal: LeaderboardObjectVal;
   }
 > = props => {
   const updateLeaderboard = useUpdateLeaderboard(
@@ -181,7 +179,7 @@ export const LeaderboardPageContentInner: React.FC<
   useEffect(() => {
     props.setName(workingLeaderboardValCopy.name);
   }, [props, workingLeaderboardValCopy.name]);
-  const {loading, data, evalData} = usePythonLeaderboardData(
+  const {loading, data, evalData} = useSavedLeaderboardData(
     props.entity,
     props.project,
     workingLeaderboardValCopy
@@ -325,7 +323,7 @@ export const ToggleLeaderboardConfig: React.FC<{
 
 const parseLeaderboardVal = (
   leaderboardVal: any
-): PythonLeaderboardObjectVal | null => {
+): LeaderboardObjectVal | null => {
   if (typeof leaderboardVal !== 'object' || leaderboardVal == null) {
     return null;
   }
@@ -376,7 +374,7 @@ const parseLeaderboardVal = (
         summary_metric_path_parts: summaryMetricParts,
       };
     })
-    .filter(column => column != null) as PythonLeaderboardObjectVal['columns'];
+    .filter(column => column != null) as LeaderboardObjectVal['columns'];
 
   return {
     name,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx
index f50654b3396..4a5d169ad29 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/SimplePageLayout.tsx
@@ -24,7 +24,7 @@ export const SimplePageLayoutContext =
   createContext<SimplePageLayoutContextType>({});
 
 export const SimplePageLayout: FC<{
-  title: ReactNode;
+  title: string;
   tabs: Array<{
     label: string;
     content: ReactNode;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
index eae71cfeb68..fd99adf2c43 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
@@ -257,6 +257,8 @@ export class DirectTraceServerClient {
     const initialObjectId = req.obj.object_id;
     const sanitizedObjectId = sanitizeObjectId(initialObjectId);
     if (sanitizedObjectId !== initialObjectId) {
+      // Caller is expected to sanitize the object id. We should be doing this
+      // on the server, but it is currently disabled.
       throw new Error(
         `Invalid object name: ${initialObjectId}, sanitized to ${sanitizedObjectId}`
       );
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
index 34a492f356a..426d76020fb 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
@@ -4,13 +4,13 @@ import {useEffect, useState} from 'react';
 import {useGetTraceServerClientContext} from '../../../pages/wfReactInterface/traceServerClientContext';
 import {
   FilterAndGroupSpec,
-  PythonLeaderboardObjectVal,
+  LeaderboardObjectVal,
 } from '../types/leaderboardConfigType';
 import {
   getLeaderboardData,
   getPythonLeaderboardData,
   GroupedLeaderboardData,
-  PythonLeaderboardEvalData,
+  LeaderboardObjectEvalData,
 } from './leaderboardQuery';
 
 type LeaderboardDataState = {
@@ -45,14 +45,14 @@ export const useLeaderboardData = (
   return state;
 };
 
-export const usePythonLeaderboardData = (
+export const useSavedLeaderboardData = (
   entity: string,
   project: string,
-  val: PythonLeaderboardObjectVal
-): LeaderboardDataState & {evalData: PythonLeaderboardEvalData} => {
+  val: LeaderboardObjectVal
+): LeaderboardDataState & {evalData: LeaderboardObjectEvalData} => {
   const getTraceServerClient = useGetTraceServerClientContext();
   const [state, setState] = useState<
-    LeaderboardDataState & {evalData: PythonLeaderboardEvalData}
+    LeaderboardDataState & {evalData: LeaderboardObjectEvalData}
   >({
     loading: true,
     data: {modelGroups: {}},
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
index 7c6eaff0427..733ed7cc6b9 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
@@ -17,7 +17,7 @@ import {
 import {
   ALL_VALUE,
   FilterAndGroupSpec,
-  PythonLeaderboardObjectVal,
+  LeaderboardObjectVal,
 } from '../types/leaderboardConfigType';
 
 export type LeaderboardValueRecord = {
@@ -42,10 +42,12 @@ export type LeaderboardValueRecord = {
   createdAt: Date;
   sourceEvaluationCallId: string;
   sourceEvaluationObjectRef: string;
-  shouldMinimize?: boolean; // hack to put this here
+  // A bit hacky to denormalize `shouldMinimize` here, but it's convenient
+  // for the caller and not externally visible
+  shouldMinimize?: boolean;
 };
 
-export type GroupableLeaderboardValueRecord = {
+type GroupableLeaderboardValueRecord = {
   modelGroup: string;
   datasetGroup: string;
   scorerGroup: string;
@@ -76,7 +78,7 @@ export type GroupedLeaderboardModelGroup<
   };
 };
 
-export const getEvaluationObjectsForSpec = async (
+const getEvaluationObjectsForSpec = async (
   client: TraceServerClient,
   entity: string,
   project: string,
@@ -117,7 +119,7 @@ export const getEvaluationObjectsForSpec = async (
   return allEvaluationObjectsRes;
 };
 
-export const getLeaderboardGroupableData = async (
+const getLeaderboardGroupableData = async (
   client: TraceServerClient,
   entity: string,
   project: string,
@@ -492,12 +494,12 @@ export const getPythonLeaderboardData = async (
   client: TraceServerClient,
   entity: string,
   project: string,
-  val: PythonLeaderboardObjectVal
+  val: LeaderboardObjectVal
 ): Promise<{
   finalData: GroupedLeaderboardData;
-  evalData: PythonLeaderboardEvalData;
+  evalData: LeaderboardObjectEvalData;
 }> => {
-  const {groupableData, evalData} = await getPythonLeaderboardGroupableData(
+  const {groupableData, evalData} = await getLeaderboardObjectGroupableData(
     client,
     entity,
     project,
@@ -537,7 +539,7 @@ export const getPythonLeaderboardData = async (
   return {finalData, evalData};
 };
 
-export type PythonLeaderboardEvalData = {
+export type LeaderboardObjectEvalData = {
   [evalRefUri: string]: {
     datasetGroup: string;
     scorers: {
@@ -546,14 +548,14 @@ export type PythonLeaderboardEvalData = {
   };
 };
 
-const getPythonLeaderboardGroupableData = async (
+const getLeaderboardObjectGroupableData = async (
   client: TraceServerClient,
   entity: string,
   project: string,
-  val: PythonLeaderboardObjectVal
+  val: LeaderboardObjectVal
 ): Promise<{
   groupableData: GroupableLeaderboardValueRecord[];
-  evalData: PythonLeaderboardEvalData;
+  evalData: LeaderboardObjectEvalData;
 }> => {
   const evalObjectRefs = _.uniq(
     val.columns.map(col => col.evaluation_object_ref)
@@ -587,7 +589,7 @@ const getPythonLeaderboardGroupableData = async (
   const allEvaluationCallsRes = await allEvaluationCallsProm;
 
   const data: GroupableLeaderboardValueRecord[] = [];
-  const evalData: PythonLeaderboardEvalData = {};
+  const evalData: LeaderboardObjectEvalData = {};
   allEvaluationCallsRes.calls.forEach(call => {
     val.columns.forEach(col => {
       const evalObjRefUri = call.inputs.self;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts
index 4b322733fc5..dd771c206b8 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts
@@ -44,7 +44,9 @@ export type FilterAndGroupModelSpec = {
   groupAllVersions?: boolean;
 };
 
-export type PythonLeaderboardObjectVal = {
+// TODO: Merge into weave-js/src/components/PagePanelComponents/Home/Browse3/collections/collectionRegistry.ts after
+// Online evals lands
+export type LeaderboardObjectVal = {
   name: string;
   description: string;
   columns: Array<{

From 9ce1d6c5ace9ec34627120d92594faebbd7df68e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 15:54:37 -0700
Subject: [PATCH 109/169] Small cleanup

---
 .../Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx  | 4 ++--
 .../Home/Browse3/pages/ObjectVersionsPage.tsx                 | 4 +++-
 .../PagePanelComponents/Home/Browse3/pages/common/Links.tsx   | 3 ++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index a87a9cfb7d6..ed1c2bbd150 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -103,7 +103,7 @@ const LeaderboardTable: React.FC<{
   const {peekingRouter} = useWeaveflowRouteContext();
   const {useRootObjectVersions} = useWFHooks();
 
-  // TODO: One `useCollectionObjects` lands from the online
+  // TODO: Once `useCollectionObjects` lands from the online
   // evals project, switch to that (much more type safe)
   const leaderboardObjectVersions = useRootObjectVersions(
     props.entity,
@@ -161,7 +161,7 @@ const useCreateLeaderboard = (entity: string, project: string) => {
   const getTsClient = useGetTraceServerClientContext();
   const client = getTsClient();
 
-  // TODO: One `useCreateCollectionObject` lands from the online
+  // TODO: Once `useCreateCollectionObject` lands from the online
   // evals project, switch to that (much more type safe)
   const createLeaderboard = async () => {
     const objectId = sanitizeObjectId(generateLeaderboardId());
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
index f9ccd0283db..eed18b9a70c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ObjectVersionsPage.tsx
@@ -244,7 +244,9 @@ export const ObjectVersionsTable: React.FC<{
               ? obj.objectId
               : objectVersionText(obj.objectId, obj.versionIndex);
 
-            // Maybe this should be conditional?
+            // This allows us to use the object name as the link text
+            // if it is available. Probably should make this workfor
+            // the object version link as well.
             if (obj.val.name) {
               text = obj.val.name;
             }
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
index 93a19fa2cb8..13b0d4d4d7c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/Links.tsx
@@ -43,7 +43,7 @@ export const Link = styled(LinkComp)<LinkProps>`
 `;
 Link.displayName = 'S.Link';
 
-export const FakeLink = styled.div<LinkProps>`
+const FakeLink = styled.div<LinkProps>`
   font-weight: 600;
   color: ${p => (p.$variant === 'secondary' ? MOON_700 : TEAL_600)};
   &:hover {
@@ -351,6 +351,7 @@ export const CustomLink: React.FC<{
   variant?: LinkVariant;
   icon?: React.ReactNode;
 }> = props => {
+  // Used to look like our other links, but delegate to a custom onClick
   return (
     <LinkWrapper
       onClick={props.onClick}

From 7ce789d1ce9f0a1202a222fcb1cc77be45b02ed9 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 17:59:20 -0700
Subject: [PATCH 110/169] Cleanups

---
 demos/leaderboard_demo.ipynb                  | 310 ++++++++++++++----
 .../views/Leaderboard/LeaderboardGrid.tsx     |   2 +-
 weave/trace/api.py                            |   9 +
 weave/trace/urls.py                           |   4 +
 4 files changed, 262 insertions(+), 63 deletions(-)

diff --git a/demos/leaderboard_demo.ipynb b/demos/leaderboard_demo.ipynb
index ceb6795b70e..109ac751e58 100644
--- a/demos/leaderboard_demo.ipynb
+++ b/demos/leaderboard_demo.ipynb
@@ -1,11 +1,60 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<!-- docusaurus_head_meta::start\n",
+    "---\n",
+    "title: Leaderboard Quickstart\n",
+    "---\n",
+    "docusaurus_head_meta::end -->\n",
+    "\n",
+    "<!--- @wandbcode{leaderboard-demo} -->"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Leaderboard Quickstart\n",
+    "\n",
+    "In this notebook we will learn to use Weave's Leaderboard to compare model performance across different datasets and scoring functions. Specifically, we will:\n",
+    "\n",
+    "1. Generate a dataset of fake zip code data\n",
+    "2. Author some scoring functions and evaluate a baseline model.\n",
+    "3. Use these techniques to evaluate a matrix of models vs evaluations.\n",
+    "4. Review the leaderboard in the Weave UI."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
+    "import weave\n",
+    "\n",
+    "weave.init(\"leaderboard-demo\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 1: Generate a dataset of fake zip code data\n",
+    "\n",
+    "First we will create a function `generate_dataset_rows` that generates a list of fake zip code data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
     "from openai import OpenAI\n",
     "from pydantic import BaseModel\n",
     "\n",
@@ -19,34 +68,48 @@
     "    median_income: int\n",
     "    known_for: str\n",
     "\n",
+    "\n",
     "class Rows(BaseModel):\n",
     "    rows: list[Row]\n",
     "\n",
-    "def generate_dataset_rows(location: str = \"United States\", count: int = 5, year: int = 2022):\n",
+    "\n",
+    "def generate_dataset_rows(\n",
+    "    location: str = \"United States\", count: int = 5, year: int = 2022\n",
+    "):\n",
     "    client = OpenAI()\n",
     "\n",
-    "    completion = client.beta.chat.completions.parse(\n",
+    "    completion = client.chat.completions.create(\n",
     "        model=\"gpt-4o-mini\",\n",
     "        messages=[\n",
     "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
-    "            {\"role\": \"user\", \"content\": f\"Please generate {count} rows of data for random zip codes in {location} for the year {year}.\"},\n",
+    "            {\n",
+    "                \"role\": \"user\",\n",
+    "                \"content\": f\"Please generate {count} rows of data for random zip codes in {location} for the year {year}.\",\n",
+    "            },\n",
     "        ],\n",
-    "        response_format=Rows\n",
-    "        \n",
+    "        response_format={\n",
+    "            \"type\": \"json_schema\",\n",
+    "            \"json_schema\": {\n",
+    "                \"name\": \"response_format\",\n",
+    "                \"schema\": Rows.model_json_schema(),\n",
+    "            },\n",
+    "        },\n",
     "    )\n",
     "\n",
-    "    return completion.choices[0].message.parsed.model_dump()['rows']\n",
-    "\n",
-    "rows = generate_dataset_rows()"
+    "    return json.loads(completion.choices[0].message.content)[\"rows\"]"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 2,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "project_name = \"leaderboard-demo-3\""
+    "## Step 2: Author scoring functions\n",
+    "\n",
+    "Next we will author 3 scoring functions:\n",
+    "\n",
+    "1. `check_concrete_fields`: Checks if the model output matches the expected city and state.\n",
+    "2. `check_value_fields`: Checks if the model output is within 10% of the expected population and median income.\n",
+    "3. `check_subjective_fields`: Uses a LLM to check if the model output matches the expected \"known for\" field.\n"
    ]
   },
   {
@@ -55,66 +118,100 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import weave\n",
-    "\n",
-    "\n",
     "@weave.op\n",
-    "def check_concrete_fields(city: str, state: str, model_output: dict):\n",
+    "def check_concrete_fields(city: str, state: str, output: dict):\n",
     "    return {\n",
-    "        \"city_match\": city == model_output[\"city\"],\n",
-    "        \"state_match\": state == model_output[\"state\"],\n",
+    "        \"city_match\": city == output[\"city\"],\n",
+    "        \"state_match\": state == output[\"state\"],\n",
     "    }\n",
     "\n",
+    "\n",
     "@weave.op\n",
-    "def check_value_fields(avg_temp_f: float, population: int, median_income: int, model_output: dict):\n",
+    "def check_value_fields(\n",
+    "    avg_temp_f: float, population: int, median_income: int, output: dict\n",
+    "):\n",
     "    return {\n",
-    "        \"avg_temp_f_err\": abs(avg_temp_f - model_output[\"avg_temp_f\"]) / avg_temp_f,\n",
-    "        \"population_err\": abs(population - model_output[\"population\"]) / population,\n",
-    "        \"median_income_err\": abs(median_income - model_output[\"median_income\"]) / median_income,\n",
+    "        \"avg_temp_f_err\": abs(avg_temp_f - output[\"avg_temp_f\"]) / avg_temp_f,\n",
+    "        \"population_err\": abs(population - output[\"population\"]) / population,\n",
+    "        \"median_income_err\": abs(median_income - output[\"median_income\"])\n",
+    "        / median_income,\n",
     "    }\n",
     "\n",
+    "\n",
     "@weave.op\n",
-    "def check_subjective_fields(zip_code: str, known_for: str, model_output: dict):\n",
+    "def check_subjective_fields(zip_code: str, known_for: str, output: dict):\n",
     "    client = OpenAI()\n",
     "\n",
     "    class Response(BaseModel):\n",
     "        correct_known_for: bool\n",
     "\n",
-    "    completion = client.beta.chat.completions.parse(\n",
+    "    completion = client.chat.completions.create(\n",
     "        model=\"gpt-4o-mini\",\n",
     "        messages=[\n",
     "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
-    "            {\"role\": \"user\", \"content\": f\"My student was asked what the zip code {zip_code} is best known best for. The right answer is '{known_for}', and they said '{model_output['known_for']}'. Is their answer correct?\"},\n",
+    "            {\n",
+    "                \"role\": \"user\",\n",
+    "                \"content\": f\"My student was asked what the zip code {zip_code} is best known best for. The right answer is '{known_for}', and they said '{output['known_for']}'. Is their answer correct?\",\n",
+    "            },\n",
     "        ],\n",
-    "        response_format=Response\n",
+    "        response_format={\n",
+    "            \"type\": \"json_schema\",\n",
+    "            \"json_schema\": {\n",
+    "                \"name\": \"response_format\",\n",
+    "                \"schema\": Response.model_json_schema(),\n",
+    "            },\n",
+    "        },\n",
     "    )\n",
     "\n",
-    "    return completion.choices[0].message.parsed.model_dump()\n",
-    "\n",
+    "    return json.loads(completion.choices[0].message.content)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 3: Create a simple Evaluation\n",
     "\n",
+    "Next we define a simple evaliation using our fake data and scoring functions.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rows = generate_dataset_rows()\n",
     "evaluation = weave.Evaluation(\n",
-    "    name=\"United States - 2022 - Concrete Fields\",\n",
+    "    name=\"United States - 2022\",\n",
     "    dataset=rows,\n",
     "    scorers=[\n",
     "        check_concrete_fields,\n",
     "        check_value_fields,\n",
     "        check_subjective_fields,\n",
-    "    ]\n",
+    "    ],\n",
     ")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 4: Evaluate a baseline model\n",
+    "\n",
+    "Now we will evaluate a baseline model which returns a static response.\n"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "weave.init(project_name)\n",
-    "\n",
     "@weave.op\n",
     "def baseline_model(zip_code: str):\n",
     "    return {\n",
-    "        \"city\": \"California\",\n",
+    "        \"city\": \"New York\",\n",
     "        \"state\": \"NY\",\n",
     "        \"avg_temp_f\": 50.0,\n",
     "        \"population\": 1000000,\n",
@@ -122,28 +219,43 @@
     "        \"known_for\": \"The Big Apple\",\n",
     "    }\n",
     "\n",
+    "\n",
     "await evaluation.evaluate(baseline_model)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 5: Create more Models\n",
+    "\n",
+    "Now we will create 2 more models to compare against the baseline."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "weave.init(project_name)\n",
-    "\n",
     "@weave.op\n",
     "def gpt_4o_mini_no_context(zip_code: str):\n",
     "    client = OpenAI()\n",
     "\n",
-    "    completion = client.beta.chat.completions.parse(\n",
+    "    completion = client.chat.completions.create(\n",
     "        model=\"gpt-4o-mini\",\n",
     "        messages=[{\"role\": \"user\", \"content\": f\"\"\"Zip code {zip_code}\"\"\"}],\n",
-    "        response_format=Row\n",
+    "        response_format={\n",
+    "            \"type\": \"json_schema\",\n",
+    "            \"json_schema\": {\n",
+    "                \"name\": \"response_format\",\n",
+    "                \"schema\": Row.model_json_schema(),\n",
+    "            },\n",
+    "        },\n",
     "    )\n",
     "\n",
-    "    return completion.choices[0].message.parsed.model_dump()\n",
+    "    return json.loads(completion.choices[0].message.content)\n",
+    "\n",
     "\n",
     "await evaluation.evaluate(gpt_4o_mini_no_context)"
    ]
@@ -154,28 +266,47 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "weave.init(project_name)\n",
-    "\n",
     "@weave.op\n",
     "def gpt_4o_mini_with_context(zip_code: str):\n",
     "    client = OpenAI()\n",
     "\n",
-    "    completion = client.beta.chat.completions.parse(\n",
+    "    completion = client.chat.completions.create(\n",
     "        model=\"gpt-4o-mini\",\n",
-    "        messages=[{\"role\": \"user\", \"content\": f\"\"\"Please answer the following questions about the zip code {zip_code}:\n",
+    "        messages=[\n",
+    "            {\n",
+    "                \"role\": \"user\",\n",
+    "                \"content\": f\"\"\"Please answer the following questions about the zip code {zip_code}:\n",
     "                   1. What is the city?\n",
     "                   2. What is the state?\n",
     "                   3. What is the average temperature in Fahrenheit?\n",
     "                   4. What is the population?\n",
     "                   5. What is the median income?\n",
     "                   6. What is the most well known thing about this zip code?\n",
-    "                   \"\"\"}],\n",
-    "        response_format=Row\n",
+    "                   \"\"\",\n",
+    "            }\n",
+    "        ],\n",
+    "        response_format={\n",
+    "            \"type\": \"json_schema\",\n",
+    "            \"json_schema\": {\n",
+    "                \"name\": \"response_format\",\n",
+    "                \"schema\": Row.model_json_schema(),\n",
+    "            },\n",
+    "        },\n",
     "    )\n",
     "\n",
-    "    return completion.choices[0].message.parsed.model_dump()\n",
+    "    return json.loads(completion.choices[0].message.content)\n",
     "\n",
-    "await evaluation.evaluate(baseline_model)"
+    "\n",
+    "await evaluation.evaluate(gpt_4o_mini_with_context)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 6: Create more Evaluations\n",
+    "\n",
+    "Now we will evaluate a matrix of models vs evaluations.\n"
    ]
   },
   {
@@ -192,19 +323,27 @@
     "evaluations = [\n",
     "    weave.Evaluation(\n",
     "        name=\"United States - 2022\",\n",
-    "        dataset=weave.Dataset(name=\"United States - 2022\", rows=generate_dataset_rows(\"United States\", 5, 2022)),\n",
+    "        dataset=weave.Dataset(\n",
+    "            name=\"United States - 2022\",\n",
+    "            rows=generate_dataset_rows(\"United States\", 5, 2022),\n",
+    "        ),\n",
     "        scorers=scorers,\n",
     "    ),\n",
     "    weave.Evaluation(\n",
     "        name=\"California - 2022\",\n",
-    "        dataset=weave.Dataset(name=\"California - 2022\", rows=generate_dataset_rows(\"California\", 5, 2022)),\n",
+    "        dataset=weave.Dataset(\n",
+    "            name=\"California - 2022\", rows=generate_dataset_rows(\"California\", 5, 2022)\n",
+    "        ),\n",
+    "        scorers=scorers,\n",
+    "    ),\n",
+    "    weave.Evaluation(\n",
+    "        name=\"United States - 2000\",\n",
+    "        dataset=weave.Dataset(\n",
+    "            name=\"United States - 2000\",\n",
+    "            rows=generate_dataset_rows(\"United States\", 5, 2000),\n",
+    "        ),\n",
     "        scorers=scorers,\n",
     "    ),\n",
-    "    # weave.Evaluation(\n",
-    "    #     name=\"United States - 2000\",\n",
-    "    #     dataset=weave.Dataset(name=\"United States - 2000\", rows=generate_dataset_rows(\"United States\", 5, 2000)),\n",
-    "    #     scorers=scorers,\n",
-    "    # ),\n",
     "]\n",
     "models = [\n",
     "    baseline_model,\n",
@@ -214,24 +353,71 @@
     "\n",
     "for evaluation in evaluations:\n",
     "    for model in models:\n",
-    "        await evaluation.evaluate(model, __weave={\"display_name\": evaluation.name + \":\" + model.__name__})"
+    "        await evaluation.evaluate(\n",
+    "            model, __weave={\"display_name\": evaluation.name + \":\" + model.__name__}\n",
+    "        )"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "\n"
+    "## Step 7: Review the Leaderboard\n",
+    "\n",
+    "You can create a new leaderboard by navigating to the leaderboard tab in the UI and clicking \"Create Leaderboard\".\n",
+    "\n",
+    "We can also generate a leaderboard directly from Python:"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": 9,
    "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "📦 Published to https://wandb.ai/timssweeney/leaderboard-demo/weave/leaderboards/Zip-Code-World-Knowledge\n"
+     ]
+    }
+   ],
    "source": [
-    "https://beta.wandb.ai/?betaVersion=234f24492fa0a1999a084aeb93d4d2128944512c\n",
-    "https://beta.wandb.ai/timssweeney/leaderboard-demo-1/weave/leaderboards/leaderboard-2024-10-23T07-43-07-604Z?edit=true"
+    "from weave.flow import leaderboard\n",
+    "from weave.trace.weave_client import get_ref\n",
+    "\n",
+    "spec = leaderboard.Leaderboard(\n",
+    "    name=\"Zip Code World Knowledge\",\n",
+    "    description=\"\"\"\n",
+    "This leaderboard compares the performance of models in terms of world knowledge about zip codes.\n",
+    "\n",
+    "### Columns\n",
+    "\n",
+    "1. **State Match against `United States - 2022`**: The fraction of zip codes that the model correctly identified the state for.\n",
+    "2. **Avg Temp F Error against `California - 2022`**: The mean absolute error of the model's average temperature prediction.\n",
+    "3. **Correct Known For against `United States - 2000`**: The fraction of zip codes that the model correctly identified the most well known thing about the zip code.\n",
+    "\"\"\",\n",
+    "    columns=[\n",
+    "        leaderboard.LeaderboardColumn(\n",
+    "            evaluation_object_ref=get_ref(evaluations[0]).uri(),\n",
+    "            scorer_name=\"check_concrete_fields\",\n",
+    "            summary_metric_path_parts=[\"state_match\", \"true_fraction\"],\n",
+    "        ),\n",
+    "        leaderboard.LeaderboardColumn(\n",
+    "            evaluation_object_ref=get_ref(evaluations[1]).uri(),\n",
+    "            scorer_name=\"check_value_fields\",\n",
+    "            should_minimize=True,\n",
+    "            summary_metric_path_parts=[\"avg_temp_f_err\", \"mean\"],\n",
+    "        ),\n",
+    "        leaderboard.LeaderboardColumn(\n",
+    "            evaluation_object_ref=get_ref(evaluations[2]).uri(),\n",
+    "            scorer_name=\"check_subjective_fields\",\n",
+    "            summary_metric_path_parts=[\"correct_known_for\", \"true_fraction\"],\n",
+    "        ),\n",
+    "    ],\n",
+    ")\n",
+    "\n",
+    "ref = weave.publish(spec)"
    ]
   }
  ],
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index 97e95189edd..9ff50fd3a3e 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -193,7 +193,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
                       if (inner == null) {
                         inner = <NotApplicable />;
                       } else if (typeof inner === 'number') {
-                        if (inner < 1) {
+                        if ((0 < inner && inner < 1) || metricPathGroupName.includes('fraction')) {
                           inner = `${(inner * 100).toFixed(2)}%`;
                         } else {
                           inner = `${inner.toFixed(2)}`;
diff --git a/weave/trace/api.py b/weave/trace/api.py
index fc4847bc0cb..35e94b7e8d2 100644
--- a/weave/trace/api.py
+++ b/weave/trace/api.py
@@ -101,6 +101,9 @@ def publish(obj: Any, name: Optional[str] = None) -> weave_client.ObjectRef:
 
     ref = client._save_object(obj, save_name, "latest")
 
+    # Avoid circular import
+    from weave.flow.leaderboard import Leaderboard
+
     if isinstance(ref, weave_client.ObjectRef):
         if isinstance(ref, weave_client.OpRef):
             url = urls.op_version_path(
@@ -109,6 +112,12 @@ def publish(obj: Any, name: Optional[str] = None) -> weave_client.ObjectRef:
                 ref.name,
                 ref.digest,
             )
+        elif isinstance(obj, Leaderboard):
+            url = urls.leaderboard_path(
+                ref.entity,
+                ref.project,
+                ref.name,
+            )
         # TODO(gst): once frontend has direct dataset/model links
         # elif isinstance(obj, weave_client.Dataset):
         else:
diff --git a/weave/trace/urls.py b/weave/trace/urls.py
index d5237c668b7..60a9b3a2a85 100644
--- a/weave/trace/urls.py
+++ b/weave/trace/urls.py
@@ -34,5 +34,9 @@ def object_version_path(
     return f"{project_weave_root_url(entity_name, project_name)}/objects/{quote(object_name)}/versions/{obj_version}"
 
 
+def leaderboard_path(entity_name: str, project_name: str, object_name: str) -> str:
+    return f"{project_weave_root_url(entity_name, project_name)}/leaderboards/{quote(object_name)}"
+
+
 def redirect_call(entity_name: str, project_name: str, call_id: str) -> str:
     return f"{remote_project_root_url(entity_name, project_name)}/r/call/{call_id}"

From 1b6f4f4887c2010eb77741580ab543e8569ca6ef Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 18:00:01 -0700
Subject: [PATCH 111/169] Cleanups

---
 .../Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx       | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index 9ff50fd3a3e..ee7238cbb23 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -193,7 +193,10 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
                       if (inner == null) {
                         inner = <NotApplicable />;
                       } else if (typeof inner === 'number') {
-                        if ((0 < inner && inner < 1) || metricPathGroupName.includes('fraction')) {
+                        if (
+                          (0 < inner && inner < 1) ||
+                          metricPathGroupName.includes('fraction')
+                        ) {
                           inner = `${(inner * 100).toFixed(2)}%`;
                         } else {
                           inner = `${inner.toFixed(2)}`;

From c0c281b2aaadfa216490d35d29c070d651dca8ef Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 18:03:10 -0700
Subject: [PATCH 112/169] Small cleanup

---
 demos/leaderboard_demo.ipynb | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/demos/leaderboard_demo.ipynb b/demos/leaderboard_demo.ipynb
index 109ac751e58..2b71ad9e244 100644
--- a/demos/leaderboard_demo.ipynb
+++ b/demos/leaderboard_demo.ipynb
@@ -27,17 +27,6 @@
     "4. Review the leaderboard in the Weave UI."
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import weave\n",
-    "\n",
-    "weave.init(\"leaderboard-demo\")"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -99,6 +88,17 @@
     "    return json.loads(completion.choices[0].message.content)[\"rows\"]"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import weave\n",
+    "\n",
+    "weave.init(\"leaderboard-demo\")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

From 6ddb9d227a1056edb584756ffa2975cda636c021 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 18:06:47 -0700
Subject: [PATCH 113/169] fixed

---
 .../Browse3/pages/LeaderboardPage/LeaderboardPage.tsx  |  2 +-
 .../Browse3/views/Leaderboard/query/hookAdapters.ts    |  8 ++++----
 .../views/Leaderboard/query/leaderboardQuery.ts        | 10 +++++-----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 34e7b1b1e3a..7fb17049925 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -182,7 +182,7 @@ export const LeaderboardPageContentInner: React.FC<
   const {loading, data, evalData} = useSavedLeaderboardData(
     props.entity,
     props.project,
-    workingLeaderboardValCopy
+    workingLeaderboardValCopy.columns
   );
   const [saving, setSaving] = useState(false);
   const discardChanges = useCallback(() => {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
index 426d76020fb..f78dd8c73e3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/hookAdapters.ts
@@ -48,7 +48,7 @@ export const useLeaderboardData = (
 export const useSavedLeaderboardData = (
   entity: string,
   project: string,
-  val: LeaderboardObjectVal
+  columns: LeaderboardObjectVal['columns']
 ): LeaderboardDataState & {evalData: LeaderboardObjectEvalData} => {
   const getTraceServerClient = useGetTraceServerClientContext();
   const [state, setState] = useState<
@@ -58,14 +58,14 @@ export const useSavedLeaderboardData = (
     data: {modelGroups: {}},
     evalData: {},
   });
-  const deepVal = useDeepMemo(val);
+  const deepColumns = useDeepMemo(columns);
   useEffect(() => {
     let mounted = true;
     getPythonLeaderboardData(
       getTraceServerClient(),
       entity,
       project,
-      deepVal
+      deepColumns
     ).then(data => {
       if (mounted) {
         setState({
@@ -78,6 +78,6 @@ export const useSavedLeaderboardData = (
     return () => {
       mounted = false;
     };
-  }, [entity, project, getTraceServerClient, deepVal]);
+  }, [entity, project, getTraceServerClient, deepColumns]);
   return state;
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
index 733ed7cc6b9..c7a3a46a09d 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
@@ -494,7 +494,7 @@ export const getPythonLeaderboardData = async (
   client: TraceServerClient,
   entity: string,
   project: string,
-  val: LeaderboardObjectVal
+  columns: LeaderboardObjectVal['columns']
 ): Promise<{
   finalData: GroupedLeaderboardData;
   evalData: LeaderboardObjectEvalData;
@@ -503,7 +503,7 @@ export const getPythonLeaderboardData = async (
     client,
     entity,
     project,
-    val
+    columns
   );
 
   const finalData: GroupedLeaderboardData = {
@@ -552,13 +552,13 @@ const getLeaderboardObjectGroupableData = async (
   client: TraceServerClient,
   entity: string,
   project: string,
-  val: LeaderboardObjectVal
+  columns: LeaderboardObjectVal['columns']
 ): Promise<{
   groupableData: GroupableLeaderboardValueRecord[];
   evalData: LeaderboardObjectEvalData;
 }> => {
   const evalObjectRefs = _.uniq(
-    val.columns.map(col => col.evaluation_object_ref)
+    columns.map(col => col.evaluation_object_ref)
   ).filter(ref => parseRefMaybe(ref)?.scheme === 'weave');
 
   if (evalObjectRefs.length === 0) {
@@ -591,7 +591,7 @@ const getLeaderboardObjectGroupableData = async (
   const data: GroupableLeaderboardValueRecord[] = [];
   const evalData: LeaderboardObjectEvalData = {};
   allEvaluationCallsRes.calls.forEach(call => {
-    val.columns.forEach(col => {
+    columns.forEach(col => {
       const evalObjRefUri = call.inputs.self;
       if (col.evaluation_object_ref === evalObjRefUri) {
         const evalVal = evalMap[evalObjRefUri];

From 16a4a79769fe7abf9ddf30fe2eb0f6bec9220efe Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 18:26:52 -0700
Subject: [PATCH 114/169] Part 1 of removing old styles

---
 .../LeaderboardConfigEditor.tsx               | 88 +++++++++----------
 1 file changed, 40 insertions(+), 48 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 065b2522aee..7e2000eab24 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -1,21 +1,13 @@
 import {
   Box,
-  Button,
-  Checkbox,
   FormControlLabel,
-  IconButton,
   MenuItem,
   Select,
   TextField,
   Typography,
 } from '@material-ui/core';
-import {
-  Add as AddIcon,
-  ArrowDownward as MoveDownIcon,
-  ArrowUpward as MoveUpIcon,
-  Delete as DeleteIcon,
-  FileCopy as CloneIcon,
-} from '@mui/icons-material';
+import { Button } from '@wandb/weave/components/Button/Button';
+import {Checkbox} from '@wandb/weave/components/Checkbox/Checkbox';
 import {refUri} from '@wandb/weave/react';
 import React, {useEffect, useState} from 'react';
 
@@ -153,7 +145,10 @@ export const LeaderboardConfigEditor: React.FC<{
             totalColumns={leaderboardVal.columns.length}
           />
         ))}
-        <Button startIcon={<AddIcon />} onClick={addColumn}>
+        <Button
+          icon="add-new"
+          variant="ghost"
+          onClick={addColumn}>
           Add Column
         </Button>
       </Box>
@@ -167,21 +162,17 @@ export const LeaderboardConfigEditor: React.FC<{
         alignItems="center"
         justifyContent="flex-end">
         <Button
-          variant="outlined"
+          variant="ghost"
           onClick={discardChanges}
           disabled={saving}
           style={{marginRight: 8}}>
           {isDirty ? 'Discard' : 'Close'}
         </Button>
-        {isDirty && (
           <Button
-            variant="contained"
-            color="primary"
             onClick={commitChanges}
-            disabled={saving}>
-            Save
+            disabled={!isDirty || saving}>
+            {saving ? 'Saving...' : isDirty ? 'Save' : 'Saved'}
           </Button>
-        )}
       </Box>
     </Box>
   );
@@ -224,10 +215,7 @@ const ColumnEditor: React.FC<{
       flexWrap="wrap"
       alignItems="center"
       mb={2}
-      p={2}
-      border={1}
-      borderColor="divider"
-      borderRadius={4}>
+      p={2}>
       <Box flexGrow={1} display="flex" flexWrap="wrap" alignItems="center">
         <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
           <Select
@@ -292,40 +280,45 @@ const ColumnEditor: React.FC<{
             ))}
           </Select>
         </Box>
-        <Box display="flex" alignItems="center" mb={2} mr={2}>
-          <FormControlLabel
-            control={
-              <Checkbox
-                checked={column.should_minimize}
-                onChange={e =>
-                  handleColumnChange(index, 'should_minimize', e.target.checked)
+        <Box display="flex" alignItems="center" mb={2} sx={{gap: 4}}>
+          <Checkbox
+                checked={column.should_minimize ?? false}
+                onCheckedChange={checked =>
+                  handleColumnChange(index, 'should_minimize', !!checked)
                 }
-              />
-            }
-            label="Minimize"
-          />
+            />
+          <span>Minimize</span>
         </Box>
-      </Box>
-      <Box display="flex" justifyContent="flex-end" mb={2}>
-        <IconButton
+        <Box display="flex" justifyContent="flex-end" mb={2}>
+        <Button
           size="small"
+          icon="chevron-up"
+          variant='quiet'
           onClick={() => moveColumn(index, index - 1)}
           disabled={index === 0}>
-          <MoveUpIcon />
-        </IconButton>
-        <IconButton
+        </Button>
+        <Button
           size="small"
+          icon="chevron-down"
+          variant="quiet"
           onClick={() => moveColumn(index, index + 1)}
           disabled={index === totalColumns - 1}>
-          <MoveDownIcon />
-        </IconButton>
-        <IconButton size="small" onClick={() => cloneColumn(index)}>
-          <CloneIcon />
-        </IconButton>
-        <IconButton size="small" onClick={() => removeColumn(index)}>
-          <DeleteIcon />
-        </IconButton>
+        </Button>
+        <Button
+          icon="copy"
+          variant="quiet"
+          size="small"
+          onClick={() => cloneColumn(index)}>
+        </Button>
+        <Button
+          icon="delete"
+          variant="quiet"
+          size="small"
+          onClick={() => removeColumn(index)}>
+        </Button>
+      </Box>
       </Box>
+      
     </Box>
   );
 };
@@ -402,7 +395,6 @@ const useScorers = (
 
     let mounted = true;
     client.readBatch({refs: [evaluationObjectRef]}).then(res => {
-      console.log(res);
       if (mounted) {
         setScorers(
           (res.vals[0].scorers ?? [])

From 9492c0ed49ae200dac7c6cec9da5dde6b8dabf0d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 19:46:16 -0700
Subject: [PATCH 115/169] Part 1 of removing old styles

---
 .../LeaderboardConfigEditor.tsx               | 243 ++++++++----------
 .../SimpleCodeLikeTextArea.tsx                | 220 ++++++++++++++++
 2 files changed, 332 insertions(+), 131 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 7e2000eab24..54938739119 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -1,13 +1,7 @@
-import {
-  Box,
-  FormControlLabel,
-  MenuItem,
-  Select,
-  TextField,
-  Typography,
-} from '@material-ui/core';
-import { Button } from '@wandb/weave/components/Button/Button';
+import {Box, MenuItem, Select} from '@material-ui/core';
+import {Button} from '@wandb/weave/components/Button/Button';
 import {Checkbox} from '@wandb/weave/components/Checkbox/Checkbox';
+import {TextField} from '@wandb/weave/components/Form/TextField';
 import {refUri} from '@wandb/weave/react';
 import React, {useEffect, useState} from 'react';
 
@@ -22,6 +16,7 @@ import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerCli
 import {TraceObjSchema} from '../wfReactInterface/traceServerClientTypes';
 import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
 import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
+import {SimpleCodeLikeTextArea} from './SimpleCodeLikeTextArea';
 
 export const LeaderboardConfigEditor: React.FC<{
   entity: string;
@@ -42,14 +37,12 @@ export const LeaderboardConfigEditor: React.FC<{
   discardChanges,
   commitChanges,
 }) => {
-  const handleNameChange = (event: React.ChangeEvent<HTMLInputElement>) => {
-    setWorkingCopy({...leaderboardVal, name: event.target.value});
+  const handleNameChange = (value: string) => {
+    setWorkingCopy({...leaderboardVal, name: value});
   };
 
-  const handleDescriptionChange = (
-    event: React.ChangeEvent<HTMLInputElement>
-  ) => {
-    setWorkingCopy({...leaderboardVal, description: event.target.value});
+  const handleDescriptionChange = (value: string) => {
+    setWorkingCopy({...leaderboardVal, description: value});
   };
 
   const handleColumnChange = (index: number, field: string, value: any) => {
@@ -104,32 +97,29 @@ export const LeaderboardConfigEditor: React.FC<{
 
   return (
     <Box display="flex" flexDirection="column" height="100%" width="100%">
-      <Box flexGrow={1} overflow="auto" p={2}>
-        <Typography variant="h6" gutterBottom>
-          Display Name
-        </Typography>
+      <Box
+        flexGrow={1}
+        overflow="auto"
+        sx={{
+          mr: 4,
+          ml: 4,
+
+          mb: 2,
+          paddingLeft: 2,
+          paddingRight: 2,
+        }}>
+        <Label>Leaderboard Title</Label>
         <TextField
-          fullWidth
+          icon="layout-grid"
           value={leaderboardVal.name}
           onChange={handleNameChange}
-          margin="normal"
         />
-        <Typography variant="h6" gutterBottom style={{marginTop: 16}}>
-          Description
-        </Typography>
-        <TextField
-          fullWidth
+        <Label>Description</Label>
+        <SimpleCodeLikeTextArea
           value={leaderboardVal.description}
           onChange={handleDescriptionChange}
-          margin="normal"
-          multiline
-          minRows={1}
-          maxRows={10}
-          InputProps={{style: {fontFamily: 'monospace', fontSize: '14px'}}}
         />
-        <Typography variant="h6" gutterBottom style={{marginTop: 16}}>
-          Columns
-        </Typography>
+        <Label>Columns</Label>
         {leaderboardVal.columns.map((column, index) => (
           <ColumnEditor
             key={index}
@@ -145,10 +135,7 @@ export const LeaderboardConfigEditor: React.FC<{
             totalColumns={leaderboardVal.columns.length}
           />
         ))}
-        <Button
-          icon="add-new"
-          variant="ghost"
-          onClick={addColumn}>
+        <Button icon="add-new" variant="ghost" onClick={addColumn}>
           Add Column
         </Button>
       </Box>
@@ -168,16 +155,22 @@ export const LeaderboardConfigEditor: React.FC<{
           style={{marginRight: 8}}>
           {isDirty ? 'Discard' : 'Close'}
         </Button>
-          <Button
-            onClick={commitChanges}
-            disabled={!isDirty || saving}>
-            {saving ? 'Saving...' : isDirty ? 'Save' : 'Saved'}
-          </Button>
+        <Button onClick={commitChanges} disabled={!isDirty || saving}>
+          {saving ? 'Saving...' : isDirty ? 'Save' : 'Saved'}
+        </Button>
       </Box>
     </Box>
   );
 };
 
+const Label: React.FC<{children: React.ReactNode}> = ({children}) => {
+  return (
+    <Box sx={{fontSize: '14px', fontWeight: 'bold', mb: 1, mt: 3}}>
+      {children}
+    </Box>
+  );
+};
+
 const ColumnEditor: React.FC<{
   column: LeaderboardObjectVal['columns'][0];
   index: number;
@@ -210,115 +203,103 @@ const ColumnEditor: React.FC<{
   );
 
   return (
-    <Box
-      display="flex"
-      flexWrap="wrap"
-      alignItems="center"
-      mb={2}
-      p={2}>
-      <Box flexGrow={1} display="flex" flexWrap="wrap" alignItems="center">
-        <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
-          <Select
-            fullWidth
-            value={column.evaluation_object_ref}
-            onChange={e =>
-              handleColumnChange(index, 'evaluation_object_ref', e.target.value)
-            }
-            displayEmpty
-            margin="dense">
-            <MenuItem value="">
-              <em>Select Evaluation Object</em>
+    <Box flexGrow={1} display="flex" flexWrap="wrap" alignItems="center">
+      <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
+        <Select
+          fullWidth
+          value={column.evaluation_object_ref}
+          onChange={e =>
+            handleColumnChange(index, 'evaluation_object_ref', e.target.value)
+          }
+          displayEmpty
+          margin="dense">
+          <MenuItem value="">
+            <em>Select Evaluation Object</em>
+          </MenuItem>
+          {evalObjs.map(obj => (
+            <MenuItem key={obj.ref} value={obj.ref}>
+              {`${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`}
             </MenuItem>
-            {evalObjs.map(obj => (
-              <MenuItem key={obj.ref} value={obj.ref}>
-                {`${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`}
-              </MenuItem>
-            ))}
-          </Select>
-        </Box>
-        <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
-          <Select
-            fullWidth
-            value={column.scorer_name}
-            onChange={e =>
-              handleColumnChange(index, 'scorer_name', e.target.value)
-            }
-            displayEmpty
-            margin="dense"
-            disabled={!column.evaluation_object_ref}>
-            <MenuItem value="">
-              <em>Select Scorer</em>
+          ))}
+        </Select>
+      </Box>
+      <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
+        <Select
+          fullWidth
+          value={column.scorer_name}
+          onChange={e =>
+            handleColumnChange(index, 'scorer_name', e.target.value)
+          }
+          displayEmpty
+          margin="dense"
+          disabled={!column.evaluation_object_ref}>
+          <MenuItem value="">
+            <em>Select Scorer</em>
+          </MenuItem>
+          {scorers.map(scorer => (
+            <MenuItem key={scorer} value={scorer}>
+              {scorer}
             </MenuItem>
-            {scorers.map(scorer => (
-              <MenuItem key={scorer} value={scorer}>
-                {scorer}
-              </MenuItem>
-            ))}
-          </Select>
-        </Box>
-        <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
-          <Select
-            fullWidth
-            value={column.summary_metric_path_parts.join('.')}
-            onChange={e =>
-              handleColumnChange(
-                index,
-                'summary_metric_path_parts',
-                (e.target.value as string).split('.')
-              )
-            }
-            displayEmpty
-            margin="dense"
-            disabled={!column.evaluation_object_ref || !column.scorer_name}>
-            <MenuItem value="">
-              <em>Select Metric Path</em>
+          ))}
+        </Select>
+      </Box>
+      <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
+        <Select
+          fullWidth
+          value={column.summary_metric_path_parts.join('.')}
+          onChange={e =>
+            handleColumnChange(
+              index,
+              'summary_metric_path_parts',
+              (e.target.value as string).split('.')
+            )
+          }
+          displayEmpty
+          margin="dense"
+          disabled={!column.evaluation_object_ref || !column.scorer_name}>
+          <MenuItem value="">
+            <em>Select Metric Path</em>
+          </MenuItem>
+          {metrics.map(path => (
+            <MenuItem key={path} value={path}>
+              {path}
             </MenuItem>
-            {metrics.map(path => (
-              <MenuItem key={path} value={path}>
-                {path}
-              </MenuItem>
-            ))}
-          </Select>
-        </Box>
-        <Box display="flex" alignItems="center" mb={2} sx={{gap: 4}}>
-          <Checkbox
-                checked={column.should_minimize ?? false}
-                onCheckedChange={checked =>
-                  handleColumnChange(index, 'should_minimize', !!checked)
-                }
-            />
-          <span>Minimize</span>
-        </Box>
-        <Box display="flex" justifyContent="flex-end" mb={2}>
+          ))}
+        </Select>
+      </Box>
+      <Box display="flex" alignItems="center" mb={2} sx={{gridGap: 4}}>
+        <Checkbox
+          checked={column.should_minimize ?? false}
+          onCheckedChange={checked =>
+            handleColumnChange(index, 'should_minimize', !!checked)
+          }
+        />
+        <span>Minimize</span>
+      </Box>
+      <Box display="flex" justifyContent="flex-end" mb={2}>
         <Button
           size="small"
           icon="chevron-up"
-          variant='quiet'
+          variant="quiet"
           onClick={() => moveColumn(index, index - 1)}
-          disabled={index === 0}>
-        </Button>
+          disabled={index === 0}></Button>
         <Button
           size="small"
           icon="chevron-down"
           variant="quiet"
           onClick={() => moveColumn(index, index + 1)}
-          disabled={index === totalColumns - 1}>
-        </Button>
+          disabled={index === totalColumns - 1}></Button>
         <Button
           icon="copy"
           variant="quiet"
           size="small"
-          onClick={() => cloneColumn(index)}>
-        </Button>
+          onClick={() => cloneColumn(index)}></Button>
         <Button
           icon="delete"
           variant="quiet"
           size="small"
-          onClick={() => removeColumn(index)}>
-        </Button>
-      </Box>
+          onClick={() => removeColumn(index)}></Button>
       </Box>
-      
     </Box>
   );
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx
new file mode 100644
index 00000000000..8d836a43c10
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx
@@ -0,0 +1,220 @@
+/**
+ * A simple multi-line text editor component that mimics code editor styling.
+ * Features auto-sizing, manual resize handle, and code-like formatting.
+ *
+ * Inspired by: weave-js/src/components/Form/TextField.tsx
+ */
+
+import {Tailwind} from '@wandb/weave/components/Tailwind';
+import classNames from 'classnames';
+import React, {useCallback, useEffect, useRef, useState} from 'react';
+
+export const TextAreaSizes = {
+  Medium: 'medium',
+  Large: 'large',
+} as const;
+export type TextAreaSize = (typeof TextAreaSizes)[keyof typeof TextAreaSizes];
+
+type TextAreaProps = {
+  placeholder?: string;
+  value?: string;
+  onChange?: (value: string) => void;
+  onKeyDown?: (
+    key: string,
+    e: React.KeyboardEvent<HTMLTextAreaElement>
+  ) => void;
+  onBlur?: (value: string) => void;
+  autoFocus?: boolean;
+  disabled?: boolean;
+  ariaLabel?: string;
+  errorState?: boolean;
+  maxLength?: number;
+  maxRows?: number;
+  dataTest?: string;
+};
+
+export const SimpleCodeLikeTextArea = ({
+  placeholder,
+  value,
+  onChange,
+  onKeyDown,
+  onBlur,
+  autoFocus,
+  disabled,
+  ariaLabel,
+  errorState,
+  maxLength,
+  maxRows = 8,
+  dataTest,
+}: TextAreaProps) => {
+  const textareaRef = useRef<HTMLTextAreaElement>(null);
+  const [isManuallyResized, setIsManuallyResized] = useState(false);
+  const isDraggingRef = useRef(false);
+  const initialHeightRef = useRef<number>(0);
+  const initialMouseYRef = useRef<number>(0);
+
+  // Automatically adjust height based on content
+  const adjustHeight = () => {
+    const textarea = textareaRef.current;
+    if (!textarea || isManuallyResized) {
+      return;
+    }
+
+    textarea.style.height = 'auto';
+    const lineHeight = parseInt(
+      getComputedStyle(textarea).lineHeight || '20',
+      10
+    );
+    const maxHeight = lineHeight * maxRows;
+    const newHeight = Math.min(textarea.scrollHeight, maxHeight);
+    textarea.style.height = `${newHeight}px`;
+  };
+
+  useEffect(() => {
+    adjustHeight();
+  }, [value, isManuallyResized]);
+
+  // Handle resize drag start
+  const handleResizeStart = (e: React.MouseEvent) => {
+    e.preventDefault();
+    if (disabled) {
+      return;
+    }
+
+    const textarea = textareaRef.current;
+    if (!textarea) {
+      return;
+    }
+
+    isDraggingRef.current = true;
+    setIsManuallyResized(true);
+    initialHeightRef.current = textarea.offsetHeight;
+    initialMouseYRef.current = e.clientY;
+
+    // Add event listeners for drag and release
+    document.addEventListener('mousemove', handleResizeMove);
+    document.addEventListener('mouseup', handleResizeEnd);
+  };
+
+  // Handle resize drag movement
+  const handleResizeMove = useCallback((e: MouseEvent) => {
+    if (!isDraggingRef.current || !textareaRef.current) {
+      return;
+    }
+
+    const deltaY = e.clientY - initialMouseYRef.current;
+    const newHeight = Math.max(80, initialHeightRef.current + deltaY); // Min height of 80px
+    textareaRef.current.style.height = `${newHeight}px`;
+  }, []);
+
+  // Handle resize drag end
+  const handleResizeEnd = useCallback(() => {
+    isDraggingRef.current = false;
+    document.removeEventListener('mousemove', handleResizeMove);
+    document.removeEventListener('mouseup', handleResizeEnd);
+  }, [handleResizeMove]);
+
+  // Cleanup event listeners
+  useEffect(() => {
+    return () => {
+      document.removeEventListener('mousemove', handleResizeMove);
+      document.removeEventListener('mouseup', handleResizeEnd);
+    };
+  }, [handleResizeEnd, handleResizeMove]);
+
+  // Double click handler to reset to auto-size
+  const handleResizeDoubleClick = () => {
+    setIsManuallyResized(false);
+    adjustHeight();
+  };
+
+  const handleChange = onChange
+    ? (e: React.ChangeEvent<HTMLTextAreaElement>) => {
+        onChange(e.target.value);
+      }
+    : undefined;
+  const handleKeyDown = onKeyDown
+    ? (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
+        onKeyDown(e.key, e);
+      }
+    : undefined;
+  const handleBlur = onBlur
+    ? (e: React.ChangeEvent<HTMLTextAreaElement>) => {
+        onBlur?.(e.target.value);
+      }
+    : undefined;
+
+  return (
+    <Tailwind style={{width: '100%'}}>
+      <div
+        className={classNames(
+          'night-aware',
+          'relative rounded-sm',
+          'bg-white dark:bg-moon-900',
+          'text-moon-800 dark:text-moon-200',
+          'outline outline-1 outline-moon-250 dark:outline-moon-700',
+          {
+            'hover:outline-2 [&:hover:not(:focus-within)]:outline-[#83E4EB] dark:[&:hover:not(:focus-within)]:outline-teal-650':
+              !errorState,
+            'focus-within:outline-2 focus-within:outline-teal-400 dark:focus-within:outline-teal-600':
+              !errorState,
+            'outline-2 outline-red-450 dark:outline-red-550': errorState,
+            'pointer-events-none opacity-50': disabled,
+          }
+        )}>
+        <div className="relative flex w-full items-start rounded-sm">
+          <textarea
+            ref={textareaRef}
+            className={classNames(
+              'w-full flex-1 rounded-sm bg-inherit px-8 py-8',
+              'appearance-none border-none',
+              'focus:outline-none',
+              'placeholder-moon-500',
+              'dark:selection:bg-moon-650 dark:selection:text-moon-200',
+              'resize-none',
+              'overflow-y-auto',
+              'font-mono',
+              'whitespace-pre-wrap',
+              'break-words',
+              'text-sm'
+            )}
+            placeholder={placeholder}
+            value={value}
+            onChange={handleChange}
+            onKeyDown={handleKeyDown}
+            onBlur={handleBlur}
+            autoFocus={autoFocus}
+            disabled={disabled}
+            aria-label={ariaLabel}
+            maxLength={maxLength}
+            data-test={dataTest}
+            rows={1}
+          />
+
+          {/* Resize Handle */}
+          <div
+            className={classNames(
+              'absolute bottom-0 right-0',
+              'h-24 w-24 cursor-se-resize',
+              'flex items-end justify-end',
+              'select-none',
+              {'cursor-default': disabled}
+            )}
+            onMouseDown={handleResizeStart}
+            onDoubleClick={handleResizeDoubleClick}
+            title="Drag to resize. Double-click to reset.">
+            <div
+              className={classNames(
+                'h-0 w-0',
+                'border-l-[10px] border-l-transparent',
+                'border-b-[10px]',
+                'border-b-moon-400 dark:border-b-moon-600',
+                'mb-2 mr-2'
+              )}
+            />
+          </div>
+        </div>
+      </div>
+    </Tailwind>
+  );
+};

From b583d6c97dd6b12e0321a1e1a48ea7ee1614bd51 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 19:47:54 -0700
Subject: [PATCH 116/169] Part 1 of removing old styles

---
 .../Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx
index 8d836a43c10..2330d89306b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx
@@ -72,6 +72,7 @@ export const SimpleCodeLikeTextArea = ({
 
   useEffect(() => {
     adjustHeight();
+  // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [value, isManuallyResized]);
 
   // Handle resize drag start

From f15172e20f5720e8c63d84a4749ab4c6a6f86c07 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 20:09:13 -0700
Subject: [PATCH 117/169] Almost done

---
 .../LeaderboardConfigEditor.tsx               | 97 ++++++++-----------
 .../SimpleCodeLikeTextArea.tsx                |  2 +-
 2 files changed, 44 insertions(+), 55 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 54938739119..c72d29deec3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -1,9 +1,10 @@
-import {Box, MenuItem, Select} from '@material-ui/core';
+import {Box} from '@material-ui/core';
 import {Button} from '@wandb/weave/components/Button/Button';
 import {Checkbox} from '@wandb/weave/components/Checkbox/Checkbox';
+import {Select} from '@wandb/weave/components/Form/Select';
 import {TextField} from '@wandb/weave/components/Form/TextField';
 import {refUri} from '@wandb/weave/react';
-import React, {useEffect, useState} from 'react';
+import React, {useEffect, useMemo, useState} from 'react';
 
 import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
 import {parseRefMaybe} from '../../../Browse2/SmallRef';
@@ -103,7 +104,6 @@ export const LeaderboardConfigEditor: React.FC<{
         sx={{
           mr: 4,
           ml: 4,
-
           mb: 2,
           paddingLeft: 2,
           paddingRight: 2,
@@ -201,71 +201,60 @@ const ColumnEditor: React.FC<{
     column.evaluation_object_ref,
     column.scorer_name
   );
-
+  const selectedEvalObj = evalObjs.find(
+    obj => obj.ref === column.evaluation_object_ref
+  );
+  const selectedScorer = useMemo(
+    () => (column.scorer_name ? {val: column.scorer_name} : undefined),
+    [column.scorer_name]
+  );
+  const selectedMetricPath = useMemo(
+    () => ({val: column.summary_metric_path_parts.join('.')}),
+    [column.summary_metric_path_parts]
+  );
   return (
     <Box flexGrow={1} display="flex" flexWrap="wrap" alignItems="center">
       <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
-        <Select
-          fullWidth
-          value={column.evaluation_object_ref}
-          onChange={e =>
-            handleColumnChange(index, 'evaluation_object_ref', e.target.value)
+        <Select<EvaluationHelperObj>
+          value={selectedEvalObj}
+          placeholder="Evaluation Definition"
+          onChange={newVal =>
+            handleColumnChange(index, 'evaluation_object_ref', newVal?.ref)
           }
-          displayEmpty
-          margin="dense">
-          <MenuItem value="">
-            <em>Select Evaluation Object</em>
-          </MenuItem>
-          {evalObjs.map(obj => (
-            <MenuItem key={obj.ref} value={obj.ref}>
-              {`${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`}
-            </MenuItem>
-          ))}
-        </Select>
+          options={evalObjs}
+          getOptionLabel={obj =>
+            `${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`
+          }
+          getOptionValue={obj => obj.ref}
+        />
       </Box>
       <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
-        <Select
-          fullWidth
-          value={column.scorer_name}
-          onChange={e =>
-            handleColumnChange(index, 'scorer_name', e.target.value)
+        <Select<{val: string}>
+          value={selectedScorer}
+          onChange={newVal =>
+            handleColumnChange(index, 'scorer_name', newVal?.val)
           }
-          displayEmpty
-          margin="dense"
-          disabled={!column.evaluation_object_ref}>
-          <MenuItem value="">
-            <em>Select Scorer</em>
-          </MenuItem>
-          {scorers.map(scorer => (
-            <MenuItem key={scorer} value={scorer}>
-              {scorer}
-            </MenuItem>
-          ))}
-        </Select>
+          options={scorers.map(scorer => ({val: scorer}))}
+          isDisabled={!column.evaluation_object_ref}
+          getOptionLabel={scorer => scorer.val}
+          getOptionValue={scorer => scorer.val}
+        />
       </Box>
       <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
-        <Select
-          fullWidth
-          value={column.summary_metric_path_parts.join('.')}
-          onChange={e =>
+        <Select<{val: string}>
+          value={selectedMetricPath}
+          onChange={newVal =>
             handleColumnChange(
               index,
               'summary_metric_path_parts',
-              (e.target.value as string).split('.')
+              newVal?.val.split('.')
             )
           }
-          displayEmpty
-          margin="dense"
-          disabled={!column.evaluation_object_ref || !column.scorer_name}>
-          <MenuItem value="">
-            <em>Select Metric Path</em>
-          </MenuItem>
-          {metrics.map(path => (
-            <MenuItem key={path} value={path}>
-              {path}
-            </MenuItem>
-          ))}
-        </Select>
+          options={metrics.map(metric => ({val: metric}))}
+          isDisabled={!column.evaluation_object_ref || !column.scorer_name}
+          getOptionLabel={metric => metric.val}
+          getOptionValue={metric => metric.val}
+        />
       </Box>
       <Box display="flex" alignItems="center" mb={2} sx={{gridGap: 4}}>
         <Checkbox
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx
index 2330d89306b..826e0717b1a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/SimpleCodeLikeTextArea.tsx
@@ -72,7 +72,7 @@ export const SimpleCodeLikeTextArea = ({
 
   useEffect(() => {
     adjustHeight();
-  // eslint-disable-next-line react-hooks/exhaustive-deps
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [value, isManuallyResized]);
 
   // Handle resize drag start

From 975f52aadcaaff443317b3c15c831d2d447df546 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 21:00:11 -0700
Subject: [PATCH 118/169] Almost done

---
 .../LeaderboardConfigEditor.tsx               | 225 ++++++++++--------
 1 file changed, 132 insertions(+), 93 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index c72d29deec3..005dc7e5e44 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -1,8 +1,16 @@
 import {Box} from '@material-ui/core';
+import {PopupDropdown} from '@wandb/weave/common/components/PopupDropdown';
 import {Button} from '@wandb/weave/components/Button/Button';
-import {Checkbox} from '@wandb/weave/components/Checkbox/Checkbox';
 import {Select} from '@wandb/weave/components/Form/Select';
 import {TextField} from '@wandb/weave/components/Form/TextField';
+import {
+  IconChevronDown,
+  IconChevronUp,
+  IconCopy,
+  IconDelete,
+  IconSortAscending,
+  IconSortDescending,
+} from '@wandb/weave/components/Icon';
 import {refUri} from '@wandb/weave/react';
 import React, {useEffect, useMemo, useState} from 'react';
 
@@ -120,21 +128,37 @@ export const LeaderboardConfigEditor: React.FC<{
           onChange={handleDescriptionChange}
         />
         <Label>Columns</Label>
-        {leaderboardVal.columns.map((column, index) => (
-          <ColumnEditor
-            key={index}
-            column={column}
-            index={index}
-            evalObjs={evalObjs}
-            entity={entity}
-            project={project}
-            handleColumnChange={handleColumnChange}
-            moveColumn={moveColumn}
-            cloneColumn={cloneColumn}
-            removeColumn={removeColumn}
-            totalColumns={leaderboardVal.columns.length}
-          />
-        ))}
+        <Box
+          sx={{
+            display: 'grid',
+            gridTemplateColumns: '1fr 1fr 1fr auto',
+            gridGap: 3,
+            mb: 2,
+            width: '100%',
+          }}>
+          <Box sx={{color: 'text.secondary', fontSize: '0.875rem'}}>
+            Evaluation
+          </Box>
+          <Box sx={{color: 'text.secondary', fontSize: '0.875rem'}}>Scorer</Box>
+          <Box sx={{color: 'text.secondary', fontSize: '0.875rem'}}>Metric</Box>
+          <Box sx={{color: 'text.secondary', fontSize: '0.875rem'}}></Box>
+          {leaderboardVal.columns.map((column, index) => (
+            <ColumnEditor
+              key={index}
+              column={column}
+              index={index}
+              evalObjs={evalObjs}
+              entity={entity}
+              project={project}
+              handleColumnChange={handleColumnChange}
+              moveColumn={moveColumn}
+              cloneColumn={cloneColumn}
+              removeColumn={removeColumn}
+              totalColumns={leaderboardVal.columns.length}
+            />
+          ))}
+        </Box>
+
         <Button icon="add-new" variant="ghost" onClick={addColumn}>
           Add Column
         </Button>
@@ -212,84 +236,99 @@ const ColumnEditor: React.FC<{
     () => ({val: column.summary_metric_path_parts.join('.')}),
     [column.summary_metric_path_parts]
   );
+  const shouldMinimize = column.should_minimize ?? false;
   return (
-    <Box flexGrow={1} display="flex" flexWrap="wrap" alignItems="center">
-      <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
-        <Select<EvaluationHelperObj>
-          value={selectedEvalObj}
-          placeholder="Evaluation Definition"
-          onChange={newVal =>
-            handleColumnChange(index, 'evaluation_object_ref', newVal?.ref)
-          }
-          options={evalObjs}
-          getOptionLabel={obj =>
-            `${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`
-          }
-          getOptionValue={obj => obj.ref}
-        />
-      </Box>
-      <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
-        <Select<{val: string}>
-          value={selectedScorer}
-          onChange={newVal =>
-            handleColumnChange(index, 'scorer_name', newVal?.val)
-          }
-          options={scorers.map(scorer => ({val: scorer}))}
-          isDisabled={!column.evaluation_object_ref}
-          getOptionLabel={scorer => scorer.val}
-          getOptionValue={scorer => scorer.val}
-        />
-      </Box>
-      <Box flexGrow={1} minWidth={200} mr={2} mb={2}>
-        <Select<{val: string}>
-          value={selectedMetricPath}
-          onChange={newVal =>
-            handleColumnChange(
-              index,
-              'summary_metric_path_parts',
-              newVal?.val.split('.')
-            )
-          }
-          options={metrics.map(metric => ({val: metric}))}
-          isDisabled={!column.evaluation_object_ref || !column.scorer_name}
-          getOptionLabel={metric => metric.val}
-          getOptionValue={metric => metric.val}
-        />
-      </Box>
-      <Box display="flex" alignItems="center" mb={2} sx={{gridGap: 4}}>
-        <Checkbox
-          checked={column.should_minimize ?? false}
-          onCheckedChange={checked =>
-            handleColumnChange(index, 'should_minimize', !!checked)
-          }
-        />
-        <span>Minimize</span>
-      </Box>
-      <Box display="flex" justifyContent="flex-end" mb={2}>
-        <Button
-          size="small"
-          icon="chevron-up"
-          variant="quiet"
-          onClick={() => moveColumn(index, index - 1)}
-          disabled={index === 0}></Button>
-        <Button
-          size="small"
-          icon="chevron-down"
-          variant="quiet"
-          onClick={() => moveColumn(index, index + 1)}
-          disabled={index === totalColumns - 1}></Button>
-        <Button
-          icon="copy"
-          variant="quiet"
-          size="small"
-          onClick={() => cloneColumn(index)}></Button>
-        <Button
-          icon="delete"
-          variant="quiet"
-          size="small"
-          onClick={() => removeColumn(index)}></Button>
-      </Box>
-    </Box>
+    <>
+      <Select<EvaluationHelperObj>
+        value={selectedEvalObj}
+        placeholder="Evaluation Definition"
+        onChange={newVal =>
+          handleColumnChange(index, 'evaluation_object_ref', newVal?.ref)
+        }
+        options={evalObjs}
+        getOptionLabel={obj =>
+          `${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`
+        }
+        getOptionValue={obj => obj.ref}
+      />
+      <Select<{val: string}>
+        value={selectedScorer}
+        onChange={newVal =>
+          handleColumnChange(index, 'scorer_name', newVal?.val)
+        }
+        options={scorers.map(scorer => ({val: scorer}))}
+        isDisabled={!column.evaluation_object_ref}
+        getOptionLabel={scorer => scorer.val}
+        getOptionValue={scorer => scorer.val}
+      />
+      <Select<{val: string}>
+        value={selectedMetricPath}
+        onChange={newVal =>
+          handleColumnChange(
+            index,
+            'summary_metric_path_parts',
+            newVal?.val.split('.')
+          )
+        }
+        options={metrics.map(metric => ({val: metric}))}
+        isDisabled={!column.evaluation_object_ref || !column.scorer_name}
+        getOptionLabel={metric => metric.val}
+        getOptionValue={metric => metric.val}
+      />
+      <PopupDropdown
+        sections={[
+          [
+            {
+              key: 'moveBefore',
+              text: 'Move Before',
+              icon: <IconChevronUp style={{marginRight: '4px'}} />,
+              onClick: () => moveColumn(index, index - 1),
+              disabled: index === 0,
+            },
+            {
+              key: 'moveAfter',
+              text: 'Move After',
+              icon: <IconChevronDown style={{marginRight: '4px'}} />,
+              onClick: () => moveColumn(index, index + 1),
+              disabled: index === totalColumns - 1,
+            },
+            {
+              key: 'duplicate',
+              text: 'Duplicate',
+              icon: <IconCopy style={{marginRight: '4px'}} />,
+              onClick: () => cloneColumn(index),
+            },
+            {
+              key: 'delete',
+              text: 'Delete',
+              icon: <IconDelete style={{marginRight: '4px'}} />,
+              onClick: () => removeColumn(index),
+            },
+            {
+              key: 'changeSortDirection',
+              text: shouldMinimize ? 'Sort Descending' : 'Sort Ascending',
+              icon: shouldMinimize ? (
+                <IconSortDescending style={{marginRight: '4px'}} />
+              ) : (
+                <IconSortAscending style={{marginRight: '4px'}} />
+              ),
+              onClick: () =>
+                handleColumnChange(index, 'should_minimize', !shouldMinimize),
+            },
+          ],
+        ]}
+        trigger={
+          <Button
+            className="row-actions-button"
+            icon="overflow-horizontal"
+            size="medium"
+            variant="ghost"
+            style={{marginLeft: '4px'}}
+          />
+        }
+        offset="-78px, -16px"
+      />
+    </>
   );
 };
 

From 6a4292ad8fca36a2b3a45962c2daf7144927d1b8 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 21:05:42 -0700
Subject: [PATCH 119/169] Fixed state change

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 7fb17049925..4085cb60376 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -173,9 +173,9 @@ export const LeaderboardPageContentInner: React.FC<
     props.project,
     props.leaderboardName
   );
-  const [workingLeaderboardValCopy, setWorkingLeaderboardValCopy] = useState(
-    props.leaderboardVal
-  );
+  const [leaderboardVal, setLeaderboardVal] = useState(props.leaderboardVal);
+  const [workingLeaderboardValCopy, setWorkingLeaderboardValCopy] =
+    useState(leaderboardVal);
   useEffect(() => {
     props.setName(workingLeaderboardValCopy.name);
   }, [props, workingLeaderboardValCopy.name]);
@@ -186,9 +186,9 @@ export const LeaderboardPageContentInner: React.FC<
   );
   const [saving, setSaving] = useState(false);
   const discardChanges = useCallback(() => {
-    setWorkingLeaderboardValCopy(props.leaderboardVal);
+    setWorkingLeaderboardValCopy(leaderboardVal);
     props.setIsEditing(false);
-  }, [props]);
+  }, [leaderboardVal, props]);
   const commitChanges = useCallback(() => {
     const mounted = true;
     setSaving(true);
@@ -196,20 +196,22 @@ export const LeaderboardPageContentInner: React.FC<
       .then(() => {
         if (mounted) {
           props.setIsEditing(false);
+          setLeaderboardVal(workingLeaderboardValCopy);
+          setWorkingLeaderboardValCopy(workingLeaderboardValCopy);
           setSaving(false);
         }
       })
       .catch(e => {
         console.error(e);
         if (mounted) {
-          setWorkingLeaderboardValCopy(props.leaderboardVal);
+          setWorkingLeaderboardValCopy(leaderboardVal);
           setSaving(false);
         }
       });
-  }, [props, updateLeaderboard, workingLeaderboardValCopy]);
+  }, [leaderboardVal, props, updateLeaderboard, workingLeaderboardValCopy]);
   const isDirty = useMemo(() => {
-    return !_.isEqual(props.leaderboardVal, workingLeaderboardValCopy);
-  }, [props.leaderboardVal, workingLeaderboardValCopy]);
+    return !_.isEqual(leaderboardVal, workingLeaderboardValCopy);
+  }, [leaderboardVal, workingLeaderboardValCopy]);
   const columnOrder = useMemo(() => {
     return workingLeaderboardValCopy.columns
       .map(col => {

From e87860f8fbffdacc2109201d77b2a48154801516 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 21:14:13 -0700
Subject: [PATCH 120/169] REVIEW CANDIDATE

---
 .../pages/LeaderboardPage/LeaderboardConfigEditor.tsx      | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 005dc7e5e44..045e60f9d43 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -12,6 +12,7 @@ import {
   IconSortDescending,
 } from '@wandb/weave/components/Icon';
 import {refUri} from '@wandb/weave/react';
+import _ from 'lodash';
 import React, {useEffect, useMemo, useState} from 'react';
 
 import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
@@ -85,7 +86,7 @@ export const LeaderboardConfigEditor: React.FC<{
   };
 
   const removeColumn = (index: number) => {
-    const newColumns = leaderboardVal.columns.filter((_, i) => i !== index);
+    const newColumns = leaderboardVal.columns.filter((v, i) => i !== index);
     setWorkingCopy({...leaderboardVal, columns: newColumns});
   };
 
@@ -245,7 +246,9 @@ const ColumnEditor: React.FC<{
         onChange={newVal =>
           handleColumnChange(index, 'evaluation_object_ref', newVal?.ref)
         }
-        options={evalObjs}
+        options={Object.entries(_.groupBy(evalObjs, 'name')).map(
+          ([name, objs]) => ({options: objs, label: name})
+        )}
         getOptionLabel={obj =>
           `${obj.name}:v${obj.versionIndex} (${obj.digest.slice(0, 6)})`
         }

From 2fc2f39ded6c1dc8d0aabacbb8d080b457249458 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 22:01:44 -0700
Subject: [PATCH 121/169] Moved Docs

---
 .../gen_notebooks/leaderboard_quickstart.md   | 343 ++++++++++++++++++
 .../notebooks/leaderboard_quickstart.ipynb    |   0
 2 files changed, 343 insertions(+)
 create mode 100644 docs/docs/reference/gen_notebooks/leaderboard_quickstart.md
 rename demos/leaderboard_demo.ipynb => docs/notebooks/leaderboard_quickstart.ipynb (100%)

diff --git a/docs/docs/reference/gen_notebooks/leaderboard_quickstart.md b/docs/docs/reference/gen_notebooks/leaderboard_quickstart.md
new file mode 100644
index 00000000000..5ba6c65dcb4
--- /dev/null
+++ b/docs/docs/reference/gen_notebooks/leaderboard_quickstart.md
@@ -0,0 +1,343 @@
+---
+title: Leaderboard Quickstart
+---
+
+
+:::tip[This is a notebook]
+
+<a href="https://colab.research.google.com/github/wandb/weave/blob/master/docs/./notebooks/leaderboard_quickstart.ipynb" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link button button--secondary button--med margin-right--sm notebook-cta-button"><div><img src="https://upload.wikimedia.org/wikipedia/commons/archive/d/d0/20221103151430%21Google_Colaboratory_SVG_Logo.svg" alt="Open In Colab" height="20px" /><div>Open in Colab</div></div></a>
+
+<a href="https://github.com/wandb/weave/blob/master/docs/./notebooks/leaderboard_quickstart.ipynb" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link button button--secondary button--med margin-right--sm notebook-cta-button"><div><img src="https://upload.wikimedia.org/wikipedia/commons/9/91/Octicons-mark-github.svg" alt="View in Github" height="15px" /><div>View in Github</div></div></a>
+
+:::
+
+
+
+<!--- @wandbcode{leaderboard-demo} -->
+
+# Leaderboard Quickstart
+
+In this notebook we will learn to use Weave's Leaderboard to compare model performance across different datasets and scoring functions. Specifically, we will:
+
+1. Generate a dataset of fake zip code data
+2. Author some scoring functions and evaluate a baseline model.
+3. Use these techniques to evaluate a matrix of models vs evaluations.
+4. Review the leaderboard in the Weave UI.
+
+## Step 1: Generate a dataset of fake zip code data
+
+First we will create a function `generate_dataset_rows` that generates a list of fake zip code data.
+
+
+```python
+import json
+
+from openai import OpenAI
+from pydantic import BaseModel
+
+
+class Row(BaseModel):
+    zip_code: str
+    city: str
+    state: str
+    avg_temp_f: float
+    population: int
+    median_income: int
+    known_for: str
+
+
+class Rows(BaseModel):
+    rows: list[Row]
+
+
+def generate_dataset_rows(
+    location: str = "United States", count: int = 5, year: int = 2022
+):
+    client = OpenAI()
+
+    completion = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {
+                "role": "user",
+                "content": f"Please generate {count} rows of data for random zip codes in {location} for the year {year}.",
+            },
+        ],
+        response_format={
+            "type": "json_schema",
+            "json_schema": {
+                "name": "response_format",
+                "schema": Rows.model_json_schema(),
+            },
+        },
+    )
+
+    return json.loads(completion.choices[0].message.content)["rows"]
+```
+
+
+```python
+import weave
+
+weave.init("leaderboard-demo")
+```
+
+## Step 2: Author scoring functions
+
+Next we will author 3 scoring functions:
+
+1. `check_concrete_fields`: Checks if the model output matches the expected city and state.
+2. `check_value_fields`: Checks if the model output is within 10% of the expected population and median income.
+3. `check_subjective_fields`: Uses a LLM to check if the model output matches the expected "known for" field.
+
+
+
+```python
+@weave.op
+def check_concrete_fields(city: str, state: str, output: dict):
+    return {
+        "city_match": city == output["city"],
+        "state_match": state == output["state"],
+    }
+
+
+@weave.op
+def check_value_fields(
+    avg_temp_f: float, population: int, median_income: int, output: dict
+):
+    return {
+        "avg_temp_f_err": abs(avg_temp_f - output["avg_temp_f"]) / avg_temp_f,
+        "population_err": abs(population - output["population"]) / population,
+        "median_income_err": abs(median_income - output["median_income"])
+        / median_income,
+    }
+
+
+@weave.op
+def check_subjective_fields(zip_code: str, known_for: str, output: dict):
+    client = OpenAI()
+
+    class Response(BaseModel):
+        correct_known_for: bool
+
+    completion = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {
+                "role": "user",
+                "content": f"My student was asked what the zip code {zip_code} is best known best for. The right answer is '{known_for}', and they said '{output['known_for']}'. Is their answer correct?",
+            },
+        ],
+        response_format={
+            "type": "json_schema",
+            "json_schema": {
+                "name": "response_format",
+                "schema": Response.model_json_schema(),
+            },
+        },
+    )
+
+    return json.loads(completion.choices[0].message.content)
+```
+
+## Step 3: Create a simple Evaluation
+
+Next we define a simple evaliation using our fake data and scoring functions.
+
+
+
+```python
+rows = generate_dataset_rows()
+evaluation = weave.Evaluation(
+    name="United States - 2022",
+    dataset=rows,
+    scorers=[
+        check_concrete_fields,
+        check_value_fields,
+        check_subjective_fields,
+    ],
+)
+```
+
+## Step 4: Evaluate a baseline model
+
+Now we will evaluate a baseline model which returns a static response.
+
+
+
+```python
+@weave.op
+def baseline_model(zip_code: str):
+    return {
+        "city": "New York",
+        "state": "NY",
+        "avg_temp_f": 50.0,
+        "population": 1000000,
+        "median_income": 100000,
+        "known_for": "The Big Apple",
+    }
+
+
+await evaluation.evaluate(baseline_model)
+```
+
+## Step 5: Create more Models
+
+Now we will create 2 more models to compare against the baseline.
+
+
+```python
+@weave.op
+def gpt_4o_mini_no_context(zip_code: str):
+    client = OpenAI()
+
+    completion = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": f"""Zip code {zip_code}"""}],
+        response_format={
+            "type": "json_schema",
+            "json_schema": {
+                "name": "response_format",
+                "schema": Row.model_json_schema(),
+            },
+        },
+    )
+
+    return json.loads(completion.choices[0].message.content)
+
+
+await evaluation.evaluate(gpt_4o_mini_no_context)
+```
+
+
+```python
+@weave.op
+def gpt_4o_mini_with_context(zip_code: str):
+    client = OpenAI()
+
+    completion = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {
+                "role": "user",
+                "content": f"""Please answer the following questions about the zip code {zip_code}:
+                   1. What is the city?
+                   2. What is the state?
+                   3. What is the average temperature in Fahrenheit?
+                   4. What is the population?
+                   5. What is the median income?
+                   6. What is the most well known thing about this zip code?
+                   """,
+            }
+        ],
+        response_format={
+            "type": "json_schema",
+            "json_schema": {
+                "name": "response_format",
+                "schema": Row.model_json_schema(),
+            },
+        },
+    )
+
+    return json.loads(completion.choices[0].message.content)
+
+
+await evaluation.evaluate(gpt_4o_mini_with_context)
+```
+
+## Step 6: Create more Evaluations
+
+Now we will evaluate a matrix of models vs evaluations.
+
+
+
+```python
+scorers = [
+    check_concrete_fields,
+    check_value_fields,
+    check_subjective_fields,
+]
+evaluations = [
+    weave.Evaluation(
+        name="United States - 2022",
+        dataset=weave.Dataset(
+            name="United States - 2022",
+            rows=generate_dataset_rows("United States", 5, 2022),
+        ),
+        scorers=scorers,
+    ),
+    weave.Evaluation(
+        name="California - 2022",
+        dataset=weave.Dataset(
+            name="California - 2022", rows=generate_dataset_rows("California", 5, 2022)
+        ),
+        scorers=scorers,
+    ),
+    weave.Evaluation(
+        name="United States - 2000",
+        dataset=weave.Dataset(
+            name="United States - 2000",
+            rows=generate_dataset_rows("United States", 5, 2000),
+        ),
+        scorers=scorers,
+    ),
+]
+models = [
+    baseline_model,
+    gpt_4o_mini_no_context,
+    gpt_4o_mini_with_context,
+]
+
+for evaluation in evaluations:
+    for model in models:
+        await evaluation.evaluate(
+            model, __weave={"display_name": evaluation.name + ":" + model.__name__}
+        )
+```
+
+## Step 7: Review the Leaderboard
+
+You can create a new leaderboard by navigating to the leaderboard tab in the UI and clicking "Create Leaderboard".
+
+We can also generate a leaderboard directly from Python:
+
+
+```python
+from weave.flow import leaderboard
+from weave.trace.weave_client import get_ref
+
+spec = leaderboard.Leaderboard(
+    name="Zip Code World Knowledge",
+    description="""
+This leaderboard compares the performance of models in terms of world knowledge about zip codes.
+
+### Columns
+
+1. **State Match against `United States - 2022`**: The fraction of zip codes that the model correctly identified the state for.
+2. **Avg Temp F Error against `California - 2022`**: The mean absolute error of the model's average temperature prediction.
+3. **Correct Known For against `United States - 2000`**: The fraction of zip codes that the model correctly identified the most well known thing about the zip code.
+""",
+    columns=[
+        leaderboard.LeaderboardColumn(
+            evaluation_object_ref=get_ref(evaluations[0]).uri(),
+            scorer_name="check_concrete_fields",
+            summary_metric_path_parts=["state_match", "true_fraction"],
+        ),
+        leaderboard.LeaderboardColumn(
+            evaluation_object_ref=get_ref(evaluations[1]).uri(),
+            scorer_name="check_value_fields",
+            should_minimize=True,
+            summary_metric_path_parts=["avg_temp_f_err", "mean"],
+        ),
+        leaderboard.LeaderboardColumn(
+            evaluation_object_ref=get_ref(evaluations[2]).uri(),
+            scorer_name="check_subjective_fields",
+            summary_metric_path_parts=["correct_known_for", "true_fraction"],
+        ),
+    ],
+)
+
+ref = weave.publish(spec)
+```
diff --git a/demos/leaderboard_demo.ipynb b/docs/notebooks/leaderboard_quickstart.ipynb
similarity index 100%
rename from demos/leaderboard_demo.ipynb
rename to docs/notebooks/leaderboard_quickstart.ipynb

From 7951875028bbe12ef047eb35e6bd090c2844db23 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Mon, 28 Oct 2024 22:04:53 -0700
Subject: [PATCH 122/169] Fixed tests

---
 tests/trace/test_leaderboard.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/trace/test_leaderboard.py b/tests/trace/test_leaderboard.py
index 3ac7ab17f9f..74a8f37f4cd 100644
--- a/tests/trace/test_leaderboard.py
+++ b/tests/trace/test_leaderboard.py
@@ -59,8 +59,8 @@ def test_leaderboard_mis_configured(client):
 
 async def do_evaluations():
     @weave.op
-    def my_scorer(target, model_output):
-        return target == model_output
+    def my_scorer(target, output):
+        return target == output
 
     evaluation_obj_1 = weave.Evaluation(
         name="test_evaluation_name",

From 14fc3fe10372dbd1ed421129935e45d3d6f40dc0 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 30 Oct 2024 11:52:58 -0700
Subject: [PATCH 123/169] init

---
 dev_docs/BaseObjectClasses.md | 98 +++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)
 create mode 100644 dev_docs/BaseObjectClasses.md

diff --git a/dev_docs/BaseObjectClasses.md b/dev_docs/BaseObjectClasses.md
new file mode 100644
index 00000000000..420b3ae15cc
--- /dev/null
+++ b/dev_docs/BaseObjectClasses.md
@@ -0,0 +1,98 @@
+# BaseObjectClasses
+
+## Refresher on Objects and object storage
+
+In Weave, we have a general-purpose data storage system for objects.
+The payloads themselves are completely free-form - basically anything that can be JSON-serialized.
+Users can "publish" runtime objects to weave using `weave.publish`.
+For example:
+
+```python
+config = {"model_name": "my_model", "model_version": "1.0"}
+ref = weave.publish(config, name="my_model_config")
+```
+
+This will create a new object "version" in the collection called "my_model_config".
+These can then be retrieved using `weave.ref().get()`:
+
+```python
+config = weave.ref("my_model_config").get()
+```
+
+Sometimes users are working with standard structured classes like `dataclasses` or `pydantic.BaseModel`. 
+In such cases, we have special serialization and deserialization logic that allows for cleaner serialization patterns.
+For example, let's say the user does:
+
+```python
+class ModelConfig(weave.Object):
+    model_name: str
+    model_version: str
+```
+
+Then the user can publish an instance of `ModelConfig` as follows:
+
+```python
+config = ModelConfig(model_name="my_model", model_version="1.0")
+ref = weave.publish(config)
+```
+
+This will result in an on-disk payload that looks like:
+
+```json
+{
+    "model_name": "my_model",
+    "model_version": "1.0",
+    "_class_name": "ModelConfig",
+    "_bases": ["Object", "BaseModel"]
+}
+```
+
+And additionally, the user can query for all objects of the `ModelConfig` class using the `base_model_classes` filter in `objs_query` or `POST objs/query`.
+Effectively, this is like creating a virtual table for that class.
+
+**Terminology**: We use the term "weave Object" (capital "O") to refer to instances of classes that subclass `weave.Object`.
+
+**Technical note**: the "base_model_class" is the first subtype of "Object", not the _class_name. 
+For example, let's say the class heirarchy is:
+* `A -> Object -> BaseModel`, then the `base_model_class` filter will be "A".
+* `B -> A -> Object -> BaseModel`, then the `base_model_class` filter will still be "A"!
+
+Finally, the Weave library itself utilizes this mechanism for common objects like `Model`, `Dataset`, `Evaluation`, etc...
+This allows the user to subclass these objects to add additional metadata or functionality, while categorizing them in the same virtual table.
+
+## The need for validation
+Objects, like people, sometimes need validation. Many of our Objects (eg. `Model`) don't really need validation as they are completely free-form and up to the user to definie the properties.
+However, there is an increasing need to have a well-defined schema for certain configuration objects - where those objects are tightly defined by Weave, not the user.
+In such cases, it is important to have a standard, validated schema that can be shared across the python sdk, the http api, the DB, the frontend UI, and the typescript sdk.
+
+### Motivating example:
+Let's consider that we are defining a new concept "Leaderboard". We want to store this as a configuration object in Weave.
+Let's say that it's pydantic representation is:
+
+```python
+class LeaderboardColumn(BaseModel):
+    name: str
+    description: str
+    evaluation_obj: str # <- reference to an Evaluation object
+    metric_name: str
+    lower_is_better: bool
+
+class Leaderboard(BaseModel):
+    name: str
+    description: str
+    columns: List[LeaderboardColumn]
+```
+
+We want to be able to store & query `Leaderboards` efficiently and ensure they always have a well-defined schema. Therefore we need:
+1. An ability to validate the schema at insertion time
+2. An ability to create such objects view the REST API
+3. An ability to construct, publish, & query these objects:
+    * via the HTTP API (future: would be nice to have these types inside the openapi schema)
+    * from Python SDK (need a `weave.Object` subclass)
+    * from Typescript SDK (TBD - not yet supported, but need the ability to generate the correct utilities)
+    * from Frontend UI (need a generated Zod schema & hooks to validate the data)
+        * Ideally, with static type-safety.
+
+And all of the above should be generated from a single source of truth: the pydantic schema.
+
+

From 298ffc559a7740836d1b70ee1e4a5dfd1e83dc30 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 30 Oct 2024 14:07:08 -0700
Subject: [PATCH 124/169] init

---
 dev_docs/BaseObjectClasses.md                 |  27 +++-
 tests/trace/test_base_object_classes.py       | 127 ++++++++++++++++++
 weave/trace/base_objects.py                   |   1 +
 weave/trace/weave_client.py                   |   2 +-
 .../base_object_classes/base_object_def.py    |   9 ++
 .../base_object_registry.py                   |   9 ++
 .../base_object_classes/test_only_example.py  |  23 ++++
 7 files changed, 192 insertions(+), 6 deletions(-)
 create mode 100644 tests/trace/test_base_object_classes.py
 create mode 100644 weave/trace/base_objects.py
 create mode 100644 weave/trace_server/interface/base_object_classes/base_object_def.py
 create mode 100644 weave/trace_server/interface/base_object_classes/base_object_registry.py
 create mode 100644 weave/trace_server/interface/base_object_classes/test_only_example.py

diff --git a/dev_docs/BaseObjectClasses.md b/dev_docs/BaseObjectClasses.md
index 420b3ae15cc..d77799d98f1 100644
--- a/dev_docs/BaseObjectClasses.md
+++ b/dev_docs/BaseObjectClasses.md
@@ -47,15 +47,15 @@ This will result in an on-disk payload that looks like:
 }
 ```
 
-And additionally, the user can query for all objects of the `ModelConfig` class using the `base_model_classes` filter in `objs_query` or `POST objs/query`.
+And additionally, the user can query for all objects of the `ModelConfig` class using the `base_object_classes` filter in `objs_query` or `POST objs/query`.
 Effectively, this is like creating a virtual table for that class.
 
 **Terminology**: We use the term "weave Object" (capital "O") to refer to instances of classes that subclass `weave.Object`.
 
-**Technical note**: the "base_model_class" is the first subtype of "Object", not the _class_name. 
+**Technical note**: the "base_object_class" is the first subtype of "Object", not the _class_name. 
 For example, let's say the class heirarchy is:
-* `A -> Object -> BaseModel`, then the `base_model_class` filter will be "A".
-* `B -> A -> Object -> BaseModel`, then the `base_model_class` filter will still be "A"!
+* `A -> Object -> BaseModel`, then the `base_object_class` filter will be "A".
+* `B -> A -> Object -> BaseModel`, then the `base_object_class` filter will still be "A"!
 
 Finally, the Weave library itself utilizes this mechanism for common objects like `Model`, `Dataset`, `Evaluation`, etc...
 This allows the user to subclass these objects to add additional metadata or functionality, while categorizing them in the same virtual table.
@@ -95,4 +95,21 @@ We want to be able to store & query `Leaderboards` efficiently and ensure they a
 
 And all of the above should be generated from a single source of truth: the pydantic schema.
 
-
+### Solution
+1. We define a set of "validated base object classes" in a common location.
+2. We use that:
+    1. At the DB layer to validate insertion schemas
+    2. Can be published from the python sdk
+       * Note: figuring out the interplay with `weave.Object` is tricky here. There are a few issues:
+            1. We don't want to depend on the weave sdk inside the common interface, meaning the class heirarchy is inherently divergent
+            2. Without `weave.Object`:
+                * Our python serialization layer's `is_instance` checks will not maintain behavior
+                * (Seems fine?) No ref tracing or nested deserialization. Actually, i think we do want to avoid this.
+                * (Seems fine?) No "handle_relocatable_object" behavior
+            3. We still want to have the same bases array in the final payload
+            - I think all of this can be resolved by extended the base_object_class to also look for `BaseObject` (objects that are pure data schema: only fields)
+    3. Can be used in the frontend UI layer
+        * via generated Zod types & hooks
+    4. Future: Exposed in the OpenAPI schema
+    5. Future: Generates Typescript SDK types
+    
diff --git a/tests/trace/test_base_object_classes.py b/tests/trace/test_base_object_classes.py
new file mode 100644
index 00000000000..5d6df3600ea
--- /dev/null
+++ b/tests/trace/test_base_object_classes.py
@@ -0,0 +1,127 @@
+import pytest
+
+import weave
+from weave.trace import base_objects
+from weave.trace.refs import ObjectRef
+from weave.trace.weave_client import WeaveClient
+from weave.trace_server import trace_server_interface as tsi
+
+
+def test_pythonic_creation(client: WeaveClient):
+    # First, let's use the high-level pythonic creation API.
+    nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
+    top_obj = base_objects.TestOnlyExample(
+        primitive=1,
+        nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
+        nested_obj=weave.publish(nested_obj).uri(),
+    )
+    ref = weave.publish(top_obj)
+
+    top_obj_gotten = weave.ref(ref.uri()).get()
+
+    assert top_obj_gotten.model_dump() == top_obj.model_dump()
+
+    objs = client.server.obj_query(
+        tsi.ObjQueryReq.model_validate({
+            "project_id": client._project_id(),
+            "filter": {"base_object_classes": ["TestOnlyExample"]}},
+        )
+    )
+
+    assert len(objs) == 1
+    assert objs[0].val == top_obj.model_dump()
+
+
+    objs = client.server.obj_query(
+        tsi.ObjQueryReq.model_validate({
+            "project_id": client._project_id(),
+            "filter": {"base_object_classes": ["TestOnlyNestedBaseObject"]}},
+        )
+    )
+
+    assert len(objs) == 1
+    assert objs[0].val == nested_obj.model_dump()
+
+def test_interface_creation(client):
+    # Now we will do the equivant operation using low-level interface.
+    nested_obj_id = "nested_obj"
+    nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
+    nested_obj_res = client.server.obj_create(
+        tsi.ObjCreateReq.model_validate({
+            "obj": {
+            "project_id": client._project_id(),
+                "object_id": nested_obj_id,
+                "val": nested_obj.model_dump(),
+            }
+        })
+    )
+    nested_obj_ref = ObjectRef(
+        entity=client.entity,
+        project=client.project,
+        name=nested_obj_id,
+        digest=nested_obj_res.digest,
+    )
+
+    top_level_obj_id = "top_obj"
+    top_obj = base_objects.TestOnlyExample(
+        primitive=1,
+        nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
+        nested_obj=nested_obj_ref.uri(),
+    )
+    top_obj_res = client.server.obj_create(
+        tsi.ObjCreateReq.model_validate({
+            "obj": {
+            "project_id": client._project_id(),
+                "object_id": top_level_obj_id,
+                "val": top_obj.model_dump(),
+            }
+        })
+    )
+    top_obj_ref = ObjectRef(
+        entity=client.entity,
+        project=client.project,
+        name=top_level_obj_id,
+        digest=top_obj_res.digest,
+    )
+
+    top_obj_gotten = weave.ref(top_obj_ref.uri()).get()
+
+    assert top_obj_gotten.model_dump() == top_obj.model_dump()
+
+    nested_obj_gotten = weave.ref(nested_obj_ref.uri()).get()
+
+    assert nested_obj_gotten.model_dump() == nested_obj.model_dump()
+
+    objs = client.server.obj_query(
+        tsi.ObjQueryReq.model_validate({
+            "project_id": client._project_id(),
+            "filter": {"base_object_classes": ["TestOnlyExample"]}},
+        )
+    )
+
+    assert len(objs) == 1
+    assert objs[0].val == top_obj.model_dump()
+
+
+    objs = client.server.obj_query(
+        tsi.ObjQueryReq.model_validate({
+            "project_id": client._project_id(),
+            "filter": {"base_object_classes": ["TestOnlyNestedBaseObject"]}},
+        )
+    )
+
+    assert len(objs) == 1
+    assert objs[0].val == nested_obj.model_dump()
+
+def test_schema_validation(client):
+    # Test that we can't create an object with the wrong schema
+    with pytest.raises(weave.errors.WeaveError):
+        client.server.obj_create(
+            tsi.ObjCreateReq.model_validate({
+                "obj": {
+                "project_id": client._project_id(),
+                    "object_id": "nested_obj",
+                    "val": {"a": 2},
+                }
+            })
+        )
diff --git a/weave/trace/base_objects.py b/weave/trace/base_objects.py
new file mode 100644
index 00000000000..5a520e87ee9
--- /dev/null
+++ b/weave/trace/base_objects.py
@@ -0,0 +1 @@
+from weave.trace_server.interface.base_object_classes.base_object_registry import *
diff --git a/weave/trace/weave_client.py b/weave/trace/weave_client.py
index 7f4d1d88ef0..57166f92fd1 100644
--- a/weave/trace/weave_client.py
+++ b/weave/trace/weave_client.py
@@ -1232,7 +1232,7 @@ def _save_nested_objects(self, obj: Any, name: Optional[str] = None) -> Any:
         # Case 1: Object:
         # Here we recurse into each of the properties of the object
         # and save them, and then save the object itself.
-        if isinstance(obj, Object):
+        if isinstance(obj, Object): # TODO: add the generic object here
             obj_rec = pydantic_object_record(obj)
             for v in obj_rec.__dict__.values():
                 self._save_nested_objects(v)
diff --git a/weave/trace_server/interface/base_object_classes/base_object_def.py b/weave/trace_server/interface/base_object_classes/base_object_def.py
new file mode 100644
index 00000000000..b9ab4fa7a8d
--- /dev/null
+++ b/weave/trace_server/interface/base_object_classes/base_object_def.py
@@ -0,0 +1,9 @@
+from typing import Optional
+
+import pydantic
+
+RefStr = str
+
+class BaseObject(pydantic.BaseModel):
+    name: Optional[str] = None
+    description: Optional[str] = None
diff --git a/weave/trace_server/interface/base_object_classes/base_object_registry.py b/weave/trace_server/interface/base_object_classes/base_object_registry.py
new file mode 100644
index 00000000000..bc679229f40
--- /dev/null
+++ b/weave/trace_server/interface/base_object_classes/base_object_registry.py
@@ -0,0 +1,9 @@
+from typing import Dict, Type
+
+from weave.trace_server.interface.base_object_classes.base_object_def import BaseObject
+from weave.trace_server.interface.base_object_classes.test_only_example import *
+
+REGISTRY: Dict[str, Type[BaseObject]] = {
+    "TestOnlyExample": TestOnlyExample,
+    "TestOnlyNestedBaseObject": TestOnlyNestedBaseObject,
+}
diff --git a/weave/trace_server/interface/base_object_classes/test_only_example.py b/weave/trace_server/interface/base_object_classes/test_only_example.py
new file mode 100644
index 00000000000..dd355110bde
--- /dev/null
+++ b/weave/trace_server/interface/base_object_classes/test_only_example.py
@@ -0,0 +1,23 @@
+from pydantic import BaseModel
+
+from weave.trace_server.interface.base_object_classes import base_object_def
+
+
+class TestOnlyNestedBaseModel(BaseModel):
+    a: int
+
+
+class TestOnlyNestedBaseObject(base_object_def.BaseObject):
+    b: int
+
+
+class TestOnlyExample(base_object_def.BaseObject):
+    primitive: int
+    nested_base_model: TestOnlyNestedBaseModel
+    # Important: `RefStr` is just an alias for `str`. When defining `BaseObject`s, we
+    # should never have a property point to another `BaseObject`. This is because each
+    # base object is stored in the database and should be treated like a foreign key.
+    nested_base_object: base_object_def.RefStr
+
+
+__all__ = ["TestOnlyExample", "TestOnlyNestedBaseObject", "TestOnlyNestedBaseModel"]
\ No newline at end of file

From f6be1cedc16981b398a8091dc493a6bc8cd72368 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 30 Oct 2024 14:17:17 -0700
Subject: [PATCH 125/169] init

---
 tests/trace/test_base_object_classes.py       | 151 ++++++++++++++----
 weave/trace/weave_client.py                   |   2 +-
 .../base_object_classes/base_object_def.py    |   1 +
 .../base_object_classes/test_only_example.py  |   2 +-
 4 files changed, 121 insertions(+), 35 deletions(-)

diff --git a/tests/trace/test_base_object_classes.py b/tests/trace/test_base_object_classes.py
index 5d6df3600ea..c0c9ba25b14 100644
--- a/tests/trace/test_base_object_classes.py
+++ b/tests/trace/test_base_object_classes.py
@@ -1,3 +1,17 @@
+"""
+This test file ensures the base_object_classes behavior is as expected. Specifically:
+1. We ensure that pythonic publishing and getting of objects:
+    a. Results in the correct base_object_class filter in the query.
+    b. Produces identical results.
+2. We ensure that using the low-level interface:
+    a. Results in the correct base_object_class filter in the query.
+    b. Produces identical results.
+3. We ensure that digests are equivalent between pythonic and interface style creation.
+   This is important to ensure that UI-based generation of objects is consistent with
+   programmatic generation.
+4. We ensure that invalid schemas are properly rejected from the server.
+"""
+
 import pytest
 
 import weave
@@ -19,41 +33,48 @@ def test_pythonic_creation(client: WeaveClient):
 
     top_obj_gotten = weave.ref(ref.uri()).get()
 
+    assert isinstance(top_obj_gotten, base_objects.TestOnlyExample)
     assert top_obj_gotten.model_dump() == top_obj.model_dump()
 
     objs = client.server.obj_query(
-        tsi.ObjQueryReq.model_validate({
-            "project_id": client._project_id(),
-            "filter": {"base_object_classes": ["TestOnlyExample"]}},
+        tsi.ObjQueryReq.model_validate(
+            {
+                "project_id": client._project_id(),
+                "filter": {"base_object_classes": ["TestOnlyExample"]},
+            },
         )
     )
 
     assert len(objs) == 1
     assert objs[0].val == top_obj.model_dump()
 
-
     objs = client.server.obj_query(
-        tsi.ObjQueryReq.model_validate({
-            "project_id": client._project_id(),
-            "filter": {"base_object_classes": ["TestOnlyNestedBaseObject"]}},
+        tsi.ObjQueryReq.model_validate(
+            {
+                "project_id": client._project_id(),
+                "filter": {"base_object_classes": ["TestOnlyNestedBaseObject"]},
+            },
         )
     )
 
     assert len(objs) == 1
     assert objs[0].val == nested_obj.model_dump()
 
+
 def test_interface_creation(client):
     # Now we will do the equivant operation using low-level interface.
     nested_obj_id = "nested_obj"
     nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
     nested_obj_res = client.server.obj_create(
-        tsi.ObjCreateReq.model_validate({
-            "obj": {
-            "project_id": client._project_id(),
-                "object_id": nested_obj_id,
-                "val": nested_obj.model_dump(),
+        tsi.ObjCreateReq.model_validate(
+            {
+                "obj": {
+                    "project_id": client._project_id(),
+                    "object_id": nested_obj_id,
+                    "val": nested_obj.model_dump(),
+                }
             }
-        })
+        )
     )
     nested_obj_ref = ObjectRef(
         entity=client.entity,
@@ -69,13 +90,15 @@ def test_interface_creation(client):
         nested_obj=nested_obj_ref.uri(),
     )
     top_obj_res = client.server.obj_create(
-        tsi.ObjCreateReq.model_validate({
-            "obj": {
-            "project_id": client._project_id(),
-                "object_id": top_level_obj_id,
-                "val": top_obj.model_dump(),
+        tsi.ObjCreateReq.model_validate(
+            {
+                "obj": {
+                    "project_id": client._project_id(),
+                    "object_id": top_level_obj_id,
+                    "val": top_obj.model_dump(),
+                }
             }
-        })
+        )
     )
     top_obj_ref = ObjectRef(
         entity=client.entity,
@@ -93,35 +116,97 @@ def test_interface_creation(client):
     assert nested_obj_gotten.model_dump() == nested_obj.model_dump()
 
     objs = client.server.obj_query(
-        tsi.ObjQueryReq.model_validate({
-            "project_id": client._project_id(),
-            "filter": {"base_object_classes": ["TestOnlyExample"]}},
+        tsi.ObjQueryReq.model_validate(
+            {
+                "project_id": client._project_id(),
+                "filter": {"base_object_classes": ["TestOnlyExample"]},
+            },
         )
     )
 
     assert len(objs) == 1
     assert objs[0].val == top_obj.model_dump()
 
-
     objs = client.server.obj_query(
-        tsi.ObjQueryReq.model_validate({
-            "project_id": client._project_id(),
-            "filter": {"base_object_classes": ["TestOnlyNestedBaseObject"]}},
+        tsi.ObjQueryReq.model_validate(
+            {
+                "project_id": client._project_id(),
+                "filter": {"base_object_classes": ["TestOnlyNestedBaseObject"]},
+            },
         )
     )
 
     assert len(objs) == 1
     assert objs[0].val == nested_obj.model_dump()
 
+
+def test_digest_equality(client):
+    # Next, let's make sure that the digests are all equivalent
+
+    nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
+    top_obj = base_objects.TestOnlyExample(
+        primitive=1,
+        nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
+        nested_obj=weave.publish(nested_obj).uri(),
+    )
+    ref = weave.publish(top_obj)
+    pythonic_digest = ref.digest
+
+    # Now we will do the equivant operation using low-level interface.
+    nested_obj_id = "nested_obj"
+    nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
+    nested_obj_res = client.server.obj_create(
+        tsi.ObjCreateReq.model_validate(
+            {
+                "obj": {
+                    "project_id": client._project_id(),
+                    "object_id": nested_obj_id,
+                    "val": nested_obj.model_dump(),
+                }
+            }
+        )
+    )
+    nested_obj_ref = ObjectRef(
+        entity=client.entity,
+        project=client.project,
+        name=nested_obj_id,
+        digest=nested_obj_res.digest,
+    )
+
+    top_level_obj_id = "top_obj"
+    top_obj = base_objects.TestOnlyExample(
+        primitive=1,
+        nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
+        nested_obj=nested_obj_ref.uri(),
+    )
+    top_obj_res = client.server.obj_create(
+        tsi.ObjCreateReq.model_validate(
+            {
+                "obj": {
+                    "project_id": client._project_id(),
+                    "object_id": top_level_obj_id,
+                    "val": top_obj.model_dump(),
+                }
+            }
+        )
+    )
+
+    interface_style_digest = top_obj_res.digest
+
+    assert pythonic_digest == interface_style_digest
+
+
 def test_schema_validation(client):
     # Test that we can't create an object with the wrong schema
-    with pytest.raises(weave.errors.WeaveError):
+    with pytest.raises():
         client.server.obj_create(
-            tsi.ObjCreateReq.model_validate({
-                "obj": {
-                "project_id": client._project_id(),
-                    "object_id": "nested_obj",
-                    "val": {"a": 2},
+            tsi.ObjCreateReq.model_validate(
+                {
+                    "obj": {
+                        "project_id": client._project_id(),
+                        "object_id": "nested_obj",
+                        "val": {"a": 2},
+                    }
                 }
-            })
+            )
         )
diff --git a/weave/trace/weave_client.py b/weave/trace/weave_client.py
index 57166f92fd1..656bf1c360f 100644
--- a/weave/trace/weave_client.py
+++ b/weave/trace/weave_client.py
@@ -1232,7 +1232,7 @@ def _save_nested_objects(self, obj: Any, name: Optional[str] = None) -> Any:
         # Case 1: Object:
         # Here we recurse into each of the properties of the object
         # and save them, and then save the object itself.
-        if isinstance(obj, Object): # TODO: add the generic object here
+        if isinstance(obj, Object):  # TODO: add the generic object here
             obj_rec = pydantic_object_record(obj)
             for v in obj_rec.__dict__.values():
                 self._save_nested_objects(v)
diff --git a/weave/trace_server/interface/base_object_classes/base_object_def.py b/weave/trace_server/interface/base_object_classes/base_object_def.py
index b9ab4fa7a8d..746aae1df5f 100644
--- a/weave/trace_server/interface/base_object_classes/base_object_def.py
+++ b/weave/trace_server/interface/base_object_classes/base_object_def.py
@@ -4,6 +4,7 @@
 
 RefStr = str
 
+
 class BaseObject(pydantic.BaseModel):
     name: Optional[str] = None
     description: Optional[str] = None
diff --git a/weave/trace_server/interface/base_object_classes/test_only_example.py b/weave/trace_server/interface/base_object_classes/test_only_example.py
index dd355110bde..b93af28790e 100644
--- a/weave/trace_server/interface/base_object_classes/test_only_example.py
+++ b/weave/trace_server/interface/base_object_classes/test_only_example.py
@@ -20,4 +20,4 @@ class TestOnlyExample(base_object_def.BaseObject):
     nested_base_object: base_object_def.RefStr
 
 
-__all__ = ["TestOnlyExample", "TestOnlyNestedBaseObject", "TestOnlyNestedBaseModel"]
\ No newline at end of file
+__all__ = ["TestOnlyExample", "TestOnlyNestedBaseObject", "TestOnlyNestedBaseModel"]

From 29da8cb7e0d431d68fbecea4ac7ca1a79198cb70 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 30 Oct 2024 15:43:59 -0700
Subject: [PATCH 126/169] generation complete

---
 Makefile                                      |   6 +-
 weave-js/package.json                         |   8 +-
 .../generatedBaseObjectClasses.zod.ts         |  29 ++
 weave-js/yarn.lock                            | 340 +++++++++++++++++-
 weave/Makefile                                |   3 +
 weave/scripts/generate_base_object_schemas.py |  28 ++
 .../generated_base_object_class_schemas.json  | 114 ++++++
 .../base_object_registry.py                   |   7 +
 8 files changed, 525 insertions(+), 10 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
 create mode 100644 weave/scripts/generate_base_object_schemas.py
 create mode 100644 weave/scripts/generated_base_object_class_schemas.json

diff --git a/Makefile b/Makefile
index a53cde0a026..d152017c58b 100644
--- a/Makefile
+++ b/Makefile
@@ -14,4 +14,8 @@ docs:
 build:
 	uv build
 
-prepare-release: docs build
\ No newline at end of file
+prepare-release: docs build
+
+synchronize-base-object-schemas:
+	cd weave && make generate_base_object_schemas && \
+	cd ../weave-js && yarn generate-schemas
\ No newline at end of file
diff --git a/weave-js/package.json b/weave-js/package.json
index d925f9d0a42..5897561a6c1 100644
--- a/weave-js/package.json
+++ b/weave-js/package.json
@@ -20,7 +20,8 @@
     "prettier": "prettier --config .prettierrc --check \"src/**/*.ts\" \"src/**/*.tsx\"",
     "prettier-fix": "prettier --loglevel warn --config .prettierrc --write \"src/**/*.ts\" \"src/**/*.tsx\"",
     "lint": "yarn eslint & yarn tslint & yarn prettier & wait",
-    "lint-fix": "yarn eslint-fix & yarn tslint-fix & yarn prettier-fix & wait"
+    "lint-fix": "yarn eslint-fix & yarn tslint-fix & yarn prettier-fix & wait",
+    "generate-schemas": "yarn quicktype -s schema  ../weave/scripts/generated_base_object_class_schemas.json -o ./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts --lang typescript-zod"
   },
   "dependencies": {
     "@apollo/client": "^3.8.4",
@@ -145,7 +146,8 @@
     "wavesurfer.js": "^2.0.0",
     "web-tree-sitter": "^0.20.5",
     "yet-another-react-lightbox": "^3.17.5",
-    "zen-observable": "^0.10.0"
+    "zen-observable": "^0.10.0",
+    "zod": "^3.23.8"
   },
   "devDependencies": {
     "@babel/core": "^7.23.2",
@@ -217,11 +219,13 @@
     "identity-obj-proxy": "^3.0.0",
     "jsdom": "^22.1.0",
     "json-schema-to-typescript": "^11.0.2",
+    "json-schema-to-zod": "^2.4.1",
     "less": "^2.7.3",
     "lodash.defaults": "^4.2.0",
     "nodemon": "^2.0.22",
     "prettier": "^2.8.7",
     "prettier-plugin-tailwindcss": "^0.2.1",
+    "quicktype": "^23.0.170",
     "rimraf": "^3.0.2",
     "rollup-plugin-visualizer": "^5.5.2",
     "tailwindcss": "^3.3.2",
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
new file mode 100644
index 00000000000..a01bf4e68ee
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
@@ -0,0 +1,29 @@
+import * as z from "zod";
+
+
+export const TestOnlyNestedBaseModelSchema = z.object({
+    "a": z.number(),
+});
+export type TestOnlyNestedBaseModel = z.infer<typeof TestOnlyNestedBaseModelSchema>;
+
+export const TestOnlyNestedBaseObjectSchema = z.object({
+    "b": z.number(),
+    "description": z.union([z.null(), z.string()]).optional(),
+    "name": z.union([z.null(), z.string()]).optional(),
+});
+export type TestOnlyNestedBaseObject = z.infer<typeof TestOnlyNestedBaseObjectSchema>;
+
+export const TestOnlyExampleSchema = z.object({
+    "description": z.union([z.null(), z.string()]).optional(),
+    "name": z.union([z.null(), z.string()]).optional(),
+    "nested_base_model": TestOnlyNestedBaseModelSchema,
+    "nested_base_object": z.string(),
+    "primitive": z.number(),
+});
+export type TestOnlyExample = z.infer<typeof TestOnlyExampleSchema>;
+
+export const GeneratedBaseObjectClassesZodSchema = z.object({
+    "TestOnlyExample": TestOnlyExampleSchema,
+    "TestOnlyNestedBaseObject": TestOnlyNestedBaseObjectSchema,
+});
+export type GeneratedBaseObjectClassesZod = z.infer<typeof GeneratedBaseObjectClassesZodSchema>;
diff --git a/weave-js/yarn.lock b/weave-js/yarn.lock
index 2ee20553257..8541ca9a5d7 100644
--- a/weave-js/yarn.lock
+++ b/weave-js/yarn.lock
@@ -2119,6 +2119,16 @@
   resolved "https://registry.yarnpkg.com/@floating-ui/utils/-/utils-0.2.5.tgz#105c37d9d9620ce69b7f692a20c821bf1ad2cbf9"
   integrity sha512-sTcG+QZ6fdEUObICavU+aB3Mp8HY4n14wYHdxK4fXjPmv3PXZZeY5RaguJmGyeH/CJQhX3fqKUtS4qc1LoHwhQ==
 
+"@glideapps/ts-necessities@2.2.3":
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/@glideapps/ts-necessities/-/ts-necessities-2.2.3.tgz#62e25b3a1ace8b8c3f47e55e66d101a0a854eb23"
+  integrity sha512-gXi0awOZLHk3TbW55GZLCPP6O+y/b5X1pBXKBVckFONSwF1z1E5ND2BGJsghQFah+pW7pkkyFb2VhUQI2qhL5w==
+
+"@glideapps/ts-necessities@^2.2.3":
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/@glideapps/ts-necessities/-/ts-necessities-2.3.2.tgz#3e7a07f41c8c07527757631f25599a7b67d39d8c"
+  integrity sha512-tOXo3SrEeLu+4X2q6O2iNPXdGI1qoXEz/KrbkElTsWiWb69tFH4GzWz2K++0nBD6O3qO2Ft1C4L4ZvUfE2QDlQ==
+
 "@graphql-codegen/add@^5.0.0":
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/@graphql-codegen/add/-/add-5.0.0.tgz#578ebaf4fa87c1e934c381cd679bcedcf79feaba"
@@ -2774,6 +2784,20 @@
   resolved "https://registry.yarnpkg.com/@mapbox/whoots-js/-/whoots-js-3.1.0.tgz#497c67a1cef50d1a2459ba60f315e448d2ad87fe"
   integrity sha512-Es6WcD0nO5l+2BOQS4uLfNPYQaNDfbot3X1XUoloz+x0mPDS3eeORZJl06HXjwBG1fOGwCRnzK88LMdxKRrd6Q==
 
+"@mark.probst/typescript-json-schema@0.55.0":
+  version "0.55.0"
+  resolved "https://registry.yarnpkg.com/@mark.probst/typescript-json-schema/-/typescript-json-schema-0.55.0.tgz#a82c0cb8b3c9ba1a14faf2ea3fa95f26c1a6a57d"
+  integrity sha512-jI48mSnRgFQxXiE/UTUCVCpX8lK3wCFKLF1Ss2aEreboKNuLQGt3e0/YFqWVHe/WENxOaqiJvwOz+L/SrN2+qQ==
+  dependencies:
+    "@types/json-schema" "^7.0.9"
+    "@types/node" "^16.9.2"
+    glob "^7.1.7"
+    path-equal "^1.1.2"
+    safe-stable-stringify "^2.2.0"
+    ts-node "^10.9.1"
+    typescript "4.9.4"
+    yargs "^17.1.1"
+
 "@material-ui/core@^4.12.4":
   version "4.12.4"
   resolved "https://registry.yarnpkg.com/@material-ui/core/-/core-4.12.4.tgz#4ac17488e8fcaf55eb6a7f5efb2a131e10138a73"
@@ -4398,6 +4422,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.53.tgz#42855629b8773535ab868238718745bf56c56219"
   integrity sha512-soGmOpVBUq+gaBMwom1M+krC/NNbWlosh4AtGA03SyWNDiqSKtwp7OulO1M6+mg8YkHMvJ/y0AkCeO8d1hNb7A==
 
+"@types/node@^16.9.2":
+  version "16.18.116"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.116.tgz#138a0ce907c9f308d43f89902a9ef79fbcbc5e5c"
+  integrity sha512-mLigUvhoaADRewggiby+XfAAFOUOMCm/SwL5DAJ+CMUGjSLIGMsJVN7BOKftuQSHGjUmS/W7hVht8fcNbi/MRA==
+
 "@types/node@^18.15.11":
   version "18.16.19"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.19.tgz#cb03fca8910fdeb7595b755126a8a78144714eea"
@@ -4997,6 +5026,13 @@ abbrev@1:
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
   integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==
 
+abort-controller@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
+  integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==
+  dependencies:
+    event-target-shim "^5.0.0"
+
 abs-svg-path@^0.1.1, abs-svg-path@~0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/abs-svg-path/-/abs-svg-path-0.1.1.tgz#df601c8e8d2ba10d4a76d625e236a9a39c2723bf"
@@ -5195,6 +5231,16 @@ aria-query@^5.0.0, aria-query@^5.1.3:
   dependencies:
     dequal "^2.0.3"
 
+array-back@^3.0.1, array-back@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/array-back/-/array-back-3.1.0.tgz#b8859d7a508871c9a7b2cf42f99428f65e96bfb0"
+  integrity sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==
+
+array-back@^6.2.2:
+  version "6.2.2"
+  resolved "https://registry.yarnpkg.com/array-back/-/array-back-6.2.2.tgz#f567d99e9af88a6d3d2f9dfcc21db6f9ba9fd157"
+  integrity sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==
+
 array-bounds@^1.0.0, array-bounds@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/array-bounds/-/array-bounds-1.0.1.tgz#da11356b4e18e075a4f0c86e1f179a67b7d7ea31"
@@ -5562,7 +5608,7 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
   integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
 
-base64-js@^1.3.1:
+base64-js@^1.3.0, base64-js@^1.3.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
@@ -5645,6 +5691,11 @@ braces@^3.0.2, braces@~3.0.2:
   dependencies:
     fill-range "^7.0.1"
 
+browser-or-node@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/browser-or-node/-/browser-or-node-3.0.0.tgz#2b11335570b28887e0bf5cd857f2e8062c6ae293"
+  integrity sha512-iczIdVJzGEYhP5DqQxYM9Hh7Ztpqqi+CXZpSmX8ALFs9ecXkQIeqRyM6TfxEfMVpwhl3dSuDvxdzzo9sUOIVBQ==
+
 browserify-package-json@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/browserify-package-json/-/browserify-package-json-1.0.1.tgz#98dde8aa5c561fd6d3fe49bbaa102b74b396fdea"
@@ -5687,6 +5738,14 @@ buffer@^5.5.0:
     base64-js "^1.3.1"
     ieee754 "^1.1.13"
 
+buffer@^6.0.3:
+  version "6.0.3"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6"
+  integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==
+  dependencies:
+    base64-js "^1.3.1"
+    ieee754 "^1.2.1"
+
 builtin-modules@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f"
@@ -5817,6 +5876,13 @@ chai@^4.3.10:
     pathval "^1.1.1"
     type-detect "^4.0.8"
 
+chalk-template@^0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/chalk-template/-/chalk-template-0.4.0.tgz#692c034d0ed62436b9062c1707fadcd0f753204b"
+  integrity sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==
+  dependencies:
+    chalk "^4.1.2"
+
 chalk@^2.0.0, chalk@^2.3.0, chalk@^2.4.2:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
@@ -5834,7 +5900,7 @@ chalk@^3.0.0:
     ansi-styles "^4.1.0"
     supports-color "^7.1.0"
 
-chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.1:
+chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.1, chalk@^4.1.2:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01"
   integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==
@@ -6047,6 +6113,11 @@ collapse-white-space@^1.0.2:
   resolved "https://registry.yarnpkg.com/collapse-white-space/-/collapse-white-space-1.0.6.tgz#e63629c0016665792060dbbeb79c42239d2c5287"
   integrity sha512-jEovNnrhMuqyCcjfEJA56v0Xq8SkIoPKDyaHahwo3POf4qcSXqMYuwNcOTzp74vTsR9Tn08z4MxWqAhcekogkQ==
 
+collection-utils@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/collection-utils/-/collection-utils-1.0.1.tgz#31d14336488674f27aefc0a7c5eccacf6df78044"
+  integrity sha512-LA2YTIlR7biSpXkKYwwuzGjwL5rjWEZVOSnvdUc7gObvWe4WkjxOpfrdhoP7Hs09YWDVfg0Mal9BpAqLfVEzQg==
+
 color-alpha@1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/color-alpha/-/color-alpha-1.0.4.tgz#c141dc926e95fc3db647d0e14e5bc3651c29e040"
@@ -6198,6 +6269,26 @@ comma-separated-tokens@^2.0.0:
   resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee"
   integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==
 
+command-line-args@^5.2.1:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/command-line-args/-/command-line-args-5.2.1.tgz#c44c32e437a57d7c51157696893c5909e9cec42e"
+  integrity sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==
+  dependencies:
+    array-back "^3.1.0"
+    find-replace "^3.0.0"
+    lodash.camelcase "^4.3.0"
+    typical "^4.0.0"
+
+command-line-usage@^7.0.1:
+  version "7.0.3"
+  resolved "https://registry.yarnpkg.com/command-line-usage/-/command-line-usage-7.0.3.tgz#6bce992354f6af10ecea2b631bfdf0c8b3bfaea3"
+  integrity sha512-PqMLy5+YGwhMh1wS04mVG44oqDsgyLRSKJBdOo1bnYhMKBW65gZF1dRp2OZRhiTjgUHljy99qkO7bsctLaw35Q==
+  dependencies:
+    array-back "^6.2.2"
+    chalk-template "^0.4.0"
+    table-layout "^4.1.0"
+    typical "^7.1.1"
+
 commander@2, commander@^2.12.1, commander@^2.15.1, commander@^2.19.0:
   version "2.20.3"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
@@ -6437,6 +6528,13 @@ cross-fetch@^3.1.5:
   dependencies:
     node-fetch "^2.6.12"
 
+cross-fetch@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-4.0.0.tgz#f037aef1580bb3a1a35164ea2a848ba81b445983"
+  integrity sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==
+  dependencies:
+    node-fetch "^2.6.12"
+
 cross-spawn@^7.0.1, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@@ -7797,7 +7895,12 @@ event-emitter@^0.3.5:
     d "1"
     es5-ext "~0.10.14"
 
-events@^3.2.0:
+event-target-shim@^5.0.0:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
+  integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==
+
+events@^3.2.0, events@^3.3.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400"
   integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==
@@ -8005,6 +8108,13 @@ filter-obj@^5.1.0:
   resolved "https://registry.yarnpkg.com/filter-obj/-/filter-obj-5.1.0.tgz#5bd89676000a713d7db2e197f660274428e524ed"
   integrity sha512-qWeTREPoT7I0bifpPUXtxkZJ1XJzxWtfoWWkdVGqa+eCr3SHW/Ocp89o8vLvbUuQnadybJpjOKu4V+RwO6sGng==
 
+find-replace@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/find-replace/-/find-replace-3.0.0.tgz#3e7e23d3b05167a76f770c9fbd5258b0def68c38"
+  integrity sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==
+  dependencies:
+    array-back "^3.0.1"
+
 find-root@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/find-root/-/find-root-1.1.0.tgz#abcfc8ba76f708c42a97b3d685b7e9450bfb9ce4"
@@ -8349,7 +8459,7 @@ glob@7.1.6:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^7.1.1, glob@^7.1.3, glob@^7.1.6:
+glob@^7.1.1, glob@^7.1.3, glob@^7.1.6, glob@^7.1.7:
   version "7.2.3"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"
   integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==
@@ -8585,6 +8695,13 @@ graphql@=16.6.0:
   resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.6.0.tgz#c2dcffa4649db149f6282af726c8c83f1c7c5fdb"
   integrity sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw==
 
+graphql@^0.11.7:
+  version "0.11.7"
+  resolved "https://registry.yarnpkg.com/graphql/-/graphql-0.11.7.tgz#e5abaa9cb7b7cccb84e9f0836bf4370d268750c6"
+  integrity sha512-x7uDjyz8Jx+QPbpCFCMQ8lltnQa4p4vSYHx6ADe8rVYRTdsyhCJbvSty5DAsLVmU6cGakl+r8HQYolKHxk/tiw==
+  dependencies:
+    iterall "1.1.3"
+
 grid-index@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/grid-index/-/grid-index-1.1.0.tgz#97f8221edec1026c8377b86446a7c71e79522ea7"
@@ -9026,7 +9143,7 @@ identity-obj-proxy@^3.0.0:
   dependencies:
     harmony-reflect "^1.4.6"
 
-ieee754@^1.1.12, ieee754@^1.1.13:
+ieee754@^1.1.12, ieee754@^1.1.13, ieee754@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
   integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
@@ -9552,6 +9669,11 @@ isstream@~0.1.2:
   resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
   integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==
 
+iterall@1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.1.3.tgz#1cbbff96204056dde6656e2ed2e2226d0e6d72c9"
+  integrity sha512-Cu/kb+4HiNSejAPhSaN1VukdNTTi/r4/e+yykqjlG/IW+1gZH5b4+Bq3whDX4tvbYugta3r8KTMUiqT3fIGxuQ==
+
 jest-diff@^29.6.2:
   version "29.6.2"
   resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.6.2.tgz#c36001e5543e82a0805051d3ceac32e6825c1c46"
@@ -9636,6 +9758,11 @@ jose@^4.11.4:
   resolved "https://registry.yarnpkg.com/jose/-/jose-4.14.6.tgz#94dca1d04a0ad8c6bff0998cdb51220d473cc3af"
   integrity sha512-EqJPEUlZD0/CSUMubKtMaYUOtWe91tZXTWMJZoKSbLk+KtdhNdcvppH8lA9XwVu2V4Ailvsj0GBZJ2ZwDjfesQ==
 
+js-base64@^3.7.7:
+  version "3.7.7"
+  resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-3.7.7.tgz#e51b84bf78fbf5702b9541e2cb7bfcb893b43e79"
+  integrity sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw==
+
 js-levenshtein@^1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/js-levenshtein/-/js-levenshtein-1.1.6.tgz#c6cee58eb3550372df8deb85fad5ce66ce01d59d"
@@ -10038,6 +10165,11 @@ locate-path@^6.0.0:
   dependencies:
     p-locate "^5.0.0"
 
+lodash.camelcase@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6"
+  integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==
+
 lodash.debounce@^4.0.8:
   version "4.0.8"
   resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af"
@@ -10850,6 +10982,11 @@ moment@^2.29.3:
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108"
   integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==
 
+moment@^2.30.1:
+  version "2.30.1"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.30.1.tgz#f8c91c07b7a786e30c59926df530b4eac96974ae"
+  integrity sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==
+
 monaco-editor@^0.29.1:
   version "0.29.1"
   resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.29.1.tgz#6ee93d8a5320704d48fd7058204deed72429c020"
@@ -11382,6 +11519,16 @@ package-json-versionify@^1.0.4:
   dependencies:
     browserify-package-json "^1.0.0"
 
+pako@^0.2.5:
+  version "0.2.9"
+  resolved "https://registry.yarnpkg.com/pako/-/pako-0.2.9.tgz#f3f7522f4ef782348da8161bad9ecfd51bf83a75"
+  integrity sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==
+
+pako@^1.0.6:
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
+  integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
+
 pako@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86"
@@ -11500,6 +11647,11 @@ path-case@^3.0.4:
     dot-case "^3.0.4"
     tslib "^2.0.3"
 
+path-equal@^1.1.2:
+  version "1.2.5"
+  resolved "https://registry.yarnpkg.com/path-equal/-/path-equal-1.2.5.tgz#9fcbdd5e5daee448e96f43f3bac06c666b5e982a"
+  integrity sha512-i73IctDr3F2W+bsOWDyyVm/lqsXO47aY9nsFZUjTT/aljSbkxHxxCoyZ9UUrM8jK0JVod+An+rl48RCsvWM+9g==
+
 path-exists@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3"
@@ -11695,6 +11847,11 @@ plotly.js@^2.23.2:
     webgl-context "^2.2.0"
     world-calendars "^1.0.3"
 
+pluralize@^8.0.0:
+  version "8.0.0"
+  resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"
+  integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==
+
 point-in-polygon@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/point-in-polygon/-/point-in-polygon-1.1.0.tgz#b0af2616c01bdee341cbf2894df643387ca03357"
@@ -11859,6 +12016,11 @@ process-nextick-args@~2.0.0:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
   integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
 
+process@^0.11.10:
+  version "0.11.10"
+  resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
+  integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==
+
 promise@^7.1.1:
   version "7.3.1"
   resolved "https://registry.yarnpkg.com/promise/-/promise-7.3.1.tgz#064b72602b18f90f29192b8b1bc418ffd1ebd3bf"
@@ -11965,6 +12127,66 @@ quickselect@^2.0.0:
   resolved "https://registry.yarnpkg.com/quickselect/-/quickselect-2.0.0.tgz#f19680a486a5eefb581303e023e98faaf25dd018"
   integrity sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw==
 
+quicktype-core@23.0.170:
+  version "23.0.170"
+  resolved "https://registry.yarnpkg.com/quicktype-core/-/quicktype-core-23.0.170.tgz#ecaab8091552980883dd587ebe7c91abed74866c"
+  integrity sha512-ZsjveG0yJUIijUx4yQshzyQ5EAXKbFSBTQJHnJ+KoSZVxcS+m3GcmDpzrdUIRYMhgLaF11ZGvLSYi5U0xcwemw==
+  dependencies:
+    "@glideapps/ts-necessities" "2.2.3"
+    browser-or-node "^3.0.0"
+    collection-utils "^1.0.1"
+    cross-fetch "^4.0.0"
+    is-url "^1.2.4"
+    js-base64 "^3.7.7"
+    lodash "^4.17.21"
+    pako "^1.0.6"
+    pluralize "^8.0.0"
+    readable-stream "4.5.2"
+    unicode-properties "^1.4.1"
+    urijs "^1.19.1"
+    wordwrap "^1.0.0"
+    yaml "^2.4.1"
+
+quicktype-graphql-input@23.0.170:
+  version "23.0.170"
+  resolved "https://registry.yarnpkg.com/quicktype-graphql-input/-/quicktype-graphql-input-23.0.170.tgz#f52bb9204a1b434b4e5f0a9003da227d50bcd9da"
+  integrity sha512-L0xPKdIFZFChwups9oqJuQw/vwEbRVKBvU9L5jAs0Z/aLyfdsuxDpKGMJXnNWa2yE7NhPX/UDX8ytxn8uc8hdQ==
+  dependencies:
+    collection-utils "^1.0.1"
+    graphql "^0.11.7"
+    quicktype-core "23.0.170"
+
+quicktype-typescript-input@23.0.170:
+  version "23.0.170"
+  resolved "https://registry.yarnpkg.com/quicktype-typescript-input/-/quicktype-typescript-input-23.0.170.tgz#13efb2f8a7846a0f685fab2852086995f8c712b2"
+  integrity sha512-lckhc//Mc95f/puRFKv4BFs7VpUUJXhw/psh+5ZAMiErxOWgoF87XthGusmaqoXNzjmEy1AVwGgMCG2pp/tJ/w==
+  dependencies:
+    "@mark.probst/typescript-json-schema" "0.55.0"
+    quicktype-core "23.0.170"
+    typescript "4.9.5"
+
+quicktype@^23.0.170:
+  version "23.0.170"
+  resolved "https://registry.yarnpkg.com/quicktype/-/quicktype-23.0.170.tgz#3a70a5d0870e327d0f7ee35e61d25b17744bb8bc"
+  integrity sha512-3gFyS7w36ktxrttEv1gMfuUlGairepnSpLN0cp7JVevkKX2N6Uk8AyMlDS2Puki09MY6PB6ch90plThvACtEHA==
+  dependencies:
+    "@glideapps/ts-necessities" "^2.2.3"
+    chalk "^4.1.2"
+    collection-utils "^1.0.1"
+    command-line-args "^5.2.1"
+    command-line-usage "^7.0.1"
+    cross-fetch "^4.0.0"
+    graphql "^0.11.7"
+    lodash "^4.17.21"
+    moment "^2.30.1"
+    quicktype-core "23.0.170"
+    quicktype-graphql-input "23.0.170"
+    quicktype-typescript-input "23.0.170"
+    readable-stream "^4.5.2"
+    stream-json "1.8.0"
+    string-to-stream "^3.0.1"
+    typescript "4.9.5"
+
 raf@^3.4.1:
   version "3.4.1"
   resolved "https://registry.yarnpkg.com/raf/-/raf-3.4.1.tgz#0742e99a4a6552f445d73e3ee0328af0ff1ede39"
@@ -12330,6 +12552,17 @@ read-cache@^1.0.0:
   dependencies:
     pify "^2.3.0"
 
+readable-stream@4.5.2, readable-stream@^4.5.2:
+  version "4.5.2"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.5.2.tgz#9e7fc4c45099baeed934bff6eb97ba6cf2729e09"
+  integrity sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==
+  dependencies:
+    abort-controller "^3.0.0"
+    buffer "^6.0.3"
+    events "^3.3.0"
+    process "^0.11.10"
+    string_decoder "^1.3.0"
+
 "readable-stream@>=1.0.33-1 <1.1.0-0":
   version "1.0.34"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c"
@@ -12936,6 +13169,11 @@ safe-regex-test@^1.0.0:
     get-intrinsic "^1.1.3"
     is-regex "^1.1.4"
 
+safe-stable-stringify@^2.2.0:
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz#4ca2f8e385f2831c432a719b108a3bf7af42a1dd"
+  integrity sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==
+
 "safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
@@ -13346,6 +13584,18 @@ std-env@^3.5.0:
   resolved "https://registry.yarnpkg.com/std-env/-/std-env-3.7.0.tgz#c9f7386ced6ecf13360b6c6c55b8aaa4ef7481d2"
   integrity sha512-JPbdCEQLj1w5GilpiHAx3qJvFndqybBysA3qUOnznweH4QbNYUsW/ea8QzSrnh0vNsezMMw5bcVool8lM0gwzg==
 
+stream-chain@^2.2.5:
+  version "2.2.5"
+  resolved "https://registry.yarnpkg.com/stream-chain/-/stream-chain-2.2.5.tgz#b30967e8f14ee033c5b9a19bbe8a2cba90ba0d09"
+  integrity sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==
+
+stream-json@1.8.0:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/stream-json/-/stream-json-1.8.0.tgz#53f486b2e3b4496c506131f8d7260ba42def151c"
+  integrity sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==
+  dependencies:
+    stream-chain "^2.2.5"
+
 stream-parser@~0.3.1:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/stream-parser/-/stream-parser-0.3.1.tgz#1618548694420021a1182ff0af1911c129761773"
@@ -13380,6 +13630,13 @@ string-split-by@^1.0.0:
   dependencies:
     parenthesis "^3.1.5"
 
+string-to-stream@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/string-to-stream/-/string-to-stream-3.0.1.tgz#480e6fb4d5476d31cb2221f75307a5dcb6638a42"
+  integrity sha512-Hl092MV3USJuUCC6mfl9sPzGloA3K5VwdIeJjYIkXY/8K+mUvaeEabWJgArp+xXrsWxCajeT2pc4axbVhIZJyg==
+  dependencies:
+    readable-stream "^3.4.0"
+
 "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
@@ -13430,7 +13687,7 @@ string.prototype.trimstart@^1.0.6:
     define-properties "^1.1.4"
     es-abstract "^1.20.4"
 
-string_decoder@^1.1.1:
+string_decoder@^1.1.1, string_decoder@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
   integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
@@ -13635,6 +13892,14 @@ tabbable@^6.0.1:
   resolved "https://registry.yarnpkg.com/tabbable/-/tabbable-6.2.0.tgz#732fb62bc0175cfcec257330be187dcfba1f3b97"
   integrity sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==
 
+table-layout@^4.1.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/table-layout/-/table-layout-4.1.1.tgz#0f72965de1a5c0c1419c9ba21cae4e73a2f73a42"
+  integrity sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==
+  dependencies:
+    array-back "^6.2.2"
+    wordwrapjs "^5.1.0"
+
 tailwind-merge@^1.14.0:
   version "1.14.0"
   resolved "https://registry.yarnpkg.com/tailwind-merge/-/tailwind-merge-1.14.0.tgz#e677f55d864edc6794562c63f5001f45093cdb8b"
@@ -13733,6 +13998,11 @@ timers-ext@^0.1.7:
     es5-ext "~0.10.46"
     next-tick "1"
 
+tiny-inflate@^1.0.0:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/tiny-inflate/-/tiny-inflate-1.0.3.tgz#122715494913a1805166aaf7c93467933eea26c4"
+  integrity sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==
+
 tiny-invariant@1.0.6:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.0.6.tgz#b3f9b38835e36a41c843a3b0907a5a7b3755de73"
@@ -14144,6 +14414,26 @@ typescript@4.7.4:
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235"
   integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==
 
+typescript@4.9.4:
+  version "4.9.4"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.4.tgz#a2a3d2756c079abda241d75f149df9d561091e78"
+  integrity sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg==
+
+typescript@4.9.5:
+  version "4.9.5"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a"
+  integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==
+
+typical@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/typical/-/typical-4.0.0.tgz#cbeaff3b9d7ae1e2bbfaf5a4e6f11eccfde94fc4"
+  integrity sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==
+
+typical@^7.1.1:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/typical/-/typical-7.2.0.tgz#b3ec4b76530d144640df86c6b061dafd70e10c1e"
+  integrity sha512-W1+HdVRUl8fS3MZ9ogD51GOb46xMmhAZzR0WPw5jcgIZQJVvkddYzAl4YTU6g5w33Y1iRQLdIi2/1jhi2RNL0g==
+
 ua-parser-js@^1.0.35:
   version "1.0.36"
   resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-1.0.36.tgz#a9ab6b9bd3a8efb90bb0816674b412717b7c428c"
@@ -14222,11 +14512,27 @@ unicode-match-property-value-ecmascript@^2.1.0:
   resolved "https://registry.yarnpkg.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.1.0.tgz#cb5fffdcd16a05124f5a4b0bf7c3770208acbbe0"
   integrity sha512-qxkjQt6qjg/mYscYMC0XKRn3Rh0wFPlfxB0xkt9CfyTvpX1Ra0+rAmdX2QyAobptSEvuy4RtpPRui6XkV+8wjA==
 
+unicode-properties@^1.4.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/unicode-properties/-/unicode-properties-1.4.1.tgz#96a9cffb7e619a0dc7368c28da27e05fc8f9be5f"
+  integrity sha512-CLjCCLQ6UuMxWnbIylkisbRj31qxHPAurvena/0iwSVbQ2G1VY5/HjV0IRabOEbDHlzZlRdCrD4NhB0JtU40Pg==
+  dependencies:
+    base64-js "^1.3.0"
+    unicode-trie "^2.0.0"
+
 unicode-property-aliases-ecmascript@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz#43d41e3be698bd493ef911077c9b131f827e8ccd"
   integrity sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==
 
+unicode-trie@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/unicode-trie/-/unicode-trie-2.0.0.tgz#8fd8845696e2e14a8b67d78fa9e0dd2cad62fec8"
+  integrity sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ==
+  dependencies:
+    pako "^0.2.5"
+    tiny-inflate "^1.0.0"
+
 unified@^10.0.0, unified@^10.1.0:
   version "10.1.2"
   resolved "https://registry.yarnpkg.com/unified/-/unified-10.1.2.tgz#b1d64e55dafe1f0b98bb6c719881103ecf6c86df"
@@ -14431,6 +14737,11 @@ uri-js@^4.2.2:
   dependencies:
     punycode "^2.1.0"
 
+urijs@^1.19.1:
+  version "1.19.11"
+  resolved "https://registry.yarnpkg.com/urijs/-/urijs-1.19.11.tgz#204b0d6b605ae80bea54bea39280cdb7c9f923cc"
+  integrity sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==
+
 url-parse@^1.5.3:
   version "1.5.10"
   resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1"
@@ -15292,6 +15603,11 @@ wordwrap@^1.0.0:
   resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb"
   integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==
 
+wordwrapjs@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-5.1.0.tgz#4c4d20446dcc670b14fa115ef4f8fd9947af2b3a"
+  integrity sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==
+
 world-calendars@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/world-calendars/-/world-calendars-1.0.3.tgz#b25c5032ba24128ffc41d09faf4a5ec1b9c14335"
@@ -15414,6 +15730,11 @@ yaml@^2.3.1:
   resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.2.tgz#f522db4313c671a0ca963a75670f1c12ea909144"
   integrity sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==
 
+yaml@^2.4.1:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.6.0.tgz#14059ad9d0b1680d0f04d3a60fe00f3a857303c3"
+  integrity sha512-a6ae//JvKDEra2kdi1qzCyrJW/WZCgFi8ydDV+eXExl95t+5R+ijnqHJbz9tmMh8FUjx3iv2fCQ4dclAQlO2UQ==
+
 yargs-parser@20.x:
   version "20.2.9"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"
@@ -15449,7 +15770,7 @@ yargs@^15.3.1:
     y18n "^4.0.0"
     yargs-parser "^18.1.2"
 
-yargs@^17.0.0, yargs@^17.3.1, yargs@^17.5.1:
+yargs@^17.0.0, yargs@^17.1.1, yargs@^17.3.1, yargs@^17.5.1:
   version "17.7.2"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269"
   integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==
@@ -15525,6 +15846,11 @@ zen-observable@^0.10.0:
   resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.10.0.tgz#ee10eba75272897dbee5f152ab26bb5e0107f0c8"
   integrity sha512-iI3lT0iojZhKwT5DaFy2Ce42n3yFcLdFyOh01G7H0flMY60P8MJuVFEoJoNwXlmAyQ45GrjL6AcZmmlv8A5rbw==
 
+zod@^3.23.8:
+  version "3.23.8"
+  resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d"
+  integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==
+
 zwitch@^2.0.0, zwitch@^2.0.4:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-2.0.4.tgz#c827d4b0acb76fc3e685a4c6ec2902d51070e9d7"
diff --git a/weave/Makefile b/weave/Makefile
index 77943c2d9cb..15f90bad6e5 100644
--- a/weave/Makefile
+++ b/weave/Makefile
@@ -1,2 +1,5 @@
 update_costs:
 	python trace_server/costs/update_costs.py
+
+generate_base_object_schemas:
+	python scripts/generate_base_object_schemas.py
\ No newline at end of file
diff --git a/weave/scripts/generate_base_object_schemas.py b/weave/scripts/generate_base_object_schemas.py
new file mode 100644
index 00000000000..c78d5636cee
--- /dev/null
+++ b/weave/scripts/generate_base_object_schemas.py
@@ -0,0 +1,28 @@
+import json
+from pathlib import Path
+
+from weave.trace_server.interface.base_object_classes.base_object_registry import (
+    REGISTRY,
+    CompositeBaseObject,
+)
+
+OUTPUT_PATH = Path(__file__).parent / "generated_base_object_class_schemas.json"
+
+
+def generate_schemas():
+    """
+    Generate JSON schemas for all registered base objects in REGISTRY.
+    Creates a top-level union type of all registered objects and writes the schemas 
+    to a file named 'base_object_schemas.json'.
+    """
+    top_level_schema = CompositeBaseObject.model_json_schema(mode='validation')
+    
+    # Write schemas to file
+    with OUTPUT_PATH.open("w") as f:
+        json.dump(top_level_schema, f, indent=2)
+    
+    print(f"Generated union schema for {len(REGISTRY)} objects")
+    print(f"Wrote schema to {OUTPUT_PATH.absolute()}")
+
+if __name__ == "__main__":
+    generate_schemas()
\ No newline at end of file
diff --git a/weave/scripts/generated_base_object_class_schemas.json b/weave/scripts/generated_base_object_class_schemas.json
new file mode 100644
index 00000000000..4f7aee9dd54
--- /dev/null
+++ b/weave/scripts/generated_base_object_class_schemas.json
@@ -0,0 +1,114 @@
+{
+  "$defs": {
+    "TestOnlyExample": {
+      "properties": {
+        "name": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Name"
+        },
+        "description": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Description"
+        },
+        "primitive": {
+          "title": "Primitive",
+          "type": "integer"
+        },
+        "nested_base_model": {
+          "$ref": "#/$defs/TestOnlyNestedBaseModel"
+        },
+        "nested_base_object": {
+          "title": "Nested Base Object",
+          "type": "string"
+        }
+      },
+      "required": [
+        "primitive",
+        "nested_base_model",
+        "nested_base_object"
+      ],
+      "title": "TestOnlyExample",
+      "type": "object"
+    },
+    "TestOnlyNestedBaseModel": {
+      "properties": {
+        "a": {
+          "title": "A",
+          "type": "integer"
+        }
+      },
+      "required": [
+        "a"
+      ],
+      "title": "TestOnlyNestedBaseModel",
+      "type": "object"
+    },
+    "TestOnlyNestedBaseObject": {
+      "properties": {
+        "name": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Name"
+        },
+        "description": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Description"
+        },
+        "b": {
+          "title": "B",
+          "type": "integer"
+        }
+      },
+      "required": [
+        "b"
+      ],
+      "title": "TestOnlyNestedBaseObject",
+      "type": "object"
+    }
+  },
+  "properties": {
+    "TestOnlyExample": {
+      "$ref": "#/$defs/TestOnlyExample"
+    },
+    "TestOnlyNestedBaseObject": {
+      "$ref": "#/$defs/TestOnlyNestedBaseObject"
+    }
+  },
+  "required": [
+    "TestOnlyExample",
+    "TestOnlyNestedBaseObject"
+  ],
+  "title": "CompositeBaseObject",
+  "type": "object"
+}
\ No newline at end of file
diff --git a/weave/trace_server/interface/base_object_classes/base_object_registry.py b/weave/trace_server/interface/base_object_classes/base_object_registry.py
index bc679229f40..b4b9ab7242d 100644
--- a/weave/trace_server/interface/base_object_classes/base_object_registry.py
+++ b/weave/trace_server/interface/base_object_classes/base_object_registry.py
@@ -1,5 +1,7 @@
 from typing import Dict, Type
 
+from pydantic import BaseModel
+
 from weave.trace_server.interface.base_object_classes.base_object_def import BaseObject
 from weave.trace_server.interface.base_object_classes.test_only_example import *
 
@@ -7,3 +9,8 @@
     "TestOnlyExample": TestOnlyExample,
     "TestOnlyNestedBaseObject": TestOnlyNestedBaseObject,
 }
+
+# This is a union type of all registered base objects.
+class CompositeBaseObject(BaseModel):
+    TestOnlyExample: TestOnlyExample
+    TestOnlyNestedBaseObject: TestOnlyNestedBaseObject

From 2a6b2ac21beac94df139964c03361aec766b2c4e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 30 Oct 2024 17:13:28 -0700
Subject: [PATCH 127/169] beginning ts implementation

---
 .../wfReactInterface/baseObjectClassQuery.ts  | 123 ++++++++++++++++++
 .../traceServerClientTypes.ts                 |  22 +++-
 .../traceServerDirectClient.ts                |  46 +++++++
 3 files changed, 187 insertions(+), 4 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
new file mode 100644
index 00000000000..f9bc49da0f9
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -0,0 +1,123 @@
+import {useDeepMemo} from '@wandb/weave/hookUtils';
+import {useEffect, useState} from 'react';
+import {z} from 'zod';
+
+import { TestOnlyExampleSchema, TestOnlyNestedBaseObjectSchema } from './generatedBaseObjectClasses.zod';
+import { TraceServerClient } from './traceServerClient';
+import { useGetTraceServerClientContext } from './traceServerClientContext';
+import {
+  TraceObjCreateReq,
+  TraceObjQueryReq,
+  TraceObjSchema,
+} from './traceServerClientTypes';
+
+const collectionRegistry = {
+    TestOnlyExample: TestOnlyExampleSchema,
+    TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
+}
+
+export const useCollectionObjects = <
+  C extends keyof typeof collectionRegistry,
+  T extends z.infer<(typeof collectionRegistry)[C]>
+>(
+  collectionName: C,
+  req: TraceObjQueryReq
+) => {
+  const [objects, setObjects] = useState<Array<TraceObjSchema<T>>>([]);
+  const getTsClient = useGetTraceServerClientContext();
+  const client = getTsClient();
+  const deepReq = useDeepMemo(req);
+
+  useEffect(() => {
+    let isMounted = true;
+    getCollectionObjects(client, collectionName, deepReq).then(
+      collectionObjects => {
+        if (isMounted) {
+          setObjects(collectionObjects as Array<TraceObjSchema<T>>);
+        }
+      }
+    );
+    return () => {
+      isMounted = false;
+    };
+  }, [client, collectionName, deepReq]);
+
+  return objects;
+};
+
+const getCollectionObjects = async <
+  C extends keyof typeof collectionRegistry,
+  T extends z.infer<(typeof collectionRegistry)[C]>
+>(
+  client: TraceServerClient,
+  collectionName: C,
+  req: TraceObjQueryReq
+): Promise<Array<TraceObjSchema<T>>> => {
+  const knownCollection = collectionRegistry[collectionName];
+  if (!knownCollection) {
+    console.warn(`Unknown collection: ${collectionName}`);
+    return [];
+  }
+
+  const reqWithCollection: TraceObjQueryReq = {
+    ...req,
+    filter: {...req.filter, base_object_classes: [collectionName]},
+  };
+
+  const objectPromise = client.objsQuery(reqWithCollection);
+
+  const objects = await objectPromise;
+
+  return objects.objs
+    .map(obj => ({obj, parsed: knownCollection.safeParse(obj.val)}))
+    .filter(({parsed}) => parsed.success)
+    .map(({obj, parsed}) => ({...obj, val: parsed.data!})) as Array<
+    TraceObjSchema<T>
+  >;
+};
+
+export const useCreateCollectionObject = <
+  C extends keyof typeof collectionRegistry,
+  T extends z.infer<(typeof collectionRegistry)[C]>
+>(
+  collectionName: C
+) => {
+  const getTsClient = useGetTraceServerClientContext();
+  const client = getTsClient();
+  return (req: TraceObjCreateReq<T>) =>
+    createCollectionObject(client, collectionName, req);
+};
+
+const createCollectionObject = async <
+  C extends keyof typeof collectionRegistry,
+  T extends z.infer<(typeof collectionRegistry)[C]>
+>(
+  client: TraceServerClient,
+  collectionName: C,
+  req: TraceObjCreateReq<T>
+) => {
+  const knownCollection = collectionRegistry[collectionName];
+  if (!knownCollection) {
+    throw new Error(`Unknown collection: ${collectionName}`);
+  }
+
+  const verifiedObject = knownCollection.safeParse(req.obj.val);
+
+  if (!verifiedObject.success) {
+    throw new Error(
+      `Invalid object: ${JSON.stringify(verifiedObject.error.errors)}`
+    );
+  }
+
+  const reqWithCollection: TraceObjCreateReq = {
+    ...req,
+    obj: {
+      ...req.obj,
+      val: {...req.obj.val, _bases: [collectionName, 'BaseModel']},
+    },
+  };
+
+  const createPromse = client.objCreate(reqWithCollection);
+
+  return createPromse;
+};
\ No newline at end of file
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
index 88113a37a74..d84563a682f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
@@ -205,7 +205,7 @@ export type TraceObjQueryReq = {
   metadata_only?: boolean;
 };
 
-export interface TraceObjSchema {
+export interface TraceObjSchema<T extends any = any> {
   project_id: string;
   object_id: string;
   created_at: string;
@@ -214,11 +214,13 @@ export interface TraceObjSchema {
   is_latest: number;
   kind: 'op' | 'object';
   base_object_class?: string;
-  val: any;
+  val: T;
 }
-export type TraceObjQueryRes = {
-  objs: TraceObjSchema[];
+
+export type TraceObjQueryRes<T extends any = any> = {
+  objs: Array<TraceObjSchema<T>>;
 };
+
 export type TraceObjReadReq = {
   project_id: string;
   object_id: string;
@@ -229,6 +231,18 @@ export type TraceObjReadRes = {
   obj: TraceObjSchema;
 };
 
+export type TraceObjCreateReq<T extends any = any> = {
+  obj: {
+    project_id: string;
+    object_id: string;
+    val: T;
+  };
+};
+
+export type TraceObjCreateRes = {
+  digest: string;
+};
+
 export type TraceRefsReadBatchReq = {
   refs: string[];
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
index caaf63b7f56..16b56273fca 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
@@ -34,6 +34,8 @@ import {
   TraceCallUpdateReq,
   TraceFileContentReadReq,
   TraceFileContentReadRes,
+  TraceObjCreateReq,
+  TraceObjCreateRes,
   TraceObjQueryReq,
   TraceObjQueryRes,
   TraceObjReadReq,
@@ -231,6 +233,22 @@ export class DirectTraceServerClient {
     );
   }
 
+  public objCreate(req: TraceObjCreateReq): Promise<TraceObjCreateRes> {
+    const initialObjectId = req.obj.object_id;
+    const sanitizedObjectId = sanitizeObjectId(initialObjectId);
+    if (sanitizedObjectId !== initialObjectId) {
+      // Caller is expected to sanitize the object id. We should be doing this
+      // on the server, but it is currently disabled.
+      throw new Error(
+        `Invalid object name: ${initialObjectId}, sanitized to ${sanitizedObjectId}`
+      );
+    }
+    return this.makeRequest<TraceObjCreateReq, TraceObjCreateRes>(
+      '/obj/create',
+      req
+    );
+  }
+
   public tableQuery(req: TraceTableQueryReq): Promise<TraceTableQueryRes> {
     return this.makeRequest<TraceTableQueryReq, TraceTableQueryRes>(
       '/table/query',
@@ -376,3 +394,31 @@ export class DirectTraceServerClient {
     return prom;
   };
 }
+
+
+/**
+ * Sanitizes an object name by replacing non-alphanumeric characters with dashes and enforcing length limits.
+ * This matches the Python implementation in weave_client.py.
+ *
+ * @param name The name to sanitize
+ * @returns The sanitized name
+ * @throws Error if the resulting name would be empty
+ */
+export function sanitizeObjectId(name: string): string {
+  // Replace any non-word chars (except dots and underscores) with dashes
+  let res = name.replace(/[^\w._]+/g, '-');
+  // Replace multiple consecutive dashes/dots/underscores with a single dash
+  res = res.replace(/([._-]{2,})+/g, '-');
+  // Remove leading/trailing dashes and underscores
+  res = res.replace(/^[-_]+|[-_]+$/g, '');
+
+  if (!res) {
+    throw new Error(`Invalid object name: ${name}`);
+  }
+
+  if (res.length > 128) {
+    res = res.slice(0, 128);
+  }
+
+  return res;
+}
\ No newline at end of file

From 92cf5cd04d53d49d79907ff8eb31c93176fd14a4 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Wed, 30 Oct 2024 20:50:28 -0700
Subject: [PATCH 128/169] Initial TS implementation complete

---
 .../baseObjectClassQuery.test.ts              | 33 ++++++++++
 .../wfReactInterface/baseObjectClassQuery.ts  | 60 ++++++++++++-------
 .../generatedBaseObjectClasses.zod.ts         | 37 +++++++-----
 .../traceServerClientTypes.ts                 |  7 ++-
 .../traceServerDirectClient.ts                |  3 +-
 5 files changed, 99 insertions(+), 41 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts
new file mode 100644
index 00000000000..4bfe3ddd066
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts
@@ -0,0 +1,33 @@
+import {useCollectionObjects} from './baseObjectClassQuery';
+import {TestOnlyExampleSchema} from './generatedBaseObjectClasses.zod';
+describe('useCollectionObjects', () => {
+  it('hasCorrectTypes', () => {
+    type CT = ReturnType<typeof useCollectionObjects<'TestOnlyExample'>>;
+    const exampleResult: CT = {
+      loading: false,
+      data: [
+        {
+          project_id: '',
+          object_id: '',
+          created_at: '',
+          digest: '',
+          version_index: 0,
+          is_latest: 0,
+          kind: 'object',
+          base_object_class: 'TestOnlyExample',
+          val: TestOnlyExampleSchema.parse({
+            name: '',
+            description: '',
+            nested_base_model: {
+              a: 1,
+            },
+            nested_base_object: '',
+            primitive: 1,
+          }),
+        },
+      ],
+    };
+    // This should fail, looking for the correct way to validate types
+    expect(exampleResult).toBeNull();
+  });
+});
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index f9bc49da0f9..0d40cc4662f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -1,39 +1,57 @@
 import {useDeepMemo} from '@wandb/weave/hookUtils';
-import {useEffect, useState} from 'react';
+import {useEffect, useRef, useState} from 'react';
 import {z} from 'zod';
 
-import { TestOnlyExampleSchema, TestOnlyNestedBaseObjectSchema } from './generatedBaseObjectClasses.zod';
-import { TraceServerClient } from './traceServerClient';
-import { useGetTraceServerClientContext } from './traceServerClientContext';
+import {
+  TestOnlyExampleSchema,
+  TestOnlyNestedBaseObjectSchema,
+} from './generatedBaseObjectClasses.zod';
+import {TraceServerClient} from './traceServerClient';
+import {useGetTraceServerClientContext} from './traceServerClientContext';
 import {
   TraceObjCreateReq,
+  TraceObjCreateRes,
   TraceObjQueryReq,
   TraceObjSchema,
 } from './traceServerClientTypes';
 
 const collectionRegistry = {
-    TestOnlyExample: TestOnlyExampleSchema,
-    TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
-}
+  TestOnlyExample: TestOnlyExampleSchema,
+  TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
+};
+
+type Loadable<T> =
+  | {
+      loading: true;
+    }
+  | {
+      loading: false;
+      data: T;
+    };
 
 export const useCollectionObjects = <
   C extends keyof typeof collectionRegistry,
-  T extends z.infer<(typeof collectionRegistry)[C]>
+  T = z.infer<(typeof collectionRegistry)[C]>
 >(
   collectionName: C,
   req: TraceObjQueryReq
-) => {
-  const [objects, setObjects] = useState<Array<TraceObjSchema<T>>>([]);
+): Loadable<Array<TraceObjSchema<T, C>>> => {
+  const [objects, setObjects] = useState<Array<TraceObjSchema<T, C>>>([]);
   const getTsClient = useGetTraceServerClientContext();
   const client = getTsClient();
   const deepReq = useDeepMemo(req);
+  const currReq = useRef(deepReq);
+  const [loading, setLoading] = useState(true);
 
   useEffect(() => {
     let isMounted = true;
+    setLoading(true);
+    currReq.current = deepReq;
     getCollectionObjects(client, collectionName, deepReq).then(
       collectionObjects => {
-        if (isMounted) {
-          setObjects(collectionObjects as Array<TraceObjSchema<T>>);
+        if (isMounted && currReq.current === deepReq) {
+          setObjects(collectionObjects as Array<TraceObjSchema<T, C>>);
+          setLoading(false);
         }
       }
     );
@@ -42,17 +60,17 @@ export const useCollectionObjects = <
     };
   }, [client, collectionName, deepReq]);
 
-  return objects;
+  return {data: objects, loading};
 };
 
 const getCollectionObjects = async <
   C extends keyof typeof collectionRegistry,
-  T extends z.infer<(typeof collectionRegistry)[C]>
+  T = z.infer<(typeof collectionRegistry)[C]>
 >(
   client: TraceServerClient,
   collectionName: C,
   req: TraceObjQueryReq
-): Promise<Array<TraceObjSchema<T>>> => {
+): Promise<Array<TraceObjSchema<T, C>>> => {
   const knownCollection = collectionRegistry[collectionName];
   if (!knownCollection) {
     console.warn(`Unknown collection: ${collectionName}`);
@@ -72,16 +90,16 @@ const getCollectionObjects = async <
     .map(obj => ({obj, parsed: knownCollection.safeParse(obj.val)}))
     .filter(({parsed}) => parsed.success)
     .map(({obj, parsed}) => ({...obj, val: parsed.data!})) as Array<
-    TraceObjSchema<T>
+    TraceObjSchema<T, C>
   >;
 };
 
 export const useCreateCollectionObject = <
   C extends keyof typeof collectionRegistry,
-  T extends z.infer<(typeof collectionRegistry)[C]>
+  T = z.infer<(typeof collectionRegistry)[C]>
 >(
   collectionName: C
-) => {
+): ((req: TraceObjCreateReq<T>) => Promise<TraceObjCreateRes>) => {
   const getTsClient = useGetTraceServerClientContext();
   const client = getTsClient();
   return (req: TraceObjCreateReq<T>) =>
@@ -90,12 +108,12 @@ export const useCreateCollectionObject = <
 
 const createCollectionObject = async <
   C extends keyof typeof collectionRegistry,
-  T extends z.infer<(typeof collectionRegistry)[C]>
+  T = z.infer<(typeof collectionRegistry)[C]>
 >(
   client: TraceServerClient,
   collectionName: C,
   req: TraceObjCreateReq<T>
-) => {
+): Promise<TraceObjCreateRes> => {
   const knownCollection = collectionRegistry[collectionName];
   if (!knownCollection) {
     throw new Error(`Unknown collection: ${collectionName}`);
@@ -120,4 +138,4 @@ const createCollectionObject = async <
   const createPromse = client.objCreate(reqWithCollection);
 
   return createPromse;
-};
\ No newline at end of file
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
index a01bf4e68ee..ba5a38ebf65 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
@@ -1,29 +1,34 @@
-import * as z from "zod";
-
+import * as z from 'zod';
 
 export const TestOnlyNestedBaseModelSchema = z.object({
-    "a": z.number(),
+  a: z.number(),
 });
-export type TestOnlyNestedBaseModel = z.infer<typeof TestOnlyNestedBaseModelSchema>;
+export type TestOnlyNestedBaseModel = z.infer<
+  typeof TestOnlyNestedBaseModelSchema
+>;
 
 export const TestOnlyNestedBaseObjectSchema = z.object({
-    "b": z.number(),
-    "description": z.union([z.null(), z.string()]).optional(),
-    "name": z.union([z.null(), z.string()]).optional(),
+  b: z.number(),
+  description: z.union([z.null(), z.string()]).optional(),
+  name: z.union([z.null(), z.string()]).optional(),
 });
-export type TestOnlyNestedBaseObject = z.infer<typeof TestOnlyNestedBaseObjectSchema>;
+export type TestOnlyNestedBaseObject = z.infer<
+  typeof TestOnlyNestedBaseObjectSchema
+>;
 
 export const TestOnlyExampleSchema = z.object({
-    "description": z.union([z.null(), z.string()]).optional(),
-    "name": z.union([z.null(), z.string()]).optional(),
-    "nested_base_model": TestOnlyNestedBaseModelSchema,
-    "nested_base_object": z.string(),
-    "primitive": z.number(),
+  description: z.union([z.null(), z.string()]).optional(),
+  name: z.union([z.null(), z.string()]).optional(),
+  nested_base_model: TestOnlyNestedBaseModelSchema,
+  nested_base_object: z.string(),
+  primitive: z.number(),
 });
 export type TestOnlyExample = z.infer<typeof TestOnlyExampleSchema>;
 
 export const GeneratedBaseObjectClassesZodSchema = z.object({
-    "TestOnlyExample": TestOnlyExampleSchema,
-    "TestOnlyNestedBaseObject": TestOnlyNestedBaseObjectSchema,
+  TestOnlyExample: TestOnlyExampleSchema,
+  TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
 });
-export type GeneratedBaseObjectClassesZod = z.infer<typeof GeneratedBaseObjectClassesZodSchema>;
+export type GeneratedBaseObjectClassesZod = z.infer<
+  typeof GeneratedBaseObjectClassesZodSchema
+>;
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
index d84563a682f..d57b2fb9a67 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
@@ -205,7 +205,10 @@ export type TraceObjQueryReq = {
   metadata_only?: boolean;
 };
 
-export interface TraceObjSchema<T extends any = any> {
+export interface TraceObjSchema<
+  T extends any = any,
+  OBC extends string = string
+> {
   project_id: string;
   object_id: string;
   created_at: string;
@@ -213,7 +216,7 @@ export interface TraceObjSchema<T extends any = any> {
   version_index: number;
   is_latest: number;
   kind: 'op' | 'object';
-  base_object_class?: string;
+  base_object_class?: OBC;
   val: T;
 }
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
index 16b56273fca..25f11624740 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
@@ -395,7 +395,6 @@ export class DirectTraceServerClient {
   };
 }
 
-
 /**
  * Sanitizes an object name by replacing non-alphanumeric characters with dashes and enforcing length limits.
  * This matches the Python implementation in weave_client.py.
@@ -421,4 +420,4 @@ export function sanitizeObjectId(name: string): string {
   }
 
   return res;
-}
\ No newline at end of file
+}

From 259e4c028fcd7cdd1bf77e60f324f85abd3743ef Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 00:28:28 -0600
Subject: [PATCH 129/169] Initial TS tests complete

---
 weave-js/package.json                         |  3 +-
 .../baseObjectClassQuery.test.ts              | 90 +++++++++++++++++--
 .../wfReactInterface/baseObjectClassQuery.ts  | 12 +--
 3 files changed, 85 insertions(+), 20 deletions(-)

diff --git a/weave-js/package.json b/weave-js/package.json
index 5897561a6c1..6dc7da04766 100644
--- a/weave-js/package.json
+++ b/weave-js/package.json
@@ -237,7 +237,8 @@
     "typescript": "4.7.4",
     "uuid": "^9.0.0",
     "vite": "5.2.9",
-    "vitest": "^1.6.0"
+    "vitest": "^1.6.0",
+    "tsd": "^0.30.4"
   },
   "resolutions": {
     "@types/react": "^17.0.26",
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts
index 4bfe3ddd066..83cfc3060de 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts
@@ -1,11 +1,52 @@
-import {useCollectionObjects} from './baseObjectClassQuery';
-import {TestOnlyExampleSchema} from './generatedBaseObjectClasses.zod';
-describe('useCollectionObjects', () => {
-  it('hasCorrectTypes', () => {
-    type CT = ReturnType<typeof useCollectionObjects<'TestOnlyExample'>>;
-    const exampleResult: CT = {
+import {expectType} from 'tsd';
+
+import {
+  useCollectionObjects,
+  useCreateCollectionObject,
+} from './baseObjectClassQuery';
+import {
+  TestOnlyExample,
+  TestOnlyExampleSchema,
+} from './generatedBaseObjectClasses.zod';
+import {
+  TraceObjCreateReq,
+  TraceObjCreateRes,
+  TraceObjSchema,
+} from './traceServerClientTypes';
+import {Loadable} from './wfDataModelHooksInterface';
+
+type TypesAreEqual<T, U> = [T] extends [U]
+  ? [U] extends [T]
+    ? true
+    : false
+  : false;
+
+describe('Type Tests', () => {
+  it('useCollectionObjects return type matches expected structure', () => {
+    type CollectionObjectsReturn = ReturnType<
+      typeof useCollectionObjects<'TestOnlyExample'>
+    >;
+
+    // Define the expected type structure
+    type ExpectedType = Loadable<
+      Array<TraceObjSchema<TestOnlyExample, 'TestOnlyExample'>>
+    >;
+
+    // Type assertion tests
+    type AssertTypesAreEqual = TypesAreEqual<
+      CollectionObjectsReturn,
+      ExpectedType
+    >;
+    type Assert = AssertTypesAreEqual extends true ? true : never;
+
+    // This will fail compilation if the types don't match exactly
+    const _assert: Assert = true;
+    expect(_assert).toBe(true);
+
+    // Additional runtime sample for documentation
+    const sampleResult: CollectionObjectsReturn = {
       loading: false,
-      data: [
+      result: [
         {
           project_id: '',
           object_id: '',
@@ -27,7 +68,38 @@ describe('useCollectionObjects', () => {
         },
       ],
     };
-    // This should fail, looking for the correct way to validate types
-    expect(exampleResult).toBeNull();
+
+    expectType<ExpectedType>(sampleResult);
+  });
+
+  it('useCreateCollectionObject return type matches expected structure', () => {
+    type CreateCollectionObjectReturn = ReturnType<
+      typeof useCreateCollectionObject<'TestOnlyExample'>
+    >;
+
+    // Define the expected type structure
+    type ExpectedType = (
+      req: TraceObjCreateReq<TestOnlyExample>
+    ) => Promise<TraceObjCreateRes>;
+
+    // Type assertion tests
+    type AssertTypesAreEqual = TypesAreEqual<
+      CreateCollectionObjectReturn,
+      ExpectedType
+    >;
+    type Assert = AssertTypesAreEqual extends true ? true : never;
+
+    // This will fail compilation if the types don't match exactly
+    const _assert: Assert = true;
+    expect(_assert).toBe(true);
+
+    // Additional runtime sample for documentation
+    const sampleResult: CreateCollectionObjectReturn = async req => {
+      return {
+        digest: '',
+      };
+    };
+
+    expectType<ExpectedType>(sampleResult);
   });
 });
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index 0d40cc4662f..945d0c0409b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -14,21 +14,13 @@ import {
   TraceObjQueryReq,
   TraceObjSchema,
 } from './traceServerClientTypes';
+import {Loadable} from './wfDataModelHooksInterface';
 
 const collectionRegistry = {
   TestOnlyExample: TestOnlyExampleSchema,
   TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
 };
 
-type Loadable<T> =
-  | {
-      loading: true;
-    }
-  | {
-      loading: false;
-      data: T;
-    };
-
 export const useCollectionObjects = <
   C extends keyof typeof collectionRegistry,
   T = z.infer<(typeof collectionRegistry)[C]>
@@ -60,7 +52,7 @@ export const useCollectionObjects = <
     };
   }, [client, collectionName, deepReq]);
 
-  return {data: objects, loading};
+  return {result: objects, loading};
 };
 
 const getCollectionObjects = async <

From ac471c4b7766d9eaaa19dcff031b2bedcf35d9b9 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 00:43:13 -0600
Subject: [PATCH 130/169] Initial TS tests complete

---
 weave-js/yarn.lock                            | 332 +++++++++++++++++-
 weave/scripts/generate_base_object_schemas.py |  13 +-
 .../base_object_registry.py                   |   1 +
 3 files changed, 330 insertions(+), 16 deletions(-)

diff --git a/weave-js/yarn.lock b/weave-js/yarn.lock
index 8541ca9a5d7..1a0ce3ab43e 100644
--- a/weave-js/yarn.lock
+++ b/weave-js/yarn.lock
@@ -4150,6 +4150,11 @@
   resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9"
   integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==
 
+"@tsd/typescript@~5.3.3":
+  version "5.3.3"
+  resolved "https://registry.yarnpkg.com/@tsd/typescript/-/typescript-5.3.3.tgz#bc01854b6e0e746b5f70a6b48c30c7b95b81a74e"
+  integrity sha512-CQlfzol0ldaU+ftWuG52vH29uRoKboLinLy84wS8TQOu+m+tWoaUfk4svL4ij2V8M5284KymJBlHUusKj6k34w==
+
 "@turf/area@^6.4.0":
   version "6.5.0"
   resolved "https://registry.yarnpkg.com/@turf/area/-/area-6.5.0.tgz#1d0d7aee01d8a4a3d4c91663ed35cc615f36ad56"
@@ -4261,6 +4266,19 @@
   resolved "https://registry.yarnpkg.com/@types/downloadjs/-/downloadjs-1.4.3.tgz#01c0414a9756afa2bd438b1e2ec6c50de7df316d"
   integrity sha512-MjJepFle/tLtT2/jmDNth6ZnwWzEhm40L+olE5HKR70ISUCfgT55eqreeHldAzFLY2HDUGsn8zgyto8KygN0CA==
 
+"@types/eslint@^7.2.13":
+  version "7.29.0"
+  resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-7.29.0.tgz#e56ddc8e542815272720bb0b4ccc2aff9c3e1c78"
+  integrity sha512-VNcvioYDH8/FxaeTKkM4/TiTwt6pBV9E3OfGmvaw8tPl0rrHCJ4Ll15HRT+pMiFAf/MLQvAzC+6RzUMEL9Ceng==
+  dependencies:
+    "@types/estree" "*"
+    "@types/json-schema" "*"
+
+"@types/estree@*":
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50"
+  integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==
+
 "@types/estree@1.0.5":
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4"
@@ -4363,6 +4381,11 @@
   resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.6.tgz#4b3afd5158b8749095b1f096967b6d0f838d862f"
   integrity sha512-ACTuifTSIIbyksx2HTon3aFtCKWcID7/h3XEmRpDYdMCXxPbl+m9GteOJeaAkiAta/NJaSFuA7ahZ0NkwajDSw==
 
+"@types/json-schema@*":
+  version "7.0.15"
+  resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841"
+  integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==
+
 "@types/json-schema@^7.0.11", "@types/json-schema@^7.0.6", "@types/json-schema@^7.0.9":
   version "7.0.12"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb"
@@ -4407,6 +4430,11 @@
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-5.1.2.tgz#07508b45797cb81ec3f273011b054cd0755eddca"
   integrity sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==
 
+"@types/minimist@^1.2.0":
+  version "1.2.5"
+  resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.5.tgz#ec10755e871497bcd83efe927e43ec46e8c0747e"
+  integrity sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==
+
 "@types/ms@*":
   version "0.7.31"
   resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197"
@@ -4432,6 +4460,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.19.tgz#cb03fca8910fdeb7595b755126a8a78144714eea"
   integrity sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==
 
+"@types/normalize-package-data@^2.4.0":
+  version "2.4.4"
+  resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz#56e2cc26c397c038fab0e3a917a12d5c5909e901"
+  integrity sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==
+
 "@types/numeral@^2.0.2":
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/@types/numeral/-/numeral-2.0.2.tgz#8ea2c4f4e64c0cc948ad7da375f6f827778a7912"
@@ -5323,6 +5356,11 @@ array.prototype.tosorted@^1.1.1:
     es-shim-unscopables "^1.0.0"
     get-intrinsic "^1.1.3"
 
+arrify@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d"
+  integrity sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==
+
 asap@~2.0.3, asap@~2.0.6:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46"
@@ -5803,7 +5841,16 @@ camelcase-css@^2.0.1:
   resolved "https://registry.yarnpkg.com/camelcase-css/-/camelcase-css-2.0.1.tgz#ee978f6947914cc30c6b44741b6ed1df7f043fd5"
   integrity sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==
 
-camelcase@^5.0.0:
+camelcase-keys@^6.2.2:
+  version "6.2.2"
+  resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-6.2.2.tgz#5e755d6ba51aa223ec7d3d52f25778210f9dc3c0"
+  integrity sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==
+  dependencies:
+    camelcase "^5.3.1"
+    map-obj "^4.0.0"
+    quick-lru "^4.0.1"
+
+camelcase@^5.0.0, camelcase@^5.3.1:
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
   integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
@@ -6972,7 +7019,15 @@ debug@^3.1.0, debug@^3.2.6, debug@^3.2.7:
   dependencies:
     ms "^2.1.1"
 
-decamelize@^1.2.0:
+decamelize-keys@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/decamelize-keys/-/decamelize-keys-1.1.1.tgz#04a2d523b2f18d80d0158a43b895d56dff8d19d8"
+  integrity sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==
+  dependencies:
+    decamelize "^1.1.0"
+    map-obj "^1.0.0"
+
+decamelize@^1.1.0, decamelize@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
   integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==
@@ -7581,6 +7636,20 @@ eslint-config-react-app@^7.0.1:
     eslint-plugin-react-hooks "^4.3.0"
     eslint-plugin-testing-library "^5.0.1"
 
+eslint-formatter-pretty@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/eslint-formatter-pretty/-/eslint-formatter-pretty-4.1.0.tgz#7a6877c14ffe2672066c853587d89603e97c7708"
+  integrity sha512-IsUTtGxF1hrH6lMWiSl1WbGaiP01eT6kzywdY1U+zLc0MP+nwEnUiS9UI8IaOTUhTeQJLlCEWIbXINBH4YJbBQ==
+  dependencies:
+    "@types/eslint" "^7.2.13"
+    ansi-escapes "^4.2.1"
+    chalk "^4.1.0"
+    eslint-rule-docs "^1.1.5"
+    log-symbols "^4.0.0"
+    plur "^4.0.0"
+    string-width "^4.2.0"
+    supports-hyperlinks "^2.0.0"
+
 eslint-import-resolver-node@^0.3.7:
   version "0.3.7"
   resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.7.tgz#83b375187d412324a1963d84fa664377a23eb4d7"
@@ -7744,6 +7813,11 @@ eslint-rule-composer@^0.3.0:
   resolved "https://registry.yarnpkg.com/eslint-rule-composer/-/eslint-rule-composer-0.3.0.tgz#79320c927b0c5c0d3d3d2b76c8b4a488f25bbaf9"
   integrity sha512-bt+Sh8CtDmn2OajxvNO+BX7Wn4CIWMpTRm3MaiKPCQcnnlm0CS2mhui6QaoeQugs+3Kj2ESKEEGJUdVafwhiCg==
 
+eslint-rule-docs@^1.1.5:
+  version "1.1.235"
+  resolved "https://registry.yarnpkg.com/eslint-rule-docs/-/eslint-rule-docs-1.1.235.tgz#be6ef1fc3525f17b3c859ae2997fedadc89bfb9b"
+  integrity sha512-+TQ+x4JdTnDoFEXXb3fDvfGOwnyNV7duH8fXWTPD1ieaBmB8omj7Gw/pMBBu4uI2uJCCU8APDaQJzWuXnTsH4A==
+
 eslint-scope@5.1.1, eslint-scope@^5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c"
@@ -8490,7 +8564,7 @@ globalthis@^1.0.3:
   dependencies:
     define-properties "^1.1.3"
 
-globby@^11.0.3, globby@^11.1.0:
+globby@^11.0.1, globby@^11.0.3, globby@^11.1.0:
   version "11.1.0"
   resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b"
   integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==
@@ -8737,6 +8811,11 @@ har-validator@~4.2.1:
     ajv "^4.9.1"
     har-schema "^1.0.5"
 
+hard-rejection@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/hard-rejection/-/hard-rejection-2.1.0.tgz#1c6eda5c1685c63942766d79bb40ae773cecd883"
+  integrity sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==
+
 harmony-reflect@^1.4.6:
   version "1.6.2"
   resolved "https://registry.yarnpkg.com/harmony-reflect/-/harmony-reflect-1.6.2.tgz#31ecbd32e648a34d030d86adb67d4d47547fe710"
@@ -8819,6 +8898,13 @@ hasown@^2.0.0:
   dependencies:
     function-bind "^1.1.2"
 
+hasown@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003"
+  integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==
+  dependencies:
+    function-bind "^1.1.2"
+
 hast-util-from-parse5@^5.0.0:
   version "5.0.3"
   resolved "https://registry.yarnpkg.com/hast-util-from-parse5/-/hast-util-from-parse5-5.0.3.tgz#3089dc0ee2ccf6ec8bc416919b51a54a589e097c"
@@ -9037,6 +9123,18 @@ hoist-non-react-statics@^3.0.0, hoist-non-react-statics@^3.1.0, hoist-non-react-
   dependencies:
     react-is "^16.7.0"
 
+hosted-git-info@^2.1.4:
+  version "2.8.9"
+  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
+  integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==
+
+hosted-git-info@^4.0.1:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz#827b82867e9ff1c8d0c4d9d53880397d2c86d224"
+  integrity sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==
+  dependencies:
+    lru-cache "^6.0.0"
+
 hsluv@^0.0.3:
   version "0.0.3"
   resolved "https://registry.yarnpkg.com/hsluv/-/hsluv-0.0.3.tgz#829107dafb4a9f8b52a1809ed02e091eade6754c"
@@ -9261,6 +9359,11 @@ invariant@^2.2.4:
   dependencies:
     loose-envify "^1.0.0"
 
+irregular-plurals@^3.2.0:
+  version "3.5.0"
+  resolved "https://registry.yarnpkg.com/irregular-plurals/-/irregular-plurals-3.5.0.tgz#0835e6639aa8425bdc8b0d33d0dc4e89d9c01d2b"
+  integrity sha512-1ANGLZ+Nkv1ptFb2pa8oG8Lem4krflKuX/gINiHJHjJUKaJHk/SXk5x6K3J+39/p0h1RQ2saROclJJ+QLvETCQ==
+
 is-absolute@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-absolute/-/is-absolute-1.0.0.tgz#395e1ae84b11f26ad1795e73c17378e48a301576"
@@ -9363,6 +9466,13 @@ is-core-module@^2.11.0, is-core-module@^2.9.0:
   dependencies:
     has "^1.0.3"
 
+is-core-module@^2.13.0, is-core-module@^2.5.0:
+  version "2.15.1"
+  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.1.tgz#a7363a25bee942fefab0de13bf6aa372c82dcc37"
+  integrity sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==
+  dependencies:
+    hasown "^2.0.2"
+
 is-date-object@^1.0.1:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f"
@@ -9674,6 +9784,16 @@ iterall@1.1.3:
   resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.1.3.tgz#1cbbff96204056dde6656e2ed2e2226d0e6d72c9"
   integrity sha512-Cu/kb+4HiNSejAPhSaN1VukdNTTi/r4/e+yykqjlG/IW+1gZH5b4+Bq3whDX4tvbYugta3r8KTMUiqT3fIGxuQ==
 
+jest-diff@^29.0.3:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.7.0.tgz#017934a66ebb7ecf6f205e84699be10afd70458a"
+  integrity sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==
+  dependencies:
+    chalk "^4.0.0"
+    diff-sequences "^29.6.3"
+    jest-get-type "^29.6.3"
+    pretty-format "^29.7.0"
+
 jest-diff@^29.6.2:
   version "29.6.2"
   resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.6.2.tgz#c36001e5543e82a0805051d3ceac32e6825c1c46"
@@ -9694,6 +9814,11 @@ jest-get-type@^29.4.3:
   resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.4.3.tgz#1ab7a5207c995161100b5187159ca82dd48b3dd5"
   integrity sha512-J5Xez4nRRMjk8emnTpWrlkyb9pfRQQanDrvWHhsR1+VUfbwxi30eVcZFlcdGInRibU4G5LwHXpI7IRHU0CY+gg==
 
+jest-get-type@^29.6.3:
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1"
+  integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==
+
 jest-matcher-utils@^29.6.2:
   version "29.6.2"
   resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.6.2.tgz#39de0be2baca7a64eacb27291f0bd834fea3a535"
@@ -9862,6 +9987,11 @@ json-schema-to-typescript@^11.0.2:
     mz "^2.7.0"
     prettier "^2.6.2"
 
+json-schema-to-zod@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/json-schema-to-zod/-/json-schema-to-zod-2.4.1.tgz#bd1e66b4ac4da4d3293e59d9a582b77d8ce9744b"
+  integrity sha512-aMoez9TxgnfLAIZaWTPaQ+j7rOt1K9Ew/TBI85XcnhcFlo/47b1MDgpi4r07XndLSZWOX/KsJiRJvhdzSvo2Dw==
+
 json-schema-traverse@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
@@ -10063,6 +10193,11 @@ keyboard-key@^1.0.4:
   resolved "https://registry.yarnpkg.com/keyboard-key/-/keyboard-key-1.1.0.tgz#6f2e8e37fa11475bb1f1d65d5174f1b35653f5b7"
   integrity sha512-qkBzPTi3rlAKvX7k0/ub44sqOfXeLc/jcnGGmj5c7BJpU8eDrEVPyhCvNYAaoubbsLm9uGWwQJO1ytQK1a9/dQ==
 
+kind-of@^6.0.3:
+  version "6.0.3"
+  resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd"
+  integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==
+
 kleur@^4.0.3:
   version "4.1.5"
   resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.1.5.tgz#95106101795f7050c6c650f350c683febddb1780"
@@ -10323,6 +10458,16 @@ map-limit@0.0.1:
   dependencies:
     once "~1.3.0"
 
+map-obj@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d"
+  integrity sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==
+
+map-obj@^4.0.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.3.0.tgz#9304f906e93faae70880da102a9f1df0ea8bb05a"
+  integrity sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==
+
 mapbox-gl@1.10.1:
   version "1.10.1"
   resolved "https://registry.yarnpkg.com/mapbox-gl/-/mapbox-gl-1.10.1.tgz#7dbd53bdf2f78e45e125c1115e94dea286ef663c"
@@ -10534,6 +10679,24 @@ memoizee@^0.4.15:
     next-tick "^1.1.0"
     timers-ext "^0.1.7"
 
+meow@^9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/meow/-/meow-9.0.0.tgz#cd9510bc5cac9dee7d03c73ee1f9ad959f4ea364"
+  integrity sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ==
+  dependencies:
+    "@types/minimist" "^1.2.0"
+    camelcase-keys "^6.2.2"
+    decamelize "^1.2.0"
+    decamelize-keys "^1.1.0"
+    hard-rejection "^2.1.0"
+    minimist-options "4.1.0"
+    normalize-package-data "^3.0.0"
+    read-pkg-up "^7.0.1"
+    redent "^3.0.0"
+    trim-newlines "^3.0.0"
+    type-fest "^0.18.0"
+    yargs-parser "^20.2.3"
+
 merge-stream@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60"
@@ -10881,6 +11044,15 @@ minimatch@^4.2.3:
   dependencies:
     brace-expansion "^1.1.7"
 
+minimist-options@4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619"
+  integrity sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==
+  dependencies:
+    arrify "^1.0.1"
+    is-plain-obj "^1.1.0"
+    kind-of "^6.0.3"
+
 minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
@@ -11211,6 +11383,26 @@ nopt@~1.0.10:
   dependencies:
     abbrev "1"
 
+normalize-package-data@^2.5.0:
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8"
+  integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==
+  dependencies:
+    hosted-git-info "^2.1.4"
+    resolve "^1.10.0"
+    semver "2 || 3 || 4 || 5"
+    validate-npm-package-license "^3.0.1"
+
+normalize-package-data@^3.0.0:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz#dbcc3e2da59509a0983422884cd172eefdfa525e"
+  integrity sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==
+  dependencies:
+    hosted-git-info "^4.0.1"
+    is-core-module "^2.5.0"
+    semver "^7.3.4"
+    validate-npm-package-license "^3.0.1"
+
 normalize-path@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9"
@@ -11847,6 +12039,13 @@ plotly.js@^2.23.2:
     webgl-context "^2.2.0"
     world-calendars "^1.0.3"
 
+plur@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/plur/-/plur-4.0.0.tgz#729aedb08f452645fe8c58ef115bf16b0a73ef84"
+  integrity sha512-4UGewrYgqDFw9vV6zNV+ADmPAUAfJPKtGvb/VdpQAx25X5f3xXdGdyOEVFwkl8Hl/tl7+xbeHqSEM+D5/TirUg==
+  dependencies:
+    irregular-plurals "^3.2.0"
+
 pluralize@^8.0.0:
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"
@@ -12122,6 +12321,11 @@ queue-microtask@^1.2.2:
   resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243"
   integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==
 
+quick-lru@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f"
+  integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==
+
 quickselect@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/quickselect/-/quickselect-2.0.0.tgz#f19680a486a5eefb581303e023e98faaf25dd018"
@@ -12552,6 +12756,25 @@ read-cache@^1.0.0:
   dependencies:
     pify "^2.3.0"
 
+read-pkg-up@^7.0.0, read-pkg-up@^7.0.1:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz#f3a6135758459733ae2b95638056e1854e7ef507"
+  integrity sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==
+  dependencies:
+    find-up "^4.1.0"
+    read-pkg "^5.2.0"
+    type-fest "^0.8.1"
+
+read-pkg@^5.2.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-5.2.0.tgz#7bf295438ca5a33e56cd30e053b34ee7250c93cc"
+  integrity sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==
+  dependencies:
+    "@types/normalize-package-data" "^2.4.0"
+    normalize-package-data "^2.5.0"
+    parse-json "^5.0.0"
+    type-fest "^0.6.0"
+
 readable-stream@4.5.2, readable-stream@^4.5.2:
   version "4.5.2"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.5.2.tgz#9e7fc4c45099baeed934bff6eb97ba6cf2729e09"
@@ -13025,6 +13248,15 @@ resolve@^1.0.0, resolve@^1.1.10, resolve@^1.1.5, resolve@^1.1.7, resolve@^1.10.1
     path-parse "^1.0.7"
     supports-preserve-symlinks-flag "^1.0.0"
 
+resolve@^1.10.0:
+  version "1.22.8"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d"
+  integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==
+  dependencies:
+    is-core-module "^2.13.0"
+    path-parse "^1.0.7"
+    supports-preserve-symlinks-flag "^1.0.0"
+
 resolve@^2.0.0-next.4:
   version "2.0.0-next.4"
   resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.4.tgz#3d37a113d6429f496ec4752d2a2e58efb1fd4660"
@@ -13247,6 +13479,11 @@ semantic-ui-react@^0.88.2:
     react-popper "^1.3.4"
     shallowequal "^1.1.0"
 
+"semver@2 || 3 || 4 || 5", semver@^5.3.0, semver@^5.7.1:
+  version "5.7.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
+  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
+
 semver@7.x, semver@^7.3.5, semver@^7.3.7:
   version "7.5.4"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e"
@@ -13254,16 +13491,16 @@ semver@7.x, semver@^7.3.5, semver@^7.3.7:
   dependencies:
     lru-cache "^6.0.0"
 
-semver@^5.3.0, semver@^5.7.1:
-  version "5.7.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
-  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
-
 semver@^6.0.0, semver@^6.1.0, semver@^6.3.0, semver@^6.3.1:
   version "6.3.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
   integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
 
+semver@^7.3.4:
+  version "7.6.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
+  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
+
 semver@~7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e"
@@ -13513,6 +13750,32 @@ spawn-command@^0.0.2-1:
   resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2-1.tgz#62f5e9466981c1b796dc5929937e11c9c6921bd0"
   integrity sha512-n98l9E2RMSJ9ON1AKisHzz7V42VDiBQGY6PB1BwRglz99wpVsSuGzQ+jOi6lFXBGVTCrRpltvjm+/XA+tpeJrg==
 
+spdx-correct@^3.0.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.2.0.tgz#4f5ab0668f0059e34f9c00dce331784a12de4e9c"
+  integrity sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==
+  dependencies:
+    spdx-expression-parse "^3.0.0"
+    spdx-license-ids "^3.0.0"
+
+spdx-exceptions@^2.1.0:
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz#5d607d27fc806f66d7b64a766650fa890f04ed66"
+  integrity sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==
+
+spdx-expression-parse@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz#cf70f50482eefdc98e3ce0a6833e4a53ceeba679"
+  integrity sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==
+  dependencies:
+    spdx-exceptions "^2.1.0"
+    spdx-license-ids "^3.0.0"
+
+spdx-license-ids@^3.0.0:
+  version "3.0.20"
+  resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.20.tgz#e44ed19ed318dd1e5888f93325cee800f0f51b89"
+  integrity sha512-jg25NiDV/1fLtSgEgyvVyDunvaNHbuwF9lfNV17gSmPFAlYzdfNBlLtLzXTevwkPj7DhGbmN9VnmJIgLnhvaBw==
+
 split-on-first@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/split-on-first/-/split-on-first-3.0.0.tgz#f04959c9ea8101b9b0bbf35a61b9ebea784a23e7"
@@ -13820,7 +14083,7 @@ supports-color@^5.3.0, supports-color@^5.5.0:
   dependencies:
     has-flag "^3.0.0"
 
-supports-color@^7.1.0:
+supports-color@^7.0.0, supports-color@^7.1.0:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da"
   integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==
@@ -13834,6 +14097,14 @@ supports-color@^8.1.0:
   dependencies:
     has-flag "^4.0.0"
 
+supports-hyperlinks@^2.0.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz#3943544347c1ff90b15effb03fc14ae45ec10624"
+  integrity sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==
+  dependencies:
+    has-flag "^4.0.0"
+    supports-color "^7.0.0"
+
 supports-preserve-symlinks-flag@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09"
@@ -14158,6 +14429,11 @@ trim-lines@^3.0.0:
   resolved "https://registry.yarnpkg.com/trim-lines/-/trim-lines-3.0.1.tgz#d802e332a07df861c48802c04321017b1bd87338"
   integrity sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==
 
+trim-newlines@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144"
+  integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==
+
 trim-trailing-lines@^1.0.0:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/trim-trailing-lines/-/trim-trailing-lines-1.1.4.tgz#bd4abbec7cc880462f10b2c8b5ce1d8d1ec7c2c0"
@@ -14243,6 +14519,19 @@ tsconfig-paths@^3.14.1:
     minimist "^1.2.6"
     strip-bom "^3.0.0"
 
+tsd@^0.30.4:
+  version "0.30.7"
+  resolved "https://registry.yarnpkg.com/tsd/-/tsd-0.30.7.tgz#319a0403073df6d3f572c4089378901662554ae5"
+  integrity sha512-oTiJ28D6B/KXoU3ww/Eji+xqHJojiuPVMwA12g4KYX1O72N93Nb6P3P3h2OAhhf92Xl8NIhb/xFmBZd5zw/xUw==
+  dependencies:
+    "@tsd/typescript" "~5.3.3"
+    eslint-formatter-pretty "^4.1.0"
+    globby "^11.0.1"
+    jest-diff "^29.0.3"
+    meow "^9.0.0"
+    path-exists "^4.0.0"
+    read-pkg-up "^7.0.0"
+
 tslib@>=1.10.0, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.1.0, tslib@^2.5.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.0.tgz#b295854684dbda164e181d259a22cd779dcd7bc3"
@@ -14351,6 +14640,11 @@ type-detect@^4.0.0, type-detect@^4.0.8:
   resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c"
   integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==
 
+type-fest@^0.18.0:
+  version "0.18.1"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.18.1.tgz#db4bc151a4a2cf4eebf9add5db75508db6cc841f"
+  integrity sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==
+
 type-fest@^0.20.2:
   version "0.20.2"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4"
@@ -14361,6 +14655,16 @@ type-fest@^0.21.3:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37"
   integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==
 
+type-fest@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b"
+  integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==
+
+type-fest@^0.8.1:
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
+  integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
+
 type@^1.0.1:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/type/-/type-1.2.0.tgz#848dd7698dafa3e54a6c479e759c4bc3f18847a0"
@@ -14832,6 +15136,14 @@ v8-compile-cache-lib@^3.0.1:
   resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
   integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==
 
+validate-npm-package-license@^3.0.1:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a"
+  integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==
+  dependencies:
+    spdx-correct "^3.0.0"
+    spdx-expression-parse "^3.0.0"
+
 value-equal@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/value-equal/-/value-equal-1.0.1.tgz#1e0b794c734c5c0cade179c437d356d931a34d6c"
@@ -15735,7 +16047,7 @@ yaml@^2.4.1:
   resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.6.0.tgz#14059ad9d0b1680d0f04d3a60fe00f3a857303c3"
   integrity sha512-a6ae//JvKDEra2kdi1qzCyrJW/WZCgFi8ydDV+eXExl95t+5R+ijnqHJbz9tmMh8FUjx3iv2fCQ4dclAQlO2UQ==
 
-yargs-parser@20.x:
+yargs-parser@20.x, yargs-parser@^20.2.3:
   version "20.2.9"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"
   integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==
diff --git a/weave/scripts/generate_base_object_schemas.py b/weave/scripts/generate_base_object_schemas.py
index c78d5636cee..3be5ee1b2a8 100644
--- a/weave/scripts/generate_base_object_schemas.py
+++ b/weave/scripts/generate_base_object_schemas.py
@@ -9,20 +9,21 @@
 OUTPUT_PATH = Path(__file__).parent / "generated_base_object_class_schemas.json"
 
 
-def generate_schemas():
+def generate_schemas() -> None:
     """
     Generate JSON schemas for all registered base objects in REGISTRY.
-    Creates a top-level union type of all registered objects and writes the schemas 
+    Creates a top-level union type of all registered objects and writes the schemas
     to a file named 'base_object_schemas.json'.
     """
-    top_level_schema = CompositeBaseObject.model_json_schema(mode='validation')
-    
+    top_level_schema = CompositeBaseObject.model_json_schema(mode="validation")
+
     # Write schemas to file
     with OUTPUT_PATH.open("w") as f:
         json.dump(top_level_schema, f, indent=2)
-    
+
     print(f"Generated union schema for {len(REGISTRY)} objects")
     print(f"Wrote schema to {OUTPUT_PATH.absolute()}")
 
+
 if __name__ == "__main__":
-    generate_schemas()
\ No newline at end of file
+    generate_schemas()
diff --git a/weave/trace_server/interface/base_object_classes/base_object_registry.py b/weave/trace_server/interface/base_object_classes/base_object_registry.py
index b4b9ab7242d..6bdcce460b9 100644
--- a/weave/trace_server/interface/base_object_classes/base_object_registry.py
+++ b/weave/trace_server/interface/base_object_classes/base_object_registry.py
@@ -10,6 +10,7 @@
     "TestOnlyNestedBaseObject": TestOnlyNestedBaseObject,
 }
 
+
 # This is a union type of all registered base objects.
 class CompositeBaseObject(BaseModel):
     TestOnlyExample: TestOnlyExample

From e326b1730ecbb9d4b3640ad3059be07383ad4d02 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 01:36:27 -0600
Subject: [PATCH 131/169] Initial python tests complete

---
 tests/trace/test_base_object_classes.py       | 80 +++++++++++++++++--
 .../wfReactInterface/baseObjectClassQuery.ts  |  1 +
 weave/scripts/generate_base_object_schemas.py |  4 +-
 weave/trace/serialize.py                      |  5 ++
 weave/trace_server/base_object_class_util.py  | 17 ++++
 .../clickhouse_trace_server_batched.py        | 15 +---
 .../base_object_registry.py                   |  4 +-
 weave/trace_server/sqlite_trace_server.py     | 15 +---
 8 files changed, 101 insertions(+), 40 deletions(-)
 create mode 100644 weave/trace_server/base_object_class_util.py

diff --git a/tests/trace/test_base_object_classes.py b/tests/trace/test_base_object_classes.py
index c0c9ba25b14..9d4922becfc 100644
--- a/tests/trace/test_base_object_classes.py
+++ b/tests/trace/test_base_object_classes.py
@@ -12,6 +12,8 @@
 4. We ensure that invalid schemas are properly rejected from the server.
 """
 
+from typing import Literal, Optional
+
 import pytest
 
 import weave
@@ -21,13 +23,34 @@
 from weave.trace_server import trace_server_interface as tsi
 
 
+def with_base_object_class_annotations(
+    val: dict,
+    class_name: str,
+    base_object_name: Optional[Literal["Object", "BaseObject"]] = None,
+):
+    """
+    When serializing pydantic objects, add additional fields to indicate the class information. This is
+    a utlity to perform that mapping for the purposes of testing. We want to ensure that both the client
+    and server agree on this structure, therefore I am adding this utility here.
+    """
+    bases = ["BaseModel"]
+    if base_object_name is not None:
+        bases.insert(0, base_object_name)
+    return {
+        **val,
+        "_type": class_name,
+        "_class_name": class_name,
+        "_bases": bases,
+    }
+
+
 def test_pythonic_creation(client: WeaveClient):
     # First, let's use the high-level pythonic creation API.
     nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
     top_obj = base_objects.TestOnlyExample(
         primitive=1,
         nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
-        nested_obj=weave.publish(nested_obj).uri(),
+        nested_base_object=weave.publish(nested_obj).uri(),
     )
     ref = weave.publish(top_obj)
 
@@ -36,7 +59,7 @@ def test_pythonic_creation(client: WeaveClient):
     assert isinstance(top_obj_gotten, base_objects.TestOnlyExample)
     assert top_obj_gotten.model_dump() == top_obj.model_dump()
 
-    objs = client.server.obj_query(
+    objs_res = client.server.objs_query(
         tsi.ObjQueryReq.model_validate(
             {
                 "project_id": client._project_id(),
@@ -44,11 +67,37 @@ def test_pythonic_creation(client: WeaveClient):
             },
         )
     )
+    objs = objs_res.objs
 
     assert len(objs) == 1
-    assert objs[0].val == top_obj.model_dump()
+    assert (
+        objs[0].val
+        == {
+            **with_base_object_class_annotations(
+                top_obj.model_dump(), "TestOnlyExample", "BaseObject"
+            ),
+            "nested_base_model": with_base_object_class_annotations(
+                top_obj.nested_base_model.model_dump(), "TestOnlyNestedBaseModel"
+            ),
+        }
+        == {
+            "_type": "TestOnlyExample",
+            "name": None,
+            "description": None,
+            "primitive": 1,
+            "nested_base_model": {
+                "_type": "TestOnlyNestedBaseModel",
+                "a": 2,
+                "_class_name": "TestOnlyNestedBaseModel",
+                "_bases": ["BaseModel"],
+            },
+            "nested_base_object": "weave:///shawn/test-project/object/TestOnlyNestedBaseObject:JyFvHfyaJ79uCKpdZ3DD3if4NYam8QgTkzUlXQXAILI",
+            "_class_name": "TestOnlyExample",
+            "_bases": ["BaseObject", "BaseModel"],
+        }
+    )
 
-    objs = client.server.obj_query(
+    objs_res = client.server.objs_query(
         tsi.ObjQueryReq.model_validate(
             {
                 "project_id": client._project_id(),
@@ -56,9 +105,23 @@ def test_pythonic_creation(client: WeaveClient):
             },
         )
     )
+    objs = objs_res.objs
 
     assert len(objs) == 1
-    assert objs[0].val == nested_obj.model_dump()
+    assert (
+        objs[0].val
+        == with_base_object_class_annotations(
+            nested_obj.model_dump(), "TestOnlyNestedBaseObject", "BaseObject"
+        )
+        == {
+            "_type": "TestOnlyNestedBaseObject",
+            "name": None,
+            "description": None,
+            "b": 3,
+            "_class_name": "TestOnlyNestedBaseObject",
+            "_bases": ["BaseObject", "BaseModel"],
+        }
+    )
 
 
 def test_interface_creation(client):
@@ -115,7 +178,7 @@ def test_interface_creation(client):
 
     assert nested_obj_gotten.model_dump() == nested_obj.model_dump()
 
-    objs = client.server.obj_query(
+    objs_res = client.server.obj_query(
         tsi.ObjQueryReq.model_validate(
             {
                 "project_id": client._project_id(),
@@ -124,10 +187,11 @@ def test_interface_creation(client):
         )
     )
 
+    objs = objs_res.objs
     assert len(objs) == 1
     assert objs[0].val == top_obj.model_dump()
 
-    objs = client.server.obj_query(
+    objs_res = client.server.obj_query(
         tsi.ObjQueryReq.model_validate(
             {
                 "project_id": client._project_id(),
@@ -135,7 +199,7 @@ def test_interface_creation(client):
             },
         )
     )
-
+    objs = objs_res.objs
     assert len(objs) == 1
     assert objs[0].val == nested_obj.model_dump()
 
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index 945d0c0409b..efb3bd3d817 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -16,6 +16,7 @@ import {
 } from './traceServerClientTypes';
 import {Loadable} from './wfDataModelHooksInterface';
 
+// TODO: This should be generated from the registry!
 const collectionRegistry = {
   TestOnlyExample: TestOnlyExampleSchema,
   TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
diff --git a/weave/scripts/generate_base_object_schemas.py b/weave/scripts/generate_base_object_schemas.py
index 3be5ee1b2a8..979565a09e1 100644
--- a/weave/scripts/generate_base_object_schemas.py
+++ b/weave/scripts/generate_base_object_schemas.py
@@ -2,7 +2,7 @@
 from pathlib import Path
 
 from weave.trace_server.interface.base_object_classes.base_object_registry import (
-    REGISTRY,
+    BASE_OBJECT_REGISTRY,
     CompositeBaseObject,
 )
 
@@ -21,7 +21,7 @@ def generate_schemas() -> None:
     with OUTPUT_PATH.open("w") as f:
         json.dump(top_level_schema, f, indent=2)
 
-    print(f"Generated union schema for {len(REGISTRY)} objects")
+    print(f"Generated union schema for {len(BASE_OBJECT_REGISTRY)} objects")
     print(f"Wrote schema to {OUTPUT_PATH.absolute()}")
 
 
diff --git a/weave/trace/serialize.py b/weave/trace/serialize.py
index 2f605e24b77..c37ae03b7a7 100644
--- a/weave/trace/serialize.py
+++ b/weave/trace/serialize.py
@@ -5,6 +5,9 @@
 from weave.trace import custom_objs
 from weave.trace.object_record import ObjectRecord
 from weave.trace.refs import ObjectRef, TableRef, parse_uri
+from weave.trace_server.interface.base_object_classes.base_object_registry import (
+    BASE_OBJECT_REGISTRY,
+)
 from weave.trace_server.trace_server_interface import (
     FileContentReadReq,
     FileCreateReq,
@@ -225,6 +228,8 @@ def from_json(obj: Any, project_id: str, server: TraceServerInterface) -> Any:
             return custom_objs.decode_custom_obj(
                 obj["weave_type"], files, obj.get("load_op")
             )
+        elif baseObject := BASE_OBJECT_REGISTRY.get(val_type):
+            return baseObject.model_validate(obj)
         else:
             return ObjectRecord(
                 {k: from_json(v, project_id, server) for k, v in obj.items()}
diff --git a/weave/trace_server/base_object_class_util.py b/weave/trace_server/base_object_class_util.py
new file mode 100644
index 00000000000..50aea59084d
--- /dev/null
+++ b/weave/trace_server/base_object_class_util.py
@@ -0,0 +1,17 @@
+from typing import Any, Optional
+
+base_object_class_names = ["BaseObject", "Object"]
+
+
+def get_base_object_class(val: Any) -> Optional[str]:
+    if isinstance(val, dict):
+        if "_bases" in val:
+            if isinstance(val["_bases"], list):
+                if len(val["_bases"]) >= 2:
+                    if val["_bases"][-1] == "BaseModel":
+                        if val["_bases"][-2] in base_object_class_names:
+                            if len(val["_bases"]) > 2:
+                                return val["_bases"][-3]
+                            elif "_class_name" in val:
+                                return val["_class_name"]
+    return None
diff --git a/weave/trace_server/clickhouse_trace_server_batched.py b/weave/trace_server/clickhouse_trace_server_batched.py
index eb4ce265f70..9ed9533787e 100644
--- a/weave/trace_server/clickhouse_trace_server_batched.py
+++ b/weave/trace_server/clickhouse_trace_server_batched.py
@@ -53,6 +53,7 @@
 from weave.trace_server import environment as wf_env
 from weave.trace_server import refs_internal as ri
 from weave.trace_server import trace_server_interface as tsi
+from weave.trace_server.base_object_class_util import get_base_object_class
 from weave.trace_server.calls_query_builder import (
     CallsQuery,
     HardCodedFilter,
@@ -2059,20 +2060,6 @@ def get_kind(val: Any) -> str:
     return "object"
 
 
-def get_base_object_class(val: Any) -> Optional[str]:
-    if isinstance(val, dict):
-        if "_bases" in val:
-            if isinstance(val["_bases"], list):
-                if len(val["_bases"]) >= 2:
-                    if val["_bases"][-1] == "BaseModel":
-                        if val["_bases"][-2] == "Object":
-                            if len(val["_bases"]) > 2:
-                                return val["_bases"][-3]
-                            elif "_class_name" in val:
-                                return val["_class_name"]
-    return None
-
-
 def find_call_descendants(
     root_ids: list[str],
     all_calls: list[tsi.CallSchema],
diff --git a/weave/trace_server/interface/base_object_classes/base_object_registry.py b/weave/trace_server/interface/base_object_classes/base_object_registry.py
index 6bdcce460b9..43191f43671 100644
--- a/weave/trace_server/interface/base_object_classes/base_object_registry.py
+++ b/weave/trace_server/interface/base_object_classes/base_object_registry.py
@@ -5,13 +5,13 @@
 from weave.trace_server.interface.base_object_classes.base_object_def import BaseObject
 from weave.trace_server.interface.base_object_classes.test_only_example import *
 
-REGISTRY: Dict[str, Type[BaseObject]] = {
+BASE_OBJECT_REGISTRY: Dict[str, Type[BaseObject]] = {
     "TestOnlyExample": TestOnlyExample,
     "TestOnlyNestedBaseObject": TestOnlyNestedBaseObject,
 }
 
 
-# This is a union type of all registered base objects.
+# TODO: Remove this helper
 class CompositeBaseObject(BaseModel):
     TestOnlyExample: TestOnlyExample
     TestOnlyNestedBaseObject: TestOnlyNestedBaseObject
diff --git a/weave/trace_server/sqlite_trace_server.py b/weave/trace_server/sqlite_trace_server.py
index 93a4f510090..51b51e7ca64 100644
--- a/weave/trace_server/sqlite_trace_server.py
+++ b/weave/trace_server/sqlite_trace_server.py
@@ -13,6 +13,7 @@
 
 from weave.trace_server import refs_internal as ri
 from weave.trace_server import trace_server_interface as tsi
+from weave.trace_server.base_object_class_util import get_base_object_class
 from weave.trace_server.emoji_util import detone_emojis
 from weave.trace_server.errors import InvalidRequest
 from weave.trace_server.feedback import (
@@ -1205,20 +1206,6 @@ def get_kind(val: Any) -> str:
     return "object"
 
 
-def get_base_object_class(val: Any) -> Optional[str]:
-    if isinstance(val, dict):
-        if "_bases" in val:
-            if isinstance(val["_bases"], list):
-                if len(val["_bases"]) >= 2:
-                    if val["_bases"][-1] == "BaseModel":
-                        if val["_bases"][-2] == "Object":
-                            if len(val["_bases"]) > 2:
-                                return val["_bases"][-3]
-                            elif "_class_name" in val:
-                                return val["_class_name"]
-    return None
-
-
 def _transform_external_calls_field_to_internal_calls_field(
     field: str,
     cast: Optional[str] = None,

From 154de4798d096fdec5dbacc822813e69a73c278f Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 02:06:15 -0600
Subject: [PATCH 132/169] Typescript improvements

---
 .../baseObjectClassQuery.test.ts              |  8 +-
 .../wfReactInterface/baseObjectClassQuery.ts  | 79 ++++++++++---------
 .../traceServerClientTypes.ts                 |  1 +
 3 files changed, 48 insertions(+), 40 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts
index 83cfc3060de..9918ae7f285 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.test.ts
@@ -1,8 +1,8 @@
 import {expectType} from 'tsd';
 
 import {
-  useCollectionObjects,
-  useCreateCollectionObject,
+  useBaseObjectInstances,
+  useCreateBaseObjectInstance,
 } from './baseObjectClassQuery';
 import {
   TestOnlyExample,
@@ -24,7 +24,7 @@ type TypesAreEqual<T, U> = [T] extends [U]
 describe('Type Tests', () => {
   it('useCollectionObjects return type matches expected structure', () => {
     type CollectionObjectsReturn = ReturnType<
-      typeof useCollectionObjects<'TestOnlyExample'>
+      typeof useBaseObjectInstances<'TestOnlyExample'>
     >;
 
     // Define the expected type structure
@@ -74,7 +74,7 @@ describe('Type Tests', () => {
 
   it('useCreateCollectionObject return type matches expected structure', () => {
     type CreateCollectionObjectReturn = ReturnType<
-      typeof useCreateCollectionObject<'TestOnlyExample'>
+      typeof useCreateBaseObjectInstance<'TestOnlyExample'>
     >;
 
     // Define the expected type structure
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index efb3bd3d817..3d2f4513e45 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -17,16 +17,16 @@ import {
 import {Loadable} from './wfDataModelHooksInterface';
 
 // TODO: This should be generated from the registry!
-const collectionRegistry = {
+const baseObjectClassRegistry = {
   TestOnlyExample: TestOnlyExampleSchema,
   TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
 };
 
-export const useCollectionObjects = <
-  C extends keyof typeof collectionRegistry,
-  T = z.infer<(typeof collectionRegistry)[C]>
+export const useBaseObjectInstances = <
+  C extends keyof typeof baseObjectClassRegistry,
+  T = z.infer<(typeof baseObjectClassRegistry)[C]>
 >(
-  collectionName: C,
+  baseObjectClassName: C,
   req: TraceObjQueryReq
 ): Loadable<Array<TraceObjSchema<T, C>>> => {
   const [objects, setObjects] = useState<Array<TraceObjSchema<T, C>>>([]);
@@ -40,7 +40,7 @@ export const useCollectionObjects = <
     let isMounted = true;
     setLoading(true);
     currReq.current = deepReq;
-    getCollectionObjects(client, collectionName, deepReq).then(
+    getBaseObjectInstances(client, baseObjectClassName, deepReq).then(
       collectionObjects => {
         if (isMounted && currReq.current === deepReq) {
           setObjects(collectionObjects as Array<TraceObjSchema<T, C>>);
@@ -51,68 +51,77 @@ export const useCollectionObjects = <
     return () => {
       isMounted = false;
     };
-  }, [client, collectionName, deepReq]);
+  }, [client, baseObjectClassName, deepReq]);
 
   return {result: objects, loading};
 };
 
-const getCollectionObjects = async <
-  C extends keyof typeof collectionRegistry,
-  T = z.infer<(typeof collectionRegistry)[C]>
+const getBaseObjectInstances = async <
+  C extends keyof typeof baseObjectClassRegistry,
+  T = z.infer<(typeof baseObjectClassRegistry)[C]>
 >(
   client: TraceServerClient,
-  collectionName: C,
+  baseObjectClassName: C,
   req: TraceObjQueryReq
 ): Promise<Array<TraceObjSchema<T, C>>> => {
-  const knownCollection = collectionRegistry[collectionName];
-  if (!knownCollection) {
-    console.warn(`Unknown collection: ${collectionName}`);
+  const knownObjectClass = baseObjectClassRegistry[baseObjectClassName];
+  if (!knownObjectClass) {
+    console.warn(`Unknown object class: ${baseObjectClassName}`);
     return [];
   }
 
-  const reqWithCollection: TraceObjQueryReq = {
+  const reqWithBaseObjectClass: TraceObjQueryReq = {
     ...req,
-    filter: {...req.filter, base_object_classes: [collectionName]},
+    filter: {...req.filter, base_object_classes: [baseObjectClassName]},
   };
 
-  const objectPromise = client.objsQuery(reqWithCollection);
+  const objectPromise = client.objsQuery(reqWithBaseObjectClass);
 
   const objects = await objectPromise;
 
   return objects.objs
-    .map(obj => ({obj, parsed: knownCollection.safeParse(obj.val)}))
+    .map(obj => ({obj, parsed: knownObjectClass.safeParse(obj.val)}))
     .filter(({parsed}) => parsed.success)
     .map(({obj, parsed}) => ({...obj, val: parsed.data!})) as Array<
     TraceObjSchema<T, C>
   >;
 };
 
-export const useCreateCollectionObject = <
-  C extends keyof typeof collectionRegistry,
-  T = z.infer<(typeof collectionRegistry)[C]>
+export const useCreateBaseObjectInstance = <
+  C extends keyof typeof baseObjectClassRegistry,
+  T = z.infer<(typeof baseObjectClassRegistry)[C]>
 >(
-  collectionName: C
+  baseObjectClassName: C
 ): ((req: TraceObjCreateReq<T>) => Promise<TraceObjCreateRes>) => {
   const getTsClient = useGetTraceServerClientContext();
   const client = getTsClient();
   return (req: TraceObjCreateReq<T>) =>
-    createCollectionObject(client, collectionName, req);
+    createBaseObjectInstance(client, baseObjectClassName, req);
 };
 
-const createCollectionObject = async <
-  C extends keyof typeof collectionRegistry,
-  T = z.infer<(typeof collectionRegistry)[C]>
+const createBaseObjectInstance = async <
+  C extends keyof typeof baseObjectClassRegistry,
+  T = z.infer<(typeof baseObjectClassRegistry)[C]>
 >(
   client: TraceServerClient,
-  collectionName: C,
+  baseObjectClassName: C,
   req: TraceObjCreateReq<T>
 ): Promise<TraceObjCreateRes> => {
-  const knownCollection = collectionRegistry[collectionName];
-  if (!knownCollection) {
-    throw new Error(`Unknown collection: ${collectionName}`);
+  if (
+    req.obj.set_base_object_class != null &&
+    req.obj.set_base_object_class !== baseObjectClassName
+  ) {
+    throw new Error(
+      `set_base_object_class must match baseObjectClassName: ${baseObjectClassName}`
+    );
   }
 
-  const verifiedObject = knownCollection.safeParse(req.obj.val);
+  const knownBaseObjectClass = baseObjectClassRegistry[baseObjectClassName];
+  if (!knownBaseObjectClass) {
+    throw new Error(`Unknown object class: ${baseObjectClassName}`);
+  }
+
+  const verifiedObject = knownBaseObjectClass.safeParse(req.obj.val);
 
   if (!verifiedObject.success) {
     throw new Error(
@@ -120,15 +129,13 @@ const createCollectionObject = async <
     );
   }
 
-  const reqWithCollection: TraceObjCreateReq = {
+  const reqWithBaseObjectClass: TraceObjCreateReq = {
     ...req,
     obj: {
       ...req.obj,
-      val: {...req.obj.val, _bases: [collectionName, 'BaseModel']},
+      set_base_object_class: baseObjectClassName,
     },
   };
 
-  const createPromse = client.objCreate(reqWithCollection);
-
-  return createPromse;
+  return client.objCreate(reqWithBaseObjectClass);
 };
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
index d57b2fb9a67..d7219af5834 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerClientTypes.ts
@@ -239,6 +239,7 @@ export type TraceObjCreateReq<T extends any = any> = {
     project_id: string;
     object_id: string;
     val: T;
+    set_base_object_class?: string;
   };
 };
 

From 454058c8a196f3c858ae65f41ac4b59124ad2286 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 02:41:41 -0600
Subject: [PATCH 133/169] Python Tests complete

---
 tests/trace/test_base_object_classes.py       | 125 +++++++++++++++---
 weave/trace_server/base_object_class_util.py  |  68 +++++++++-
 .../clickhouse_trace_server_batched.py        |  21 +--
 weave/trace_server/sqlite_trace_server.py     |  23 ++--
 weave/trace_server/trace_server_interface.py  |   1 +
 5 files changed, 201 insertions(+), 37 deletions(-)

diff --git a/tests/trace/test_base_object_classes.py b/tests/trace/test_base_object_classes.py
index 9d4922becfc..2d52fd1dc27 100644
--- a/tests/trace/test_base_object_classes.py
+++ b/tests/trace/test_base_object_classes.py
@@ -15,6 +15,7 @@
 from typing import Literal, Optional
 
 import pytest
+from pydantic import ValidationError
 
 import weave
 from weave.trace import base_objects
@@ -126,7 +127,7 @@ def test_pythonic_creation(client: WeaveClient):
 
 def test_interface_creation(client):
     # Now we will do the equivant operation using low-level interface.
-    nested_obj_id = "nested_obj"
+    nested_obj_id = "TestOnlyNestedBaseObject"
     nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
     nested_obj_res = client.server.obj_create(
         tsi.ObjCreateReq.model_validate(
@@ -135,6 +136,7 @@ def test_interface_creation(client):
                     "project_id": client._project_id(),
                     "object_id": nested_obj_id,
                     "val": nested_obj.model_dump(),
+                    "set_base_object_class": "TestOnlyNestedBaseObject",
                 }
             }
         )
@@ -143,14 +145,14 @@ def test_interface_creation(client):
         entity=client.entity,
         project=client.project,
         name=nested_obj_id,
-        digest=nested_obj_res.digest,
+        _digest=nested_obj_res.digest,
     )
 
-    top_level_obj_id = "top_obj"
+    top_level_obj_id = "TestOnlyExample"
     top_obj = base_objects.TestOnlyExample(
         primitive=1,
         nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
-        nested_obj=nested_obj_ref.uri(),
+        nested_base_object=nested_obj_ref.uri(),
     )
     top_obj_res = client.server.obj_create(
         tsi.ObjCreateReq.model_validate(
@@ -159,6 +161,7 @@ def test_interface_creation(client):
                     "project_id": client._project_id(),
                     "object_id": top_level_obj_id,
                     "val": top_obj.model_dump(),
+                    "set_base_object_class": "TestOnlyExample",
                 }
             }
         )
@@ -167,7 +170,7 @@ def test_interface_creation(client):
         entity=client.entity,
         project=client.project,
         name=top_level_obj_id,
-        digest=top_obj_res.digest,
+        _digest=top_obj_res.digest,
     )
 
     top_obj_gotten = weave.ref(top_obj_ref.uri()).get()
@@ -178,7 +181,7 @@ def test_interface_creation(client):
 
     assert nested_obj_gotten.model_dump() == nested_obj.model_dump()
 
-    objs_res = client.server.obj_query(
+    objs_res = client.server.objs_query(
         tsi.ObjQueryReq.model_validate(
             {
                 "project_id": client._project_id(),
@@ -189,9 +192,34 @@ def test_interface_creation(client):
 
     objs = objs_res.objs
     assert len(objs) == 1
-    assert objs[0].val == top_obj.model_dump()
+    assert (
+        objs[0].val
+        == {
+            **with_base_object_class_annotations(
+                top_obj.model_dump(), "TestOnlyExample", "BaseObject"
+            ),
+            "nested_base_model": with_base_object_class_annotations(
+                top_obj.nested_base_model.model_dump(), "TestOnlyNestedBaseModel"
+            ),
+        }
+        == {
+            "_type": "TestOnlyExample",
+            "name": None,
+            "description": None,
+            "primitive": 1,
+            "nested_base_model": {
+                "_type": "TestOnlyNestedBaseModel",
+                "a": 2,
+                "_class_name": "TestOnlyNestedBaseModel",
+                "_bases": ["BaseModel"],
+            },
+            "nested_base_object": "weave:///shawn/test-project/object/TestOnlyNestedBaseObject:JyFvHfyaJ79uCKpdZ3DD3if4NYam8QgTkzUlXQXAILI",
+            "_class_name": "TestOnlyExample",
+            "_bases": ["BaseObject", "BaseModel"],
+        }
+    )
 
-    objs_res = client.server.obj_query(
+    objs_res = client.server.objs_query(
         tsi.ObjQueryReq.model_validate(
             {
                 "project_id": client._project_id(),
@@ -201,23 +229,37 @@ def test_interface_creation(client):
     )
     objs = objs_res.objs
     assert len(objs) == 1
-    assert objs[0].val == nested_obj.model_dump()
+    assert (
+        objs[0].val
+        == with_base_object_class_annotations(
+            nested_obj.model_dump(), "TestOnlyNestedBaseObject", "BaseObject"
+        )
+        == {
+            "_type": "TestOnlyNestedBaseObject",
+            "name": None,
+            "description": None,
+            "b": 3,
+            "_class_name": "TestOnlyNestedBaseObject",
+            "_bases": ["BaseObject", "BaseModel"],
+        }
+    )
 
 
 def test_digest_equality(client):
     # Next, let's make sure that the digests are all equivalent
-
     nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
+    nested_ref = weave.publish(nested_obj)
     top_obj = base_objects.TestOnlyExample(
         primitive=1,
         nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
-        nested_obj=weave.publish(nested_obj).uri(),
+        nested_base_object=nested_ref.uri(),
     )
     ref = weave.publish(top_obj)
-    pythonic_digest = ref.digest
+    nested_pythonic_digest = nested_ref.digest
+    top_level_pythonic_digest = ref.digest
 
     # Now we will do the equivant operation using low-level interface.
-    nested_obj_id = "nested_obj"
+    nested_obj_id = "TestOnlyNestedBaseObject"
     nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
     nested_obj_res = client.server.obj_create(
         tsi.ObjCreateReq.model_validate(
@@ -226,6 +268,7 @@ def test_digest_equality(client):
                     "project_id": client._project_id(),
                     "object_id": nested_obj_id,
                     "val": nested_obj.model_dump(),
+                    "set_base_object_class": "TestOnlyNestedBaseObject",
                 }
             }
         )
@@ -234,14 +277,18 @@ def test_digest_equality(client):
         entity=client.entity,
         project=client.project,
         name=nested_obj_id,
-        digest=nested_obj_res.digest,
+        _digest=nested_obj_res.digest,
     )
 
-    top_level_obj_id = "top_obj"
+    nested_interface_style_digest = nested_obj_ref.digest
+
+    assert nested_pythonic_digest == nested_interface_style_digest
+
+    top_level_obj_id = "TestOnlyExample"
     top_obj = base_objects.TestOnlyExample(
         primitive=1,
         nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
-        nested_obj=nested_obj_ref.uri(),
+        nested_base_object=nested_obj_ref.uri(),
     )
     top_obj_res = client.server.obj_create(
         tsi.ObjCreateReq.model_validate(
@@ -250,26 +297,66 @@ def test_digest_equality(client):
                     "project_id": client._project_id(),
                     "object_id": top_level_obj_id,
                     "val": top_obj.model_dump(),
+                    "set_base_object_class": "TestOnlyExample",
                 }
             }
         )
     )
 
-    interface_style_digest = top_obj_res.digest
+    top_level_interface_style_digest = top_obj_res.digest
 
-    assert pythonic_digest == interface_style_digest
+    assert top_level_pythonic_digest == top_level_interface_style_digest
 
 
 def test_schema_validation(client):
     # Test that we can't create an object with the wrong schema
-    with pytest.raises():
+    with pytest.raises(ValidationError):
         client.server.obj_create(
             tsi.ObjCreateReq.model_validate(
                 {
                     "obj": {
                         "project_id": client._project_id(),
                         "object_id": "nested_obj",
+                        # Incorrect schema, should raise!
                         "val": {"a": 2},
+                        "set_base_object_class": "TestOnlyNestedBaseObject",
+                    }
+                }
+            )
+        )
+
+    # Correct schema, should work
+    client.server.obj_create(
+        tsi.ObjCreateReq.model_validate(
+            {
+                "obj": {
+                    "project_id": client._project_id(),
+                    "object_id": "nested_obj",
+                    "val": {
+                        "b": 2,
+                        "_class_name": "TestOnlyNestedBaseObject",
+                        "_bases": ["BaseObject", "BaseModel"],
+                    },
+                    "set_base_object_class": "TestOnlyNestedBaseObject",
+                }
+            }
+        )
+    )
+
+    with pytest.raises(ValueError):
+        # Mismatching base object class, should raise
+        client.server.obj_create(
+            tsi.ObjCreateReq.model_validate(
+                {
+                    "obj": {
+                        "project_id": client._project_id(),
+                        "object_id": "nested_obj",
+                        "val": {
+                            "b": 2,
+                            "_class_name": "TestOnlyNestedBaseObject",
+                            "_bases": ["BaseObject", "BaseModel"],
+                        },
+                        "set_base_object_class": "TestOnlyExample",
                     }
                 }
             )
diff --git a/weave/trace_server/base_object_class_util.py b/weave/trace_server/base_object_class_util.py
index 50aea59084d..5fdce65709b 100644
--- a/weave/trace_server/base_object_class_util.py
+++ b/weave/trace_server/base_object_class_util.py
@@ -1,4 +1,10 @@
-from typing import Any, Optional
+from typing import Any, Optional, Tuple
+
+from pydantic import BaseModel
+
+from weave.trace_server.interface.base_object_classes.base_object_registry import (
+    BASE_OBJECT_REGISTRY,
+)
 
 base_object_class_names = ["BaseObject", "Object"]
 
@@ -15,3 +21,63 @@ def get_base_object_class(val: Any) -> Optional[str]:
                             elif "_class_name" in val:
                                 return val["_class_name"]
     return None
+
+
+def process_incoming_object(
+    dict_val: dict, req_base_object_class: Optional[str] = None
+) -> Tuple[dict, Optional[str]]:
+    dict_val = dict_val.copy()
+    val_base_object_class = get_base_object_class(dict_val)
+
+    if (
+        val_base_object_class != None
+        and req_base_object_class != None
+        and val_base_object_class != req_base_object_class
+    ):
+        raise ValueError(
+            f"set_base_object_class must match base_object_class: {val_base_object_class} != {req_base_object_class}"
+        )
+
+    if val_base_object_class is not None:
+        # In this case, we simply validate if the match is found
+        if base_object_class_type := BASE_OBJECT_REGISTRY.get(val_base_object_class):
+            base_object_class_type.model_validate(dict_val)
+    elif req_base_object_class is not None:
+        # In this case, we require that the base object class is registered
+        if base_object_class_type := BASE_OBJECT_REGISTRY.get(req_base_object_class):
+            dict_val = dump_base_object(base_object_class_type.model_validate(dict_val))
+        else:
+            raise ValueError(f"Unknown base object class: {req_base_object_class}")
+
+    base_object_class = val_base_object_class or req_base_object_class
+
+    return dict_val, base_object_class
+
+
+# Server-side version of `pydantic_object_record`
+def dump_base_object(val: BaseModel) -> dict:
+    cls = val.__class__
+    cls_name = val.__class__.__name__
+    bases = [c.__name__ for c in cls.mro()[1:-1]]
+
+    dump = {}
+    # Order matters here due to the way we calculate the digest!
+    # This matches the client
+    dump["_type"] = cls_name
+    for k in val.model_fields:
+        dump[k] = _general_dump(getattr(val, k))
+    # yes, this is done twice, to match the client
+    dump["_class_name"] = cls_name
+    dump["_bases"] = bases
+    return dump
+
+
+def _general_dump(val: Any) -> Any:
+    if isinstance(val, BaseModel):
+        return dump_base_object(val)
+    elif isinstance(val, dict):
+        return {k: _general_dump(v) for k, v in val.items()}
+    elif isinstance(val, list):
+        return [_general_dump(v) for v in val]
+    else:
+        return val
diff --git a/weave/trace_server/clickhouse_trace_server_batched.py b/weave/trace_server/clickhouse_trace_server_batched.py
index 9ed9533787e..47f5380fa43 100644
--- a/weave/trace_server/clickhouse_trace_server_batched.py
+++ b/weave/trace_server/clickhouse_trace_server_batched.py
@@ -53,7 +53,9 @@
 from weave.trace_server import environment as wf_env
 from weave.trace_server import refs_internal as ri
 from weave.trace_server import trace_server_interface as tsi
-from weave.trace_server.base_object_class_util import get_base_object_class
+from weave.trace_server.base_object_class_util import (
+    process_incoming_object,
+)
 from weave.trace_server.calls_query_builder import (
     CallsQuery,
     HardCodedFilter,
@@ -600,16 +602,19 @@ def ops_query(self, req: tsi.OpQueryReq) -> tsi.OpQueryRes:
         return tsi.OpQueryRes(op_objs=objs)
 
     def obj_create(self, req: tsi.ObjCreateReq) -> tsi.ObjCreateRes:
-        json_val = json.dumps(req.obj.val)
+        dict_val, base_object_class = process_incoming_object(
+            req.obj.val, req.obj.set_base_object_class
+        )
+
+        json_val = json.dumps(dict_val)
         digest = str_digest(json_val)
 
-        req_obj = req.obj
         ch_obj = ObjCHInsertable(
-            project_id=req_obj.project_id,
-            object_id=req_obj.object_id,
-            kind=get_kind(req.obj.val),
-            base_object_class=get_base_object_class(req.obj.val),
-            refs=extract_refs_from_values(req.obj.val),
+            project_id=req.obj.project_id,
+            object_id=req.obj.object_id,
+            kind=get_kind(dict_val),
+            base_object_class=base_object_class,
+            refs=extract_refs_from_values(dict_val),
             val_dump=json_val,
             digest=digest,
         )
diff --git a/weave/trace_server/sqlite_trace_server.py b/weave/trace_server/sqlite_trace_server.py
index 51b51e7ca64..78221df2d2f 100644
--- a/weave/trace_server/sqlite_trace_server.py
+++ b/weave/trace_server/sqlite_trace_server.py
@@ -13,7 +13,9 @@
 
 from weave.trace_server import refs_internal as ri
 from weave.trace_server import trace_server_interface as tsi
-from weave.trace_server.base_object_class_util import get_base_object_class
+from weave.trace_server.base_object_class_util import (
+    process_incoming_object,
+)
 from weave.trace_server.emoji_util import detone_emojis
 from weave.trace_server.errors import InvalidRequest
 from weave.trace_server.feedback import (
@@ -609,25 +611,28 @@ def ops_query(self, req: tsi.OpQueryReq) -> tsi.OpQueryRes:
 
     def obj_create(self, req: tsi.ObjCreateReq) -> tsi.ObjCreateRes:
         conn, cursor = get_conn_cursor(self.db_path)
-        json_val = json.dumps(req.obj.val)
+
+        dict_val, base_object_class = process_incoming_object(
+            req.obj.val, req.obj.set_base_object_class
+        )
+        json_val = json.dumps(dict_val)
         digest = str_digest(json_val)
 
         # Validate
         object_id_validator(req.obj.object_id)
 
-        req_obj = req.obj
         # TODO: version index isn't right here, what if we delete stuff?
         with self.lock:
             cursor.execute("BEGIN TRANSACTION")
             # Mark all existing objects with such id as not latest
             cursor.execute(
                 """UPDATE objects SET is_latest = 0 WHERE project_id = ? AND object_id = ?""",
-                (req_obj.project_id, req_obj.object_id),
+                (req.obj.project_id, req.obj.object_id),
             )
             # first get version count
             cursor.execute(
                 """SELECT COUNT(*) FROM objects WHERE project_id = ? AND object_id = ?""",
-                (req_obj.project_id, req_obj.object_id),
+                (req.obj.project_id, req.obj.object_id),
             )
             version_index = cursor.fetchone()[0]
 
@@ -645,11 +650,11 @@ def obj_create(self, req: tsi.ObjCreateReq) -> tsi.ObjCreateRes:
                     is_latest
                     ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                 (
-                    req_obj.project_id,
-                    req_obj.object_id,
+                    req.obj.project_id,
+                    req.obj.object_id,
                     datetime.datetime.now().isoformat(),
-                    get_kind(req_obj.val),
-                    get_base_object_class(req_obj.val),
+                    get_kind(dict_val),
+                    base_object_class,
                     json.dumps([]),
                     json_val,
                     digest,
diff --git a/weave/trace_server/trace_server_interface.py b/weave/trace_server/trace_server_interface.py
index abdfeae38ac..af8778dc1d3 100644
--- a/weave/trace_server/trace_server_interface.py
+++ b/weave/trace_server/trace_server_interface.py
@@ -190,6 +190,7 @@ class ObjSchemaForInsert(BaseModel):
     project_id: str
     object_id: str
     val: Any
+    set_base_object_class: Optional[str] = None
 
 
 class TableSchemaForInsert(BaseModel):

From ec3959f8237f97b16122151f68e70bf4ca987c5a Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 02:50:22 -0600
Subject: [PATCH 134/169] Attempted fix

---
 .../pages/wfReactInterface/baseObjectClassQuery.ts        | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index 3d2f4513e45..79a3d03e86a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -17,7 +17,7 @@ import {
 import {Loadable} from './wfDataModelHooksInterface';
 
 // TODO: This should be generated from the registry!
-const baseObjectClassRegistry = {
+const baseObjectClassRegistry: Record<string, z.ZodType> = {
   TestOnlyExample: TestOnlyExampleSchema,
   TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
 };
@@ -82,9 +82,9 @@ const getBaseObjectInstances = async <
   return objects.objs
     .map(obj => ({obj, parsed: knownObjectClass.safeParse(obj.val)}))
     .filter(({parsed}) => parsed.success)
-    .map(({obj, parsed}) => ({...obj, val: parsed.data!})) as Array<
-    TraceObjSchema<T, C>
-  >;
+    .map(
+      ({obj, parsed}) => ({...obj, val: parsed.data} as TraceObjSchema<T, C>)
+    );
 };
 
 export const useCreateBaseObjectInstance = <

From 00516d0d6cf636c7d585696900ce4f25bfaa4dfa Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 02:55:03 -0600
Subject: [PATCH 135/169] Attempted fix

---
 .../Browse3/pages/wfReactInterface/baseObjectClassQuery.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index 79a3d03e86a..b1f7b4b148f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -64,7 +64,8 @@ const getBaseObjectInstances = async <
   baseObjectClassName: C,
   req: TraceObjQueryReq
 ): Promise<Array<TraceObjSchema<T, C>>> => {
-  const knownObjectClass = baseObjectClassRegistry[baseObjectClassName];
+  const knownObjectClass: z.ZodType<T> =
+    baseObjectClassRegistry[baseObjectClassName];
   if (!knownObjectClass) {
     console.warn(`Unknown object class: ${baseObjectClassName}`);
     return [];
@@ -79,9 +80,13 @@ const getBaseObjectInstances = async <
 
   const objects = await objectPromise;
 
+  // We would expect that this  filtering does not filter anything
+  // out because the backend enforces the base object class, but this
+  // is here as a sanity check.
   return objects.objs
     .map(obj => ({obj, parsed: knownObjectClass.safeParse(obj.val)}))
     .filter(({parsed}) => parsed.success)
+    .filter(({obj}) => obj.base_object_class === baseObjectClassName)
     .map(
       ({obj, parsed}) => ({...obj, val: parsed.data} as TraceObjSchema<T, C>)
     );

From a81daed63299064508ebebeea5d7e2164fb29300 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 03:00:06 -0600
Subject: [PATCH 136/169] clean

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d152017c58b..6c1ed017c0f 100644
--- a/Makefile
+++ b/Makefile
@@ -18,4 +18,4 @@ prepare-release: docs build
 
 synchronize-base-object-schemas:
 	cd weave && make generate_base_object_schemas && \
-	cd ../weave-js && yarn generate-schemas
\ No newline at end of file
+	cd ../weave-js && yarn generate-schemas

From 45c1756e553ca54f2afa1b4f2abb868f8302bbe5 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 03:08:48 -0600
Subject: [PATCH 137/169] clean

---
 weave/trace/weave_client.py                                    | 2 +-
 .../interface/base_object_classes/test_only_example.py         | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/weave/trace/weave_client.py b/weave/trace/weave_client.py
index 656bf1c360f..7f4d1d88ef0 100644
--- a/weave/trace/weave_client.py
+++ b/weave/trace/weave_client.py
@@ -1232,7 +1232,7 @@ def _save_nested_objects(self, obj: Any, name: Optional[str] = None) -> Any:
         # Case 1: Object:
         # Here we recurse into each of the properties of the object
         # and save them, and then save the object itself.
-        if isinstance(obj, Object):  # TODO: add the generic object here
+        if isinstance(obj, Object):
             obj_rec = pydantic_object_record(obj)
             for v in obj_rec.__dict__.values():
                 self._save_nested_objects(v)
diff --git a/weave/trace_server/interface/base_object_classes/test_only_example.py b/weave/trace_server/interface/base_object_classes/test_only_example.py
index b93af28790e..87a48907eca 100644
--- a/weave/trace_server/interface/base_object_classes/test_only_example.py
+++ b/weave/trace_server/interface/base_object_classes/test_only_example.py
@@ -17,6 +17,9 @@ class TestOnlyExample(base_object_def.BaseObject):
     # Important: `RefStr` is just an alias for `str`. When defining `BaseObject`s, we
     # should never have a property point to another `BaseObject`. This is because each
     # base object is stored in the database and should be treated like a foreign key.
+    #
+    # It would be nice to have a way to ensure that no `BaseObject` has any `BaseObject`
+    # properties.
     nested_base_object: base_object_def.RefStr
 
 

From 5e4457d815f7c343f98a1cd481ee0942bdb2f4c5 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 03:32:43 -0600
Subject: [PATCH 138/169] fixed bug

---
 weave/trace_server/base_object_class_util.py          | 11 +++++++++--
 weave/trace_server/clickhouse_trace_server_batched.py |  8 ++++----
 weave/trace_server/sqlite_trace_server.py             |  6 +++---
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/weave/trace_server/base_object_class_util.py b/weave/trace_server/base_object_class_util.py
index 5fdce65709b..698b422aad1 100644
--- a/weave/trace_server/base_object_class_util.py
+++ b/weave/trace_server/base_object_class_util.py
@@ -24,9 +24,16 @@ def get_base_object_class(val: Any) -> Optional[str]:
 
 
 def process_incoming_object(
-    dict_val: dict, req_base_object_class: Optional[str] = None
+    val: Any, req_base_object_class: Optional[str] = None
 ) -> Tuple[dict, Optional[str]]:
-    dict_val = dict_val.copy()
+    if not isinstance(val, dict):
+        if req_base_object_class is not None:
+            raise ValueError(
+                "set_base_object_class cannot be provided for non-dict objects"
+            )
+        return val, None
+
+    dict_val = val.copy()
     val_base_object_class = get_base_object_class(dict_val)
 
     if (
diff --git a/weave/trace_server/clickhouse_trace_server_batched.py b/weave/trace_server/clickhouse_trace_server_batched.py
index 9c880056707..cb032104f6a 100644
--- a/weave/trace_server/clickhouse_trace_server_batched.py
+++ b/weave/trace_server/clickhouse_trace_server_batched.py
@@ -599,19 +599,19 @@ def ops_query(self, req: tsi.OpQueryReq) -> tsi.OpQueryRes:
         return tsi.OpQueryRes(op_objs=objs)
 
     def obj_create(self, req: tsi.ObjCreateReq) -> tsi.ObjCreateRes:
-        dict_val, base_object_class = process_incoming_object(
+        val, base_object_class = process_incoming_object(
             req.obj.val, req.obj.set_base_object_class
         )
 
-        json_val = json.dumps(dict_val)
+        json_val = json.dumps(val)
         digest = str_digest(json_val)
 
         ch_obj = ObjCHInsertable(
             project_id=req.obj.project_id,
             object_id=req.obj.object_id,
-            kind=get_kind(dict_val),
+            kind=get_kind(val),
             base_object_class=base_object_class,
-            refs=extract_refs_from_values(dict_val),
+            refs=extract_refs_from_values(val),
             val_dump=json_val,
             digest=digest,
         )
diff --git a/weave/trace_server/sqlite_trace_server.py b/weave/trace_server/sqlite_trace_server.py
index 78221df2d2f..1fd50cf1cb2 100644
--- a/weave/trace_server/sqlite_trace_server.py
+++ b/weave/trace_server/sqlite_trace_server.py
@@ -612,10 +612,10 @@ def ops_query(self, req: tsi.OpQueryReq) -> tsi.OpQueryRes:
     def obj_create(self, req: tsi.ObjCreateReq) -> tsi.ObjCreateRes:
         conn, cursor = get_conn_cursor(self.db_path)
 
-        dict_val, base_object_class = process_incoming_object(
+        val, base_object_class = process_incoming_object(
             req.obj.val, req.obj.set_base_object_class
         )
-        json_val = json.dumps(dict_val)
+        json_val = json.dumps(val)
         digest = str_digest(json_val)
 
         # Validate
@@ -653,7 +653,7 @@ def obj_create(self, req: tsi.ObjCreateReq) -> tsi.ObjCreateRes:
                     req.obj.project_id,
                     req.obj.object_id,
                     datetime.datetime.now().isoformat(),
-                    get_kind(dict_val),
+                    get_kind(val),
                     base_object_class,
                     json.dumps([]),
                     json_val,

From 10f7e1d5310b5d929376d0c289f512d9799c15d9 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 03:52:08 -0600
Subject: [PATCH 139/169] maybe fix

---
 weave/trace/serialize.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/weave/trace/serialize.py b/weave/trace/serialize.py
index c37ae03b7a7..09df639e91a 100644
--- a/weave/trace/serialize.py
+++ b/weave/trace/serialize.py
@@ -228,7 +228,11 @@ def from_json(obj: Any, project_id: str, server: TraceServerInterface) -> Any:
             return custom_objs.decode_custom_obj(
                 obj["weave_type"], files, obj.get("load_op")
             )
-        elif baseObject := BASE_OBJECT_REGISTRY.get(val_type):
+        elif (
+            isinstance(val_type, str)
+            and obj.get("_class_name") == val_type
+            and (baseObject := BASE_OBJECT_REGISTRY.get(val_type))
+        ):
             return baseObject.model_validate(obj)
         else:
             return ObjectRecord(

From 166dbdb0fd4b748e4c2cd8fc43544d1f3bff66b1 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 04:10:06 -0600
Subject: [PATCH 140/169] maybe fix

---
 tests/trace/test_base_object_classes.py               | 11 +++++++----
 .../base_object_classes/test_only_example.py          |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/trace/test_base_object_classes.py b/tests/trace/test_base_object_classes.py
index 2d52fd1dc27..a264941f7b0 100644
--- a/tests/trace/test_base_object_classes.py
+++ b/tests/trace/test_base_object_classes.py
@@ -22,6 +22,9 @@
 from weave.trace.refs import ObjectRef
 from weave.trace.weave_client import WeaveClient
 from weave.trace_server import trace_server_interface as tsi
+from weave.trace_server.interface.base_object_classes.test_only_example import (
+    TestOnlyNestedBaseModel,
+)
 
 
 def with_base_object_class_annotations(
@@ -50,7 +53,7 @@ def test_pythonic_creation(client: WeaveClient):
     nested_obj = base_objects.TestOnlyNestedBaseObject(b=3)
     top_obj = base_objects.TestOnlyExample(
         primitive=1,
-        nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
+        nested_base_model=TestOnlyNestedBaseModel(a=2),
         nested_base_object=weave.publish(nested_obj).uri(),
     )
     ref = weave.publish(top_obj)
@@ -151,7 +154,7 @@ def test_interface_creation(client):
     top_level_obj_id = "TestOnlyExample"
     top_obj = base_objects.TestOnlyExample(
         primitive=1,
-        nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
+        nested_base_model=TestOnlyNestedBaseModel(a=2),
         nested_base_object=nested_obj_ref.uri(),
     )
     top_obj_res = client.server.obj_create(
@@ -251,7 +254,7 @@ def test_digest_equality(client):
     nested_ref = weave.publish(nested_obj)
     top_obj = base_objects.TestOnlyExample(
         primitive=1,
-        nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
+        nested_base_model=TestOnlyNestedBaseModel(a=2),
         nested_base_object=nested_ref.uri(),
     )
     ref = weave.publish(top_obj)
@@ -287,7 +290,7 @@ def test_digest_equality(client):
     top_level_obj_id = "TestOnlyExample"
     top_obj = base_objects.TestOnlyExample(
         primitive=1,
-        nested_base_model=base_objects.TestOnlyNestedBaseModel(a=2),
+        nested_base_model=TestOnlyNestedBaseModel(a=2),
         nested_base_object=nested_obj_ref.uri(),
     )
     top_obj_res = client.server.obj_create(
diff --git a/weave/trace_server/interface/base_object_classes/test_only_example.py b/weave/trace_server/interface/base_object_classes/test_only_example.py
index 87a48907eca..49afb93bf89 100644
--- a/weave/trace_server/interface/base_object_classes/test_only_example.py
+++ b/weave/trace_server/interface/base_object_classes/test_only_example.py
@@ -23,4 +23,4 @@ class TestOnlyExample(base_object_def.BaseObject):
     nested_base_object: base_object_def.RefStr
 
 
-__all__ = ["TestOnlyExample", "TestOnlyNestedBaseObject", "TestOnlyNestedBaseModel"]
+__all__ = ["TestOnlyExample", "TestOnlyNestedBaseObject"]

From 185f247a89dc659b624ba59d182ad57321bba134 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 11:38:35 -0600
Subject: [PATCH 141/169] Added diagram

---
 dev_docs/BaseObjectClasses.md                 | 198 +++++++++++++-----
 .../base_object_registry.py                   |  13 +-
 2 files changed, 160 insertions(+), 51 deletions(-)

diff --git a/dev_docs/BaseObjectClasses.md b/dev_docs/BaseObjectClasses.md
index d77799d98f1..7dbf2ba0673 100644
--- a/dev_docs/BaseObjectClasses.md
+++ b/dev_docs/BaseObjectClasses.md
@@ -42,6 +42,7 @@ This will result in an on-disk payload that looks like:
 {
     "model_name": "my_model",
     "model_version": "1.0",
+    "_type": "ModelConfig",
     "_class_name": "ModelConfig",
     "_bases": ["Object", "BaseModel"]
 }
@@ -53,63 +54,166 @@ Effectively, this is like creating a virtual table for that class.
 **Terminology**: We use the term "weave Object" (capital "O") to refer to instances of classes that subclass `weave.Object`.
 
 **Technical note**: the "base_object_class" is the first subtype of "Object", not the _class_name. 
-For example, let's say the class heirarchy is:
+For example, let's say the class hierarchy is:
 * `A -> Object -> BaseModel`, then the `base_object_class` filter will be "A".
 * `B -> A -> Object -> BaseModel`, then the `base_object_class` filter will still be "A"!
 
 Finally, the Weave library itself utilizes this mechanism for common objects like `Model`, `Dataset`, `Evaluation`, etc...
 This allows the user to subclass these objects to add additional metadata or functionality, while categorizing them in the same virtual table.
 
-## The need for validation
-Objects, like people, sometimes need validation. Many of our Objects (eg. `Model`) don't really need validation as they are completely free-form and up to the user to definie the properties.
-However, there is an increasing need to have a well-defined schema for certain configuration objects - where those objects are tightly defined by Weave, not the user.
-In such cases, it is important to have a standard, validated schema that can be shared across the python sdk, the http api, the DB, the frontend UI, and the typescript sdk.
+## Validated Base Objects
 
-### Motivating example:
-Let's consider that we are defining a new concept "Leaderboard". We want to store this as a configuration object in Weave.
-Let's say that it's pydantic representation is:
+While many Weave Objects are free-form and user-defined, there is often a need for well-defined schemas for configuration objects that are tightly defined by Weave itself. The BaseObject system provides a way to define these schemas once and use them consistently across the entire stack.
+
+### Key Features
+
+1. **Single Source of Truth**: Define your schema once using Pydantic models
+2. **Full Stack Integration**: The schema is used for:
+   - Python SDK validation
+   - Server-side HTTP API validation
+   - Frontend UI validation with generated TypeScript types
+   - Future: OpenAPI schema generation
+   - Future: TypeScript SDK type generation
+
+### Usage Example
+
+Here's how to define and use a validated base object:
+
+1. **Define your schema** (in `weave/trace_server/interface/base_object_classes/your_schema.py`):
 
 ```python
-class LeaderboardColumn(BaseModel):
-    name: str
-    description: str
-    evaluation_obj: str # <- reference to an Evaluation object
-    metric_name: str
-    lower_is_better: bool
+from pydantic import BaseModel
+from weave.trace_server.interface.base_object_classes import base_object_def
+
+class NestedConfig(BaseModel):
+    setting_a: int
 
-class Leaderboard(BaseModel):
+class MyConfig(base_object_def.BaseObject):
     name: str
-    description: str
-    columns: List[LeaderboardColumn]
+    nested: NestedConfig
+    reference: base_object_def.RefStr
+
+__all__ = ["MyConfig"]
+```
+
+2. **Use in Python**:
+```python
+# Publishing
+ref = weave.publish(MyConfig(...))
+
+# Fetching (maintains type)
+config = ref.get()
+assert isinstance(config, MyConfig)
+```
+
+3. **Use via HTTP API**:
+```bash
+# Creating
+curl -X POST 'https://trace.wandb.ai/obj/create' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "obj": {
+      "project_id": "user/project",
+      "object_id": "my_config",
+      "val": {...},
+      "set_base_object_class": "MyConfig"
+    }
+  }'
+
+# Querying
+curl -X POST 'https://trace.wandb.ai/objs/query' \
+  -d '{
+    "project_id": "user/project",
+    "filter": {
+      "base_object_classes": ["MyConfig"]
+    }
+  }'
 ```
 
-We want to be able to store & query `Leaderboards` efficiently and ensure they always have a well-defined schema. Therefore we need:
-1. An ability to validate the schema at insertion time
-2. An ability to create such objects view the REST API
-3. An ability to construct, publish, & query these objects:
-    * via the HTTP API (future: would be nice to have these types inside the openapi schema)
-    * from Python SDK (need a `weave.Object` subclass)
-    * from Typescript SDK (TBD - not yet supported, but need the ability to generate the correct utilities)
-    * from Frontend UI (need a generated Zod schema & hooks to validate the data)
-        * Ideally, with static type-safety.
-
-And all of the above should be generated from a single source of truth: the pydantic schema.
-
-### Solution
-1. We define a set of "validated base object classes" in a common location.
-2. We use that:
-    1. At the DB layer to validate insertion schemas
-    2. Can be published from the python sdk
-       * Note: figuring out the interplay with `weave.Object` is tricky here. There are a few issues:
-            1. We don't want to depend on the weave sdk inside the common interface, meaning the class heirarchy is inherently divergent
-            2. Without `weave.Object`:
-                * Our python serialization layer's `is_instance` checks will not maintain behavior
-                * (Seems fine?) No ref tracing or nested deserialization. Actually, i think we do want to avoid this.
-                * (Seems fine?) No "handle_relocatable_object" behavior
-            3. We still want to have the same bases array in the final payload
-            - I think all of this can be resolved by extended the base_object_class to also look for `BaseObject` (objects that are pure data schema: only fields)
-    3. Can be used in the frontend UI layer
-        * via generated Zod types & hooks
-    4. Future: Exposed in the OpenAPI schema
-    5. Future: Generates Typescript SDK types
-    
+4. **Use in React**:
+```typescript
+// Read with type safety
+const result = useBaseObjectInstances("MyConfig", ...);
+
+// Write with validation
+const createFn = useCreateBaseObjectInstance("MyConfig");
+createFn({...}); // TypeScript enforced schema
+```
+
+### Keeping Frontend Types in Sync
+
+Run `make synchronize-base-object-schemas` to ensure the frontend TypeScript types are up to date with your Pydantic schemas.
+
+### Implementation Notes
+
+- Base objects are pure data schemas (fields only)
+- The system is designed to work independently of the weave SDK to maintain clean separation of concerns
+- Server-side validation ensures data integrity
+- Client-side validation (both Python and TypeScript) provides early feedback
+- Generated TypeScript types ensure type safety in the frontend
+
+### Architecture Flow
+
+1. Define your schema in a python file in the `weave/trace_server/interface/base_object_classes/test_only_example.py` directory. See `weave/trace_server/interface/base_object_classes/test_only_example.py` as an example.
+2. Make sure to register your schemas in `weave/trace_server/interface/base_object_classes/base_object_registry.py` by calling `register_base_object`.
+3. Run `make synchronize-base-object-schemas` to generate the frontend types.
+    * The first step (`make generate_base_object_schemas`) will run `weave/scripts/generate_base_object_schemas.py` to generate a JSON schema in `weave/scripts/generated_base_object_class_schemas.json`.
+    * The second step (yarn `generate-schemas`) will read this file and use it to generate the frontend types located in `weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts`.
+        * Note, you need to run prettier to fix the output. These files should be checked in.
+4. Now, each use case uses different parts:
+    1. `Python Writing`. Users can directly import these classes and use them as normal Pydantic models, which get published with `weave.publish`. The python client correct builds the requisite payload.
+    2. `Python Reading`. Users can `weave.ref().get()` and the weave python SDK will return the instance with the correct type. Note: we do some special handling such that the returned object is not a WeaveObject, but literally the exact pydantic class.
+    3. `HTTP Writing`. In cases where the client/user does not want to add the special type information, users can publish base objects by setting the `set_base_object_class` setting on `POST obj/create` to the name of the class. The weave server will validate the object against the schema, update the metadata fields, and store the object.
+    4. `HTTP Reading`. When querying for objects, the server will return the object with the correct type if the `base_object_class` metadata field is set.
+    5. `Frontend`. The frontend will read the zod schema from `weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts` and use that to provide compile time type safety when using `useBaseObjectInstances` and runtime type safety when using `useCreateBaseObjectInstance`.
+* Note: it is critical that all techniques produce the same digest for the same data - which is tested in the tests. This way versions are not thrashed by different clients/users.
+
+```mermaid
+graph TD
+    subgraph Schema Definition
+        F["weave/trace_server/interface/<br>base_object_classes/your_schema.py"] --> |defines| P[Pydantic BaseObject]
+        P --> |register_base_object| R["base_object_registry.py"]
+    end
+
+    subgraph Schema Generation
+        M["make synchronize-base-object-schemas"] --> G["make generate_base_object_schemas"]
+        G --> |runs| S["weave/scripts/<br>generate_base_object_schemas.py"]
+        R --> |import registered classes| S
+        S --> |generates| J["generated_base_object_class_schemas.json"]
+        M --> |yarn generate-schemas| Z["generatedBaseObjectClasses.zod.ts"]
+        J --> Z
+    end
+
+    subgraph "Trace Server"
+        subgraph "HTTP API"
+            R --> |validates using| HW["POST obj/create<br>set_base_object_class"]
+            HW --> DB[(Weave Object Store)]
+            HR["POST objs/query<br>base_object_classes"] --> |Filters base_object_class| DB
+        end
+    end
+
+    subgraph "Python SDK"
+        PW[Client Code] --> |import & publish| W[weave.publish]
+        W --> |store| HW
+        R --> |validates using| W
+        PR["weave ref get()"] --> |queries| HR
+        R --> |deserializes using| PR
+    end
+
+    subgraph "Frontend"
+        Z --> |import| UBI["useBaseObjectInstances"]
+        Z --> |import| UCI["useCreateBaseObjectInstance"]
+        UBI --> |Filters base_object_class| HR
+        UCI --> |set_base_object_class| HW
+        UI[React UI] --> UBI
+        UI --> UCI
+    end
+
+    style F fill:#f9f,stroke:#333,stroke-width:2px
+    style P fill:#f9f,stroke:#333,stroke-width:2px
+    style R fill:#bbf,stroke:#333,stroke-width:2px
+    style DB fill:#dfd,stroke:#333,stroke-width:2px
+    style J fill:#ffd,stroke:#333,stroke-width:2px
+    style Z fill:#ffd,stroke:#333,stroke-width:2px
+    style M fill:#faa,stroke:#333,stroke-width:4px
+```
diff --git a/weave/trace_server/interface/base_object_classes/base_object_registry.py b/weave/trace_server/interface/base_object_classes/base_object_registry.py
index 43191f43671..e2f891032af 100644
--- a/weave/trace_server/interface/base_object_classes/base_object_registry.py
+++ b/weave/trace_server/interface/base_object_classes/base_object_registry.py
@@ -5,10 +5,15 @@
 from weave.trace_server.interface.base_object_classes.base_object_def import BaseObject
 from weave.trace_server.interface.base_object_classes.test_only_example import *
 
-BASE_OBJECT_REGISTRY: Dict[str, Type[BaseObject]] = {
-    "TestOnlyExample": TestOnlyExample,
-    "TestOnlyNestedBaseObject": TestOnlyNestedBaseObject,
-}
+BASE_OBJECT_REGISTRY: Dict[str, Type[BaseObject]] = {}
+
+
+def register_base_object(cls: Type[BaseObject]) -> None:
+    BASE_OBJECT_REGISTRY[cls.__name__] = cls
+
+
+register_base_object(TestOnlyExample)
+register_base_object(TestOnlyNestedBaseObject)
 
 
 # TODO: Remove this helper

From e44d6347ac151972ea366a35fea861c6fc02b8c1 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 11:57:09 -0600
Subject: [PATCH 142/169] Removed first hack

---
 dev_docs/BaseObjectClasses.md                 |  1 -
 weave-js/package.json                         |  2 +-
 weave/scripts/generate_base_object_schemas.py | 22 +++++++++++++------
 .../base_object_registry.py                   | 14 +++++-------
 4 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/dev_docs/BaseObjectClasses.md b/dev_docs/BaseObjectClasses.md
index 7dbf2ba0673..fe3024c5a20 100644
--- a/dev_docs/BaseObjectClasses.md
+++ b/dev_docs/BaseObjectClasses.md
@@ -159,7 +159,6 @@ Run `make synchronize-base-object-schemas` to ensure the frontend TypeScript typ
 3. Run `make synchronize-base-object-schemas` to generate the frontend types.
     * The first step (`make generate_base_object_schemas`) will run `weave/scripts/generate_base_object_schemas.py` to generate a JSON schema in `weave/scripts/generated_base_object_class_schemas.json`.
     * The second step (yarn `generate-schemas`) will read this file and use it to generate the frontend types located in `weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts`.
-        * Note, you need to run prettier to fix the output. These files should be checked in.
 4. Now, each use case uses different parts:
     1. `Python Writing`. Users can directly import these classes and use them as normal Pydantic models, which get published with `weave.publish`. The python client correct builds the requisite payload.
     2. `Python Reading`. Users can `weave.ref().get()` and the weave python SDK will return the instance with the correct type. Note: we do some special handling such that the returned object is not a WeaveObject, but literally the exact pydantic class.
diff --git a/weave-js/package.json b/weave-js/package.json
index 6dc7da04766..2e3fdeaea92 100644
--- a/weave-js/package.json
+++ b/weave-js/package.json
@@ -21,7 +21,7 @@
     "prettier-fix": "prettier --loglevel warn --config .prettierrc --write \"src/**/*.ts\" \"src/**/*.tsx\"",
     "lint": "yarn eslint & yarn tslint & yarn prettier & wait",
     "lint-fix": "yarn eslint-fix & yarn tslint-fix & yarn prettier-fix & wait",
-    "generate-schemas": "yarn quicktype -s schema  ../weave/scripts/generated_base_object_class_schemas.json -o ./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts --lang typescript-zod"
+    "generate-schemas": "yarn quicktype -s schema  ../weave/scripts/generated_base_object_class_schemas.json -o ./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts --lang typescript-zod && prettier --write ./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts"
   },
   "dependencies": {
     "@apollo/client": "^3.8.4",
diff --git a/weave/scripts/generate_base_object_schemas.py b/weave/scripts/generate_base_object_schemas.py
index 979565a09e1..fc6ae2a5d1f 100644
--- a/weave/scripts/generate_base_object_schemas.py
+++ b/weave/scripts/generate_base_object_schemas.py
@@ -1,9 +1,10 @@
 import json
 from pathlib import Path
 
+from pydantic import create_model
+
 from weave.trace_server.interface.base_object_classes.base_object_registry import (
     BASE_OBJECT_REGISTRY,
-    CompositeBaseObject,
 )
 
 OUTPUT_PATH = Path(__file__).parent / "generated_base_object_class_schemas.json"
@@ -11,17 +12,24 @@
 
 def generate_schemas() -> None:
     """
-    Generate JSON schemas for all registered base objects in REGISTRY.
-    Creates a top-level union type of all registered objects and writes the schemas
-    to a file named 'base_object_schemas.json'.
+    Generate JSON schemas for all registered base objects in BASE_OBJECT_REGISTRY.
+    Creates a top-level schema that includes all registered objects and writes it
+    to 'generated_base_object_class_schemas.json'.
     """
-    top_level_schema = CompositeBaseObject.model_json_schema(mode="validation")
+    # Dynamically create a parent model with all registered objects as properties
+    CompositeModel = create_model(
+        "CompositeBaseObject",
+        **{name: (cls, ...) for name, cls in BASE_OBJECT_REGISTRY.items()},
+    )
+
+    # Generate the schema using the composite model
+    top_level_schema = CompositeModel.model_json_schema(mode="validation")
 
-    # Write schemas to file
+    # Write schema to file
     with OUTPUT_PATH.open("w") as f:
         json.dump(top_level_schema, f, indent=2)
 
-    print(f"Generated union schema for {len(BASE_OBJECT_REGISTRY)} objects")
+    print(f"Generated schema for {len(BASE_OBJECT_REGISTRY)} objects")
     print(f"Wrote schema to {OUTPUT_PATH.absolute()}")
 
 
diff --git a/weave/trace_server/interface/base_object_classes/base_object_registry.py b/weave/trace_server/interface/base_object_classes/base_object_registry.py
index e2f891032af..8941dfa75ad 100644
--- a/weave/trace_server/interface/base_object_classes/base_object_registry.py
+++ b/weave/trace_server/interface/base_object_classes/base_object_registry.py
@@ -1,7 +1,5 @@
 from typing import Dict, Type
 
-from pydantic import BaseModel
-
 from weave.trace_server.interface.base_object_classes.base_object_def import BaseObject
 from weave.trace_server.interface.base_object_classes.test_only_example import *
 
@@ -9,14 +7,14 @@
 
 
 def register_base_object(cls: Type[BaseObject]) -> None:
+    """
+    Register a BaseObject class in the global registry.
+
+    Args:
+        cls: The BaseObject class to register
+    """
     BASE_OBJECT_REGISTRY[cls.__name__] = cls
 
 
 register_base_object(TestOnlyExample)
 register_base_object(TestOnlyNestedBaseObject)
-
-
-# TODO: Remove this helper
-class CompositeBaseObject(BaseModel):
-    TestOnlyExample: TestOnlyExample
-    TestOnlyNestedBaseObject: TestOnlyNestedBaseObject

From c723b327b70966f73a8076435251e16639402277 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 12:03:48 -0600
Subject: [PATCH 143/169] Removed second hack

---
 .../pages/wfReactInterface/baseObjectClassQuery.ts     | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index b1f7b4b148f..e89b0929d54 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -3,8 +3,7 @@ import {useEffect, useRef, useState} from 'react';
 import {z} from 'zod';
 
 import {
-  TestOnlyExampleSchema,
-  TestOnlyNestedBaseObjectSchema,
+  GeneratedBaseObjectClassesZodSchema,
 } from './generatedBaseObjectClasses.zod';
 import {TraceServerClient} from './traceServerClient';
 import {useGetTraceServerClientContext} from './traceServerClientContext';
@@ -16,11 +15,8 @@ import {
 } from './traceServerClientTypes';
 import {Loadable} from './wfDataModelHooksInterface';
 
-// TODO: This should be generated from the registry!
-const baseObjectClassRegistry: Record<string, z.ZodType> = {
-  TestOnlyExample: TestOnlyExampleSchema,
-  TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
-};
+const baseObjectClassRegistry: Record<string, z.ZodType> = 
+  GeneratedBaseObjectClassesZodSchema.shape;
 
 export const useBaseObjectInstances = <
   C extends keyof typeof baseObjectClassRegistry,

From ec7c0b79c7ff41b9af3929de5ae9709cc69ae579 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 12:14:11 -0600
Subject: [PATCH 144/169] lint

---
 .../Browse3/pages/wfReactInterface/baseObjectClassQuery.ts  | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index e89b0929d54..d1d4bbaeb48 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -2,9 +2,7 @@ import {useDeepMemo} from '@wandb/weave/hookUtils';
 import {useEffect, useRef, useState} from 'react';
 import {z} from 'zod';
 
-import {
-  GeneratedBaseObjectClassesZodSchema,
-} from './generatedBaseObjectClasses.zod';
+import {GeneratedBaseObjectClassesZodSchema} from './generatedBaseObjectClasses.zod';
 import {TraceServerClient} from './traceServerClient';
 import {useGetTraceServerClientContext} from './traceServerClientContext';
 import {
@@ -15,7 +13,7 @@ import {
 } from './traceServerClientTypes';
 import {Loadable} from './wfDataModelHooksInterface';
 
-const baseObjectClassRegistry: Record<string, z.ZodType> = 
+const baseObjectClassRegistry: Record<string, z.ZodType> =
   GeneratedBaseObjectClassesZodSchema.shape;
 
 export const useBaseObjectInstances = <

From 26e6ecc20ae3f2051e97c5edf509c6e07228931d Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 12:32:08 -0600
Subject: [PATCH 145/169] Initial uptake of changes - tests passing

---
 tests/trace/test_leaderboard.py               | 27 ++++++-----
 .../{leaderboard.py => leaderboard_util.py}   | 48 ++++---------------
 weave/trace/api.py                            |  6 +--
 .../base_object_registry.py                   |  2 +
 .../base_object_classes/leaderboard.py        | 29 +++++++++++
 5 files changed, 56 insertions(+), 56 deletions(-)
 rename weave/flow/{leaderboard.py => leaderboard_util.py} (64%)
 create mode 100644 weave/trace_server/interface/base_object_classes/leaderboard.py

diff --git a/tests/trace/test_leaderboard.py b/tests/trace/test_leaderboard.py
index 74a8f37f4cd..37257223916 100644
--- a/tests/trace/test_leaderboard.py
+++ b/tests/trace/test_leaderboard.py
@@ -1,8 +1,9 @@
 import pytest
 
 import weave
-from weave.flow import leaderboard
+from weave.flow.leaderboard_util import get_leaderboard_results
 from weave.trace.weave_client import get_ref
+from weave.trace_server.interface.base_object_classes import leaderboard
 
 
 def test_leaderboard_empty(client):
@@ -28,10 +29,10 @@ def test_leaderboard_empty(client):
 
     ref = weave.publish(spec)
 
-    # TODO: this is not working
-    # spec = ref.get()
+    # Overriding spec to show that this works
+    spec = ref.get()
 
-    results = leaderboard.get_leaderboard_results(spec, client)
+    results = get_leaderboard_results(spec, client)
     assert len(results) == 0
 
 
@@ -50,10 +51,10 @@ def test_leaderboard_mis_configured(client):
 
     ref = weave.publish(spec)
 
-    # TODO: this is not working
-    # spec = ref.get()
+    # Overriding spec to show that this works
+    spec = ref.get()
 
-    results = leaderboard.get_leaderboard_results(spec, client)
+    results = get_leaderboard_results(spec, client)
     assert len(results) == 0
 
 
@@ -119,10 +120,10 @@ async def test_leaderboard_with_results(client):
 
     ref = weave.publish(spec)
 
-    # TODO: this is not working
-    # spec = ref.get()
+    # Overriding spec to show that this works
+    spec = ref.get()
 
-    results = leaderboard.get_leaderboard_results(spec, client)
+    results = get_leaderboard_results(spec, client)
     assert len(results) == 1
     assert results[0].model_ref == get_ref(simple_model).uri()
     assert results[0].column_scores[0].scores[0].value == 1.0
@@ -169,10 +170,10 @@ async def test_leaderboard_with_results(client):
 
     ref = weave.publish(spec)
 
-    # TODO: this is not working
-    # spec = ref.get()
+    # Overriding spec to show that this works
+    spec = ref.get()
 
-    results = leaderboard.get_leaderboard_results(spec, client)
+    results = get_leaderboard_results(spec, client)
     assert len(results) == 3
     assert results[0].model_ref == get_ref(simple_model).uri()
     assert len(results[0].column_scores) == 3
diff --git a/weave/flow/leaderboard.py b/weave/flow/leaderboard_util.py
similarity index 64%
rename from weave/flow/leaderboard.py
rename to weave/flow/leaderboard_util.py
index 26b8191d417..f21176e91bd 100644
--- a/weave/flow/leaderboard.py
+++ b/weave/flow/leaderboard_util.py
@@ -1,44 +1,12 @@
-from dataclasses import field
-from typing import Any, Optional
-
-from pydantic import BaseModel
-
-import weave
 from weave.trace.refs import OpRef
 from weave.trace.weave_client import WeaveClient, get_ref
+from weave.trace_server.interface.base_object_classes import leaderboard
 from weave.trace_server.trace_server_interface import CallsFilter
 
 
-class LeaderboardColumn(BaseModel):
-    evaluation_object_ref: str
-    scorer_name: str
-    summary_metric_path_parts: list[str] = field(default_factory=list)
-    should_minimize: Optional[bool] = None
-
-
-# TODO: Merge `Leaderboard` into weave/trace_server/interface/base_models/base_model_registry.py after
-# Online evals lands
-class Leaderboard(weave.Object):
-    columns: list[LeaderboardColumn]
-
-
-class LeaderboardModelEvaluationResult(BaseModel):
-    evaluate_call_ref: str
-    value: Any
-
-
-class ModelScoresForColumn(BaseModel):
-    scores: list[LeaderboardModelEvaluationResult]
-
-
-class LeaderboardModelResult(BaseModel):
-    model_ref: str
-    column_scores: list[ModelScoresForColumn]
-
-
 def get_leaderboard_results(
-    spec: Leaderboard, client: WeaveClient
-) -> list[LeaderboardModelResult]:
+    spec: leaderboard.Leaderboard, client: WeaveClient
+) -> list[leaderboard.LeaderboardModelResult]:
     entity, project = client._project_id().split("/")
     calls = client.get_calls(
         filter=CallsFilter(
@@ -54,7 +22,7 @@ def get_leaderboard_results(
         )
     )
 
-    res_map: dict[str, LeaderboardModelResult] = {}
+    res_map: dict[str, leaderboard.LeaderboardModelResult] = {}
     for call in calls:
         # Frustrating that we have to get the ref like this. Since the
         # `Call` object auto-derefs the inputs (making a network request),
@@ -69,9 +37,11 @@ def get_leaderboard_results(
             continue
         model_ref_uri = model_ref.uri()
         if model_ref_uri not in res_map:
-            res_map[model_ref_uri] = LeaderboardModelResult(
+            res_map[model_ref_uri] = leaderboard.LeaderboardModelResult(
                 model_ref=model_ref_uri,
-                column_scores=[ModelScoresForColumn(scores=[]) for _ in spec.columns],
+                column_scores=[
+                    leaderboard.ModelScoresForColumn(scores=[]) for _ in spec.columns
+                ],
             )
         for col_idx, c in enumerate(spec.columns):
             eval_obj_ref = get_ref(call.inputs["self"])
@@ -89,7 +59,7 @@ def get_leaderboard_results(
                 else:
                     break
             res_map[model_ref_uri].column_scores[col_idx].scores.append(
-                LeaderboardModelEvaluationResult(
+                leaderboard.LeaderboardModelEvaluationResult(
                     evaluate_call_ref=call_ref_uri, value=val
                 )
             )
diff --git a/weave/trace/api.py b/weave/trace/api.py
index 35e94b7e8d2..38881c34e81 100644
--- a/weave/trace/api.py
+++ b/weave/trace/api.py
@@ -22,6 +22,7 @@
     should_disable_weave,
 )
 from weave.trace.table import Table
+from weave.trace_server.interface.base_object_classes import leaderboard
 
 
 def init(
@@ -101,9 +102,6 @@ def publish(obj: Any, name: Optional[str] = None) -> weave_client.ObjectRef:
 
     ref = client._save_object(obj, save_name, "latest")
 
-    # Avoid circular import
-    from weave.flow.leaderboard import Leaderboard
-
     if isinstance(ref, weave_client.ObjectRef):
         if isinstance(ref, weave_client.OpRef):
             url = urls.op_version_path(
@@ -112,7 +110,7 @@ def publish(obj: Any, name: Optional[str] = None) -> weave_client.ObjectRef:
                 ref.name,
                 ref.digest,
             )
-        elif isinstance(obj, Leaderboard):
+        elif isinstance(obj, leaderboard.Leaderboard):
             url = urls.leaderboard_path(
                 ref.entity,
                 ref.project,
diff --git a/weave/trace_server/interface/base_object_classes/base_object_registry.py b/weave/trace_server/interface/base_object_classes/base_object_registry.py
index 8941dfa75ad..5e94582718d 100644
--- a/weave/trace_server/interface/base_object_classes/base_object_registry.py
+++ b/weave/trace_server/interface/base_object_classes/base_object_registry.py
@@ -1,6 +1,7 @@
 from typing import Dict, Type
 
 from weave.trace_server.interface.base_object_classes.base_object_def import BaseObject
+from weave.trace_server.interface.base_object_classes.leaderboard import Leaderboard
 from weave.trace_server.interface.base_object_classes.test_only_example import *
 
 BASE_OBJECT_REGISTRY: Dict[str, Type[BaseObject]] = {}
@@ -18,3 +19,4 @@ def register_base_object(cls: Type[BaseObject]) -> None:
 
 register_base_object(TestOnlyExample)
 register_base_object(TestOnlyNestedBaseObject)
+register_base_object(Leaderboard)
diff --git a/weave/trace_server/interface/base_object_classes/leaderboard.py b/weave/trace_server/interface/base_object_classes/leaderboard.py
new file mode 100644
index 00000000000..f1f255a24fc
--- /dev/null
+++ b/weave/trace_server/interface/base_object_classes/leaderboard.py
@@ -0,0 +1,29 @@
+from dataclasses import field
+from typing import Any, Optional
+
+from pydantic import BaseModel
+from weave.trace_server.interface.base_object_classes import base_object_def
+
+class LeaderboardColumn(BaseModel):
+    evaluation_object_ref: base_object_def.RefStr
+    scorer_name: str
+    summary_metric_path_parts: list[str] = field(default_factory=list)
+    should_minimize: Optional[bool] = None
+
+
+class Leaderboard(base_object_def.BaseObject):
+    columns: list[LeaderboardColumn]
+
+
+class LeaderboardModelEvaluationResult(BaseModel):
+    evaluate_call_ref: base_object_def.RefStr
+    value: Any
+
+
+class ModelScoresForColumn(BaseModel):
+    scores: list[LeaderboardModelEvaluationResult]
+
+
+class LeaderboardModelResult(BaseModel):
+    model_ref: str
+    column_scores: list[ModelScoresForColumn]

From 2e3cb7aa22240bb5c66152ee6e20966a5f5b8211 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 12:36:36 -0600
Subject: [PATCH 146/169] Paydantic fix + schema gen

---
 .../generatedBaseObjectClasses.zod.ts         | 16 ++++
 weave/flow/leaderboard_util.py                | 30 +++++--
 .../generated_base_object_class_schemas.json  | 83 ++++++++++++++++++-
 .../base_object_classes/leaderboard.py        | 18 +---
 4 files changed, 124 insertions(+), 23 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
index ba5a38ebf65..2111a1c5ae9 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
@@ -1,5 +1,13 @@
 import * as z from 'zod';
 
+export const LeaderboardColumnSchema = z.object({
+  evaluation_object_ref: z.string(),
+  scorer_name: z.string(),
+  should_minimize: z.union([z.boolean(), z.null()]).optional(),
+  summary_metric_path_parts: z.array(z.string()).optional(),
+});
+export type LeaderboardColumn = z.infer<typeof LeaderboardColumnSchema>;
+
 export const TestOnlyNestedBaseModelSchema = z.object({
   a: z.number(),
 });
@@ -16,6 +24,13 @@ export type TestOnlyNestedBaseObject = z.infer<
   typeof TestOnlyNestedBaseObjectSchema
 >;
 
+export const LeaderboardSchema = z.object({
+  columns: z.array(LeaderboardColumnSchema),
+  description: z.union([z.null(), z.string()]).optional(),
+  name: z.union([z.null(), z.string()]).optional(),
+});
+export type Leaderboard = z.infer<typeof LeaderboardSchema>;
+
 export const TestOnlyExampleSchema = z.object({
   description: z.union([z.null(), z.string()]).optional(),
   name: z.union([z.null(), z.string()]).optional(),
@@ -26,6 +41,7 @@ export const TestOnlyExampleSchema = z.object({
 export type TestOnlyExample = z.infer<typeof TestOnlyExampleSchema>;
 
 export const GeneratedBaseObjectClassesZodSchema = z.object({
+  Leaderboard: LeaderboardSchema,
   TestOnlyExample: TestOnlyExampleSchema,
   TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
 });
diff --git a/weave/flow/leaderboard_util.py b/weave/flow/leaderboard_util.py
index f21176e91bd..12f5b36d1d3 100644
--- a/weave/flow/leaderboard_util.py
+++ b/weave/flow/leaderboard_util.py
@@ -1,12 +1,30 @@
+from typing import Any
+
+from pydantic import BaseModel
+
 from weave.trace.refs import OpRef
 from weave.trace.weave_client import WeaveClient, get_ref
 from weave.trace_server.interface.base_object_classes import leaderboard
 from weave.trace_server.trace_server_interface import CallsFilter
 
 
+class LeaderboardModelEvaluationResult(BaseModel):
+    evaluate_call_ref: str
+    value: Any
+
+
+class ModelScoresForColumn(BaseModel):
+    scores: list[LeaderboardModelEvaluationResult]
+
+
+class LeaderboardModelResult(BaseModel):
+    model_ref: str
+    column_scores: list[ModelScoresForColumn]
+
+
 def get_leaderboard_results(
     spec: leaderboard.Leaderboard, client: WeaveClient
-) -> list[leaderboard.LeaderboardModelResult]:
+) -> list[LeaderboardModelResult]:
     entity, project = client._project_id().split("/")
     calls = client.get_calls(
         filter=CallsFilter(
@@ -22,7 +40,7 @@ def get_leaderboard_results(
         )
     )
 
-    res_map: dict[str, leaderboard.LeaderboardModelResult] = {}
+    res_map: dict[str, LeaderboardModelResult] = {}
     for call in calls:
         # Frustrating that we have to get the ref like this. Since the
         # `Call` object auto-derefs the inputs (making a network request),
@@ -37,11 +55,9 @@ def get_leaderboard_results(
             continue
         model_ref_uri = model_ref.uri()
         if model_ref_uri not in res_map:
-            res_map[model_ref_uri] = leaderboard.LeaderboardModelResult(
+            res_map[model_ref_uri] = LeaderboardModelResult(
                 model_ref=model_ref_uri,
-                column_scores=[
-                    leaderboard.ModelScoresForColumn(scores=[]) for _ in spec.columns
-                ],
+                column_scores=[ModelScoresForColumn(scores=[]) for _ in spec.columns],
             )
         for col_idx, c in enumerate(spec.columns):
             eval_obj_ref = get_ref(call.inputs["self"])
@@ -59,7 +75,7 @@ def get_leaderboard_results(
                 else:
                     break
             res_map[model_ref_uri].column_scores[col_idx].scores.append(
-                leaderboard.LeaderboardModelEvaluationResult(
+                LeaderboardModelEvaluationResult(
                     evaluate_call_ref=call_ref_uri, value=val
                 )
             )
diff --git a/weave/scripts/generated_base_object_class_schemas.json b/weave/scripts/generated_base_object_class_schemas.json
index 4f7aee9dd54..512a9ac4c57 100644
--- a/weave/scripts/generated_base_object_class_schemas.json
+++ b/weave/scripts/generated_base_object_class_schemas.json
@@ -1,5 +1,82 @@
 {
   "$defs": {
+    "Leaderboard": {
+      "properties": {
+        "name": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Name"
+        },
+        "description": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Description"
+        },
+        "columns": {
+          "items": {
+            "$ref": "#/$defs/LeaderboardColumn"
+          },
+          "title": "Columns",
+          "type": "array"
+        }
+      },
+      "required": [
+        "columns"
+      ],
+      "title": "Leaderboard",
+      "type": "object"
+    },
+    "LeaderboardColumn": {
+      "properties": {
+        "evaluation_object_ref": {
+          "title": "Evaluation Object Ref",
+          "type": "string"
+        },
+        "scorer_name": {
+          "title": "Scorer Name",
+          "type": "string"
+        },
+        "summary_metric_path_parts": {
+          "items": {
+            "type": "string"
+          },
+          "title": "Summary Metric Path Parts",
+          "type": "array"
+        },
+        "should_minimize": {
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Should Minimize"
+        }
+      },
+      "required": [
+        "evaluation_object_ref",
+        "scorer_name"
+      ],
+      "title": "LeaderboardColumn",
+      "type": "object"
+    },
     "TestOnlyExample": {
       "properties": {
         "name": {
@@ -103,11 +180,15 @@
     },
     "TestOnlyNestedBaseObject": {
       "$ref": "#/$defs/TestOnlyNestedBaseObject"
+    },
+    "Leaderboard": {
+      "$ref": "#/$defs/Leaderboard"
     }
   },
   "required": [
     "TestOnlyExample",
-    "TestOnlyNestedBaseObject"
+    "TestOnlyNestedBaseObject",
+    "Leaderboard"
   ],
   "title": "CompositeBaseObject",
   "type": "object"
diff --git a/weave/trace_server/interface/base_object_classes/leaderboard.py b/weave/trace_server/interface/base_object_classes/leaderboard.py
index f1f255a24fc..8d8a013b222 100644
--- a/weave/trace_server/interface/base_object_classes/leaderboard.py
+++ b/weave/trace_server/interface/base_object_classes/leaderboard.py
@@ -1,9 +1,11 @@
 from dataclasses import field
-from typing import Any, Optional
+from typing import Optional
 
 from pydantic import BaseModel
+
 from weave.trace_server.interface.base_object_classes import base_object_def
 
+
 class LeaderboardColumn(BaseModel):
     evaluation_object_ref: base_object_def.RefStr
     scorer_name: str
@@ -13,17 +15,3 @@ class LeaderboardColumn(BaseModel):
 
 class Leaderboard(base_object_def.BaseObject):
     columns: list[LeaderboardColumn]
-
-
-class LeaderboardModelEvaluationResult(BaseModel):
-    evaluate_call_ref: base_object_def.RefStr
-    value: Any
-
-
-class ModelScoresForColumn(BaseModel):
-    scores: list[LeaderboardModelEvaluationResult]
-
-
-class LeaderboardModelResult(BaseModel):
-    model_ref: str
-    column_scores: list[ModelScoresForColumn]

From 2b67837698edf9c478c0943ef5aa441eeb846a71 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 12:43:44 -0600
Subject: [PATCH 147/169] typescript

---
 .../traceServerDirectClient.ts                | 43 -------------------
 1 file changed, 43 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
index 31ed9532ff8..25f11624740 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/traceServerDirectClient.ts
@@ -48,33 +48,6 @@ import {
   TraceTableQueryStatsRes,
 } from './traceServerClientTypes';
 
-/**
- * Sanitizes an object name by replacing non-alphanumeric characters with dashes and enforcing length limits.
- * This matches the Python implementation in weave_client.py.
- *
- * @param name The name to sanitize
- * @returns The sanitized name
- * @throws Error if the resulting name would be empty
- */
-export function sanitizeObjectId(name: string): string {
-  // Replace any non-word chars (except dots and underscores) with dashes
-  let res = name.replace(/[^\w._]+/g, '-');
-  // Replace multiple consecutive dashes/dots/underscores with a single dash
-  res = res.replace(/([._-]{2,})+/g, '-');
-  // Remove leading/trailing dashes and underscores
-  res = res.replace(/^[-_]+|[-_]+$/g, '');
-
-  if (!res) {
-    throw new Error(`Invalid object name: ${name}`);
-  }
-
-  if (res.length > 128) {
-    res = res.slice(0, 128);
-  }
-
-  return res;
-}
-
 export class DirectTraceServerClient {
   private baseUrl: string;
   private inFlightFetchesRequests: Record<
@@ -253,22 +226,6 @@ export class DirectTraceServerClient {
     return this.makeRequest<TraceObjReadReq, TraceObjReadRes>('/obj/read', req);
   }
 
-  public objCreate(req: TraceObjCreateReq): Promise<TraceObjCreateRes> {
-    const initialObjectId = req.obj.object_id;
-    const sanitizedObjectId = sanitizeObjectId(initialObjectId);
-    if (sanitizedObjectId !== initialObjectId) {
-      // Caller is expected to sanitize the object id. We should be doing this
-      // on the server, but it is currently disabled.
-      throw new Error(
-        `Invalid object name: ${initialObjectId}, sanitized to ${sanitizedObjectId}`
-      );
-    }
-    return this.makeRequest<TraceObjCreateReq, TraceObjCreateRes>(
-      '/obj/create',
-      req
-    );
-  }
-
   public readBatch(req: TraceRefsReadBatchReq): Promise<TraceRefsReadBatchRes> {
     return this.makeRequest<TraceRefsReadBatchReq, TraceRefsReadBatchRes>(
       '/refs/read_batch',

From f8bf2a67cb285fbc4b42811b6874b37638829767 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 12:56:26 -0600
Subject: [PATCH 148/169] style

---
 .../views/Leaderboard/LeaderboardGrid.tsx     | 33 +++++++++++--------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index ee7238cbb23..94a9d2c5cc4 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -121,19 +121,7 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
         minWidth: 150,
         flex: 1,
         renderCell: (params: GridRenderCellParams) => {
-          let isOp = false;
-          try {
-            isOp =
-              Object.values(
-                Object.values(
-                  Object.values(
-                    (params.row as RowData).modelGroup.datasetGroups
-                  )[0].scorerGroups
-                )[0].metricPathGroups
-              )[0][0].modelType === 'op';
-          } catch (e) {
-            console.log(e);
-          }
+          const isOp = modelGroupIsOp((params.row as RowData).modelGroup);
           const modelRef = parseRefMaybe(
             `weave:///${entity}/${project}/${isOp ? 'op' : 'object'}/${
               params.value
@@ -480,6 +468,25 @@ const getColumnStats = (data: GroupedLeaderboardData): ColumnStats => {
   return stats;
 };
 
+/**
+ * Check if a model group is an op. This is a little hacky - we just look
+ * at the first entry down the chain and see if it's an op.
+ */
+const modelGroupIsOp = (modelGroup: GroupedLeaderboardModelGroup) => {
+  let isOp = false;
+  try {
+    isOp =
+      Object.values(
+        Object.values(
+          Object.values(modelGroup.datasetGroups)[0].scorerGroups
+        )[0].metricPathGroups
+      )[0][0].modelType === 'op';
+  } catch (e) {
+    console.log(e);
+  }
+  return isOp;
+};
+
 const valueFromRowData = (
   rowData: RowData,
   datasetGroupName: string,

From 7b9a49cdd727789be8db2c614de4509ab7676bf9 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 13:22:43 -0600
Subject: [PATCH 149/169] convert to path instead of path parts

---
 tests/trace/test_leaderboard.py                      |  8 ++++----
 .../generatedBaseObjectClasses.zod.ts                |  2 +-
 weave/flow/leaderboard_util.py                       |  3 ++-
 .../scripts/generated_base_object_class_schemas.json | 12 +++++-------
 .../interface/base_object_classes/leaderboard.py     |  3 +--
 5 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/tests/trace/test_leaderboard.py b/tests/trace/test_leaderboard.py
index 37257223916..242891e76d5 100644
--- a/tests/trace/test_leaderboard.py
+++ b/tests/trace/test_leaderboard.py
@@ -113,7 +113,7 @@ async def test_leaderboard_with_results(client):
             leaderboard.LeaderboardColumn(
                 evaluation_object_ref=get_ref(evaluation_obj_1).uri(),
                 scorer_name="my_scorer",
-                summary_metric_path_parts=["true_fraction"],
+                summary_metric_path="true_fraction",
             )
         ],
     )
@@ -152,18 +152,18 @@ async def test_leaderboard_with_results(client):
             leaderboard.LeaderboardColumn(
                 evaluation_object_ref=get_ref(evaluation_obj_2).uri(),
                 scorer_name="my_scorer",
-                summary_metric_path_parts=["true_count"],
+                summary_metric_path="true_count",
             ),
             leaderboard.LeaderboardColumn(
                 evaluation_object_ref=get_ref(evaluation_obj_2).uri(),
                 scorer_name="my_scorer",
                 should_minimize=True,
-                summary_metric_path_parts=["true_fraction"],
+                summary_metric_path="true_fraction",
             ),
             leaderboard.LeaderboardColumn(
                 evaluation_object_ref=get_ref(evaluation_obj_1).uri(),
                 scorer_name="my_scorer",
-                summary_metric_path_parts=["true_fraction"],
+                summary_metric_path="true_fraction",
             ),
         ],
     )
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
index 2111a1c5ae9..959e7a29fb6 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
@@ -4,7 +4,7 @@ export const LeaderboardColumnSchema = z.object({
   evaluation_object_ref: z.string(),
   scorer_name: z.string(),
   should_minimize: z.union([z.boolean(), z.null()]).optional(),
-  summary_metric_path_parts: z.array(z.string()).optional(),
+  summary_metric_path: z.string(),
 });
 export type LeaderboardColumn = z.infer<typeof LeaderboardColumnSchema>;
 
diff --git a/weave/flow/leaderboard_util.py b/weave/flow/leaderboard_util.py
index 12f5b36d1d3..d7d12e00c00 100644
--- a/weave/flow/leaderboard_util.py
+++ b/weave/flow/leaderboard_util.py
@@ -67,7 +67,8 @@ def get_leaderboard_results(
             if c.evaluation_object_ref != eval_obj_ref_uri:
                 continue
             val = call.output.get(c.scorer_name)
-            for part in c.summary_metric_path_parts:
+            summary_metric_path_parts = c.summary_metric_path.split(".")
+            for part in summary_metric_path_parts:
                 if isinstance(val, dict):
                     val = val.get(part)
                 elif isinstance(val, list):
diff --git a/weave/scripts/generated_base_object_class_schemas.json b/weave/scripts/generated_base_object_class_schemas.json
index 512a9ac4c57..2207d2c1f3c 100644
--- a/weave/scripts/generated_base_object_class_schemas.json
+++ b/weave/scripts/generated_base_object_class_schemas.json
@@ -50,12 +50,9 @@
           "title": "Scorer Name",
           "type": "string"
         },
-        "summary_metric_path_parts": {
-          "items": {
-            "type": "string"
-          },
-          "title": "Summary Metric Path Parts",
-          "type": "array"
+        "summary_metric_path": {
+          "title": "Summary Metric Path",
+          "type": "string"
         },
         "should_minimize": {
           "anyOf": [
@@ -72,7 +69,8 @@
       },
       "required": [
         "evaluation_object_ref",
-        "scorer_name"
+        "scorer_name",
+        "summary_metric_path"
       ],
       "title": "LeaderboardColumn",
       "type": "object"
diff --git a/weave/trace_server/interface/base_object_classes/leaderboard.py b/weave/trace_server/interface/base_object_classes/leaderboard.py
index 8d8a013b222..c55be78e80f 100644
--- a/weave/trace_server/interface/base_object_classes/leaderboard.py
+++ b/weave/trace_server/interface/base_object_classes/leaderboard.py
@@ -1,4 +1,3 @@
-from dataclasses import field
 from typing import Optional
 
 from pydantic import BaseModel
@@ -9,7 +8,7 @@
 class LeaderboardColumn(BaseModel):
     evaluation_object_ref: base_object_def.RefStr
     scorer_name: str
-    summary_metric_path_parts: list[str] = field(default_factory=list)
+    summary_metric_path: str
     should_minimize: Optional[bool] = None
 
 

From c217b602430447b90dc1c38d318fec2951304b43 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 13:28:05 -0600
Subject: [PATCH 150/169] Uptake formal type

---
 .../LeaderboardConfigEditor.tsx               | 16 +++---
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 56 +++++++++----------
 .../Leaderboard/query/leaderboardQuery.ts     |  8 +--
 .../types/leaderboardConfigType.ts            | 16 ++----
 4 files changed, 44 insertions(+), 52 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 045e60f9d43..24c0c1c20ef 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -62,9 +62,9 @@ export const LeaderboardConfigEditor: React.FC<{
     // Reset dependent fields when changing evaluation_object_ref or scorer_name
     if (field === 'evaluation_object_ref') {
       newColumns[index].scorer_name = '';
-      newColumns[index].summary_metric_path_parts = [];
+      newColumns[index].summary_metric_path = '';
     } else if (field === 'scorer_name') {
-      newColumns[index].summary_metric_path_parts = [];
+      newColumns[index].summary_metric_path = '';
     }
 
     setWorkingCopy({...leaderboardVal, columns: newColumns});
@@ -79,7 +79,7 @@ export const LeaderboardConfigEditor: React.FC<{
           evaluation_object_ref: '',
           scorer_name: '',
           should_minimize: false,
-          summary_metric_path_parts: [],
+          summary_metric_path: '',
         },
       ],
     });
@@ -120,12 +120,12 @@ export const LeaderboardConfigEditor: React.FC<{
         <Label>Leaderboard Title</Label>
         <TextField
           icon="layout-grid"
-          value={leaderboardVal.name}
+          value={leaderboardVal.name ?? ''}
           onChange={handleNameChange}
         />
         <Label>Description</Label>
         <SimpleCodeLikeTextArea
-          value={leaderboardVal.description}
+          value={leaderboardVal.description ?? ''}
           onChange={handleDescriptionChange}
         />
         <Label>Columns</Label>
@@ -234,8 +234,8 @@ const ColumnEditor: React.FC<{
     [column.scorer_name]
   );
   const selectedMetricPath = useMemo(
-    () => ({val: column.summary_metric_path_parts.join('.')}),
-    [column.summary_metric_path_parts]
+    () => ({val: column.summary_metric_path}),
+    [column.summary_metric_path]
   );
   const shouldMinimize = column.should_minimize ?? false;
   return (
@@ -269,7 +269,7 @@ const ColumnEditor: React.FC<{
         onChange={newVal =>
           handleColumnChange(
             index,
-            'summary_metric_path_parts',
+            'summary_metric_path',
             newVal?.val.split('.')
           )
         }
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index 4085cb60376..cbd4ef8cce6 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -177,7 +177,7 @@ export const LeaderboardPageContentInner: React.FC<
   const [workingLeaderboardValCopy, setWorkingLeaderboardValCopy] =
     useState(leaderboardVal);
   useEffect(() => {
-    props.setName(workingLeaderboardValCopy.name);
+    props.setName(workingLeaderboardValCopy.name ?? '');
   }, [props, workingLeaderboardValCopy.name]);
   const {loading, data, evalData} = useSavedLeaderboardData(
     props.entity,
@@ -218,7 +218,7 @@ export const LeaderboardPageContentInner: React.FC<
         const datasetGroup = evalData[col.evaluation_object_ref]?.datasetGroup;
         const scorerGroup =
           evalData[col.evaluation_object_ref]?.scorers[col.scorer_name];
-        const metricGroup = col.summary_metric_path_parts.join('.');
+        const metricGroup = col.summary_metric_path;
 
         if (datasetGroup && scorerGroup && metricGroup) {
           return {
@@ -346,36 +346,36 @@ const parseLeaderboardVal = (
   }
 
   const finalColumns = columns
-    .map(column => {
-      const evaluationObjectRef = column.evaluation_object_ref;
-      if (typeof evaluationObjectRef !== 'string') {
-        return null;
-      }
+    // .map(column => {
+    //   const evaluationObjectRef = column.evaluation_object_ref;
+    //   if (typeof evaluationObjectRef !== 'string') {
+    //     return null;
+    //   }
 
-      const scorerName = column.scorer_name;
-      if (typeof scorerName !== 'string') {
-        return null;
-      }
+    //   const scorerName = column.scorer_name;
+    //   if (typeof scorerName !== 'string') {
+    //     return null;
+    //   }
 
-      const shouldMinimize = column.should_minimize;
-      if (shouldMinimize != null && typeof shouldMinimize !== 'boolean') {
-        return null;
-      }
+    //   const shouldMinimize = column.should_minimize;
+    //   if (shouldMinimize != null && typeof shouldMinimize !== 'boolean') {
+    //     return null;
+    //   }
 
-      const summaryMetricParts = column.summary_metric_path_parts;
-      if (!Array.isArray(summaryMetricParts)) {
-        return null;
-      } else if (summaryMetricParts.some(part => typeof part !== 'string')) {
-        return null;
-      }
+    //   const summaryMetricParts = column.summary_metric_path_parts;
+    //   if (!Array.isArray(summaryMetricParts)) {
+    //     return null;
+    //   } else if (summaryMetricParts.some(part => typeof part !== 'string')) {
+    //     return null;
+    //   }
 
-      return {
-        evaluation_object_ref: evaluationObjectRef,
-        scorer_name: scorerName,
-        should_minimize: shouldMinimize,
-        summary_metric_path_parts: summaryMetricParts,
-      };
-    })
+    //   return {
+    //     evaluation_object_ref: evaluationObjectRef,
+    //     scorer_name: scorerName,
+    //     should_minimize: shouldMinimize,
+    //     summary_metric_path_parts: summaryMetricParts,
+    //   };
+    // })
     .filter(column => column != null) as LeaderboardObjectVal['columns'];
 
   return {
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
index c7a3a46a09d..80a7ce8bd34 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/query/leaderboardQuery.ts
@@ -619,7 +619,7 @@ const getLeaderboardObjectGroupableData = async (
         } else {
           value = (value as any)[col.scorer_name];
         }
-        col.summary_metric_path_parts.forEach(part => {
+        col.summary_metric_path.split('.').forEach(part => {
           if (value == null) {
             return;
           }
@@ -642,7 +642,7 @@ const getLeaderboardObjectGroupableData = async (
           modelGroup,
           datasetGroup,
           scorerGroup,
-          metricPathGroup: col.summary_metric_path_parts.join('.'),
+          metricPathGroup: col.summary_metric_path,
           sortKey: -convertISOToDate(call.started_at).getTime(),
           row: {
             datasetName: datasetRef.artifactName,
@@ -650,7 +650,7 @@ const getLeaderboardObjectGroupableData = async (
             metricType: 'scorerMetric',
             scorerName: scorerRef.artifactName,
             scorerVersion: scorerRef.artifactVersion,
-            metricPath: col.summary_metric_path_parts.join('.'),
+            metricPath: col.summary_metric_path,
             metricValue: value as any,
             modelName: modelRef.artifactName,
             modelVersion: modelRef.artifactVersion,
@@ -659,7 +659,7 @@ const getLeaderboardObjectGroupableData = async (
             createdAt: convertISOToDate(call.started_at),
             sourceEvaluationCallId: call.id,
             sourceEvaluationObjectRef: col.evaluation_object_ref,
-            shouldMinimize: col.should_minimize,
+            shouldMinimize: col.should_minimize ?? false,
           },
         };
         data.push(row);
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts
index dd771c206b8..5207633ebca 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/types/leaderboardConfigType.ts
@@ -1,3 +1,5 @@
+import {Leaderboard} from '../../../pages/wfReactInterface/generatedBaseObjectClasses.zod';
+
 export const ALL_VALUE = '*';
 
 export type LeaderboardConfigType = {
@@ -44,15 +46,5 @@ export type FilterAndGroupModelSpec = {
   groupAllVersions?: boolean;
 };
 
-// TODO: Merge into weave-js/src/components/PagePanelComponents/Home/Browse3/collections/collectionRegistry.ts after
-// Online evals lands
-export type LeaderboardObjectVal = {
-  name: string;
-  description: string;
-  columns: Array<{
-    evaluation_object_ref: string;
-    scorer_name: string;
-    should_minimize?: boolean;
-    summary_metric_path_parts: string[];
-  }>;
-};
+// This alias could go away, just here to make refactoring easier
+export type LeaderboardObjectVal = Leaderboard;

From 5a51ef144941ffa606bcba385a0d0790c9d99c5f Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 13:29:11 -0600
Subject: [PATCH 151/169] Uptake formal type

---
 docs/docs/reference/gen_notebooks/leaderboard_quickstart.md | 6 +++---
 docs/notebooks/leaderboard_quickstart.ipynb                 | 6 +++---
 weave/flow/leaderboard_util.py                              | 3 +--
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/docs/docs/reference/gen_notebooks/leaderboard_quickstart.md b/docs/docs/reference/gen_notebooks/leaderboard_quickstart.md
index 5ba6c65dcb4..1830209847f 100644
--- a/docs/docs/reference/gen_notebooks/leaderboard_quickstart.md
+++ b/docs/docs/reference/gen_notebooks/leaderboard_quickstart.md
@@ -323,18 +323,18 @@ This leaderboard compares the performance of models in terms of world knowledge
         leaderboard.LeaderboardColumn(
             evaluation_object_ref=get_ref(evaluations[0]).uri(),
             scorer_name="check_concrete_fields",
-            summary_metric_path_parts=["state_match", "true_fraction"],
+            summary_metric_path="state_match.true_fraction",
         ),
         leaderboard.LeaderboardColumn(
             evaluation_object_ref=get_ref(evaluations[1]).uri(),
             scorer_name="check_value_fields",
             should_minimize=True,
-            summary_metric_path_parts=["avg_temp_f_err", "mean"],
+            summary_metric_path="avg_temp_f_err.mean",
         ),
         leaderboard.LeaderboardColumn(
             evaluation_object_ref=get_ref(evaluations[2]).uri(),
             scorer_name="check_subjective_fields",
-            summary_metric_path_parts=["correct_known_for", "true_fraction"],
+            summary_metric_path="correct_known_for.true_fraction",
         ),
     ],
 )
diff --git a/docs/notebooks/leaderboard_quickstart.ipynb b/docs/notebooks/leaderboard_quickstart.ipynb
index 2b71ad9e244..b09d336de51 100644
--- a/docs/notebooks/leaderboard_quickstart.ipynb
+++ b/docs/notebooks/leaderboard_quickstart.ipynb
@@ -401,18 +401,18 @@
     "        leaderboard.LeaderboardColumn(\n",
     "            evaluation_object_ref=get_ref(evaluations[0]).uri(),\n",
     "            scorer_name=\"check_concrete_fields\",\n",
-    "            summary_metric_path_parts=[\"state_match\", \"true_fraction\"],\n",
+    "            summary_metric_path=\"state_match.true_fraction\",\n",
     "        ),\n",
     "        leaderboard.LeaderboardColumn(\n",
     "            evaluation_object_ref=get_ref(evaluations[1]).uri(),\n",
     "            scorer_name=\"check_value_fields\",\n",
     "            should_minimize=True,\n",
-    "            summary_metric_path_parts=[\"avg_temp_f_err\", \"mean\"],\n",
+    "            summary_metric_path=\"avg_temp_f_err.mean\",\n",
     "        ),\n",
     "        leaderboard.LeaderboardColumn(\n",
     "            evaluation_object_ref=get_ref(evaluations[2]).uri(),\n",
     "            scorer_name=\"check_subjective_fields\",\n",
-    "            summary_metric_path_parts=[\"correct_known_for\", \"true_fraction\"],\n",
+    "            summary_metric_path=\"correct_known_for.true_fraction\",\n",
     "        ),\n",
     "    ],\n",
     ")\n",
diff --git a/weave/flow/leaderboard_util.py b/weave/flow/leaderboard_util.py
index d7d12e00c00..4084597d751 100644
--- a/weave/flow/leaderboard_util.py
+++ b/weave/flow/leaderboard_util.py
@@ -67,8 +67,7 @@ def get_leaderboard_results(
             if c.evaluation_object_ref != eval_obj_ref_uri:
                 continue
             val = call.output.get(c.scorer_name)
-            summary_metric_path_parts = c.summary_metric_path.split(".")
-            for part in summary_metric_path_parts:
+            for part in c.summary_metric_path.split("."):
                 if isinstance(val, dict):
                     val = val.get(part)
                 elif isinstance(val, list):

From 998f467da6c48c3ee9aad65e634488fb089035cf Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 13:35:45 -0600
Subject: [PATCH 152/169] split out hooks

---
 .../LeaderboardConfigEditor.tsx               | 175 +-----------------
 .../leaderboardConfigEditorHooks.ts           | 167 +++++++++++++++++
 2 files changed, 174 insertions(+), 168 deletions(-)
 create mode 100644 weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigEditorHooks.ts

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 24c0c1c20ef..9b100e9e200 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -11,21 +11,16 @@ import {
   IconSortAscending,
   IconSortDescending,
 } from '@wandb/weave/components/Icon';
-import {refUri} from '@wandb/weave/react';
 import _ from 'lodash';
-import React, {useEffect, useMemo, useState} from 'react';
+import React, {useMemo} from 'react';
 
-import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
-import {parseRefMaybe} from '../../../Browse2/SmallRef';
+import {LeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {
-  ALL_VALUE,
-  LeaderboardObjectVal,
-} from '../../views/Leaderboard/types/leaderboardConfigType';
-import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
-import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
-import {TraceObjSchema} from '../wfReactInterface/traceServerClientTypes';
-import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
-import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
+  EvaluationHelperObj,
+  useEvaluationObjects,
+  useMetrics,
+  useScorers,
+} from './leaderboardConfigEditorHooks';
 import {SimpleCodeLikeTextArea} from './SimpleCodeLikeTextArea';
 
 export const LeaderboardConfigEditor: React.FC<{
@@ -334,159 +329,3 @@ const ColumnEditor: React.FC<{
     </>
   );
 };
-
-type EvaluationHelperObj = {
-  obj: TraceObjSchema;
-  ref: string;
-  name: string;
-  versionIndex: number;
-  digest: string;
-};
-
-const useEvaluationObjects = (
-  entity: string,
-  project: string
-): EvaluationHelperObj[] => {
-  const getClient = useGetTraceServerClientContext();
-  const client = getClient();
-  const [evalObjs, setEvalObjs] = useState<EvaluationHelperObj[]>([]);
-  useEffect(() => {
-    let mounted = true;
-    client
-      .objsQuery({
-        project_id: projectIdFromParts({entity, project}),
-        filter: {
-          base_object_classes: ['Evaluation'],
-          is_op: false,
-        },
-        metadata_only: true,
-        sort_by: [{field: 'created_at', direction: 'desc'}],
-      })
-      .then(res => {
-        if (mounted) {
-          setEvalObjs(
-            res.objs.map(obj => ({
-              obj,
-              ref: refUri({
-                scheme: 'weave',
-                entityName: entity,
-                projectName: project,
-                weaveKind: 'object',
-                artifactName: obj.object_id,
-                artifactVersion: obj.digest,
-              }),
-              name: obj.object_id,
-              versionIndex: obj.version_index,
-              digest: obj.digest,
-            }))
-          );
-        }
-      });
-
-    return () => {
-      mounted = false;
-    };
-  }, [client, entity, project]);
-  return evalObjs;
-};
-
-const useScorers = (
-  entity: string,
-  project: string,
-  evaluationObjectRef: string
-): string[] => {
-  const getClient = useGetTraceServerClientContext();
-  const client = getClient();
-  const [scorers, setScorers] = useState<string[]>([]);
-
-  useEffect(() => {
-    if (!evaluationObjectRef) {
-      setScorers([]);
-      return;
-    }
-
-    let mounted = true;
-    client.readBatch({refs: [evaluationObjectRef]}).then(res => {
-      if (mounted) {
-        setScorers(
-          (res.vals[0].scorers ?? [])
-            .map((scorer: string) => parseRefMaybe(scorer)?.artifactName)
-            .filter(Boolean) as string[]
-        );
-      }
-    });
-
-    return () => {
-      mounted = false;
-    };
-  }, [client, entity, project, evaluationObjectRef]);
-
-  return scorers;
-};
-
-const useMetrics = (
-  entity: string,
-  project: string,
-  evaluationObjectRef: string,
-  scorerName: string
-): string[] => {
-  const getClient = useGetTraceServerClientContext();
-  const client = getClient();
-  const [metrics, setMetrics] = useState<string[]>([]);
-
-  useEffect(() => {
-    if (!evaluationObjectRef || !scorerName) {
-      setMetrics([]);
-      return;
-    }
-
-    let mounted = true;
-    client
-      .callsStreamQuery({
-        project_id: projectIdFromParts({entity, project}),
-        filter: {
-          op_names: [
-            opVersionKeyToRefUri({
-              entity,
-              project,
-              opId: EVALUATE_OP_NAME_POST_PYDANTIC,
-              versionHash: ALL_VALUE,
-            }),
-          ],
-          input_refs: [evaluationObjectRef],
-        },
-        limit: 1,
-      })
-      .then(res => {
-        if (mounted) {
-          const calls = res.calls;
-          if (calls.length === 0) {
-            setMetrics([]);
-          } else {
-            const output = calls[0].output ?? {};
-            if (
-              typeof output === 'object' &&
-              output != null &&
-              scorerName in output
-            ) {
-              setMetrics(
-                Object.keys(
-                  flattenObjectPreservingWeaveTypes(
-                    (output as Record<string, any>)[scorerName]
-                  )
-                )
-              );
-            } else {
-              setMetrics([]);
-            }
-          }
-        }
-      });
-
-    return () => {
-      mounted = false;
-    };
-  }, [client, entity, project, evaluationObjectRef, scorerName]);
-
-  return metrics;
-};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigEditorHooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigEditorHooks.ts
new file mode 100644
index 00000000000..b7e5caabb52
--- /dev/null
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/leaderboardConfigEditorHooks.ts
@@ -0,0 +1,167 @@
+import {refUri} from '@wandb/weave/react';
+import {useEffect, useState} from 'react';
+
+import {flattenObjectPreservingWeaveTypes} from '../../../Browse2/browse2Util';
+import {parseRefMaybe} from '../../../Browse2/SmallRef';
+import {ALL_VALUE} from '../../views/Leaderboard/types/leaderboardConfigType';
+import {EVALUATE_OP_NAME_POST_PYDANTIC} from '../common/heuristics';
+import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
+import {TraceObjSchema} from '../wfReactInterface/traceServerClientTypes';
+import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
+import {opVersionKeyToRefUri} from '../wfReactInterface/utilities';
+
+export type EvaluationHelperObj = {
+  obj: TraceObjSchema;
+  ref: string;
+  name: string;
+  versionIndex: number;
+  digest: string;
+};
+
+export const useEvaluationObjects = (
+  entity: string,
+  project: string
+): EvaluationHelperObj[] => {
+  const getClient = useGetTraceServerClientContext();
+  const client = getClient();
+  const [evalObjs, setEvalObjs] = useState<EvaluationHelperObj[]>([]);
+  useEffect(() => {
+    let mounted = true;
+    client
+      .objsQuery({
+        project_id: projectIdFromParts({entity, project}),
+        filter: {
+          base_object_classes: ['Evaluation'],
+          is_op: false,
+        },
+        metadata_only: true,
+        sort_by: [{field: 'created_at', direction: 'desc'}],
+      })
+      .then(res => {
+        if (mounted) {
+          setEvalObjs(
+            res.objs.map(obj => ({
+              obj,
+              ref: refUri({
+                scheme: 'weave',
+                entityName: entity,
+                projectName: project,
+                weaveKind: 'object',
+                artifactName: obj.object_id,
+                artifactVersion: obj.digest,
+              }),
+              name: obj.object_id,
+              versionIndex: obj.version_index,
+              digest: obj.digest,
+            }))
+          );
+        }
+      });
+
+    return () => {
+      mounted = false;
+    };
+  }, [client, entity, project]);
+  return evalObjs;
+};
+
+export const useScorers = (
+  entity: string,
+  project: string,
+  evaluationObjectRef: string
+): string[] => {
+  const getClient = useGetTraceServerClientContext();
+  const client = getClient();
+  const [scorers, setScorers] = useState<string[]>([]);
+
+  useEffect(() => {
+    if (!evaluationObjectRef) {
+      setScorers([]);
+      return;
+    }
+
+    let mounted = true;
+    client.readBatch({refs: [evaluationObjectRef]}).then(res => {
+      if (mounted) {
+        setScorers(
+          (res.vals[0].scorers ?? [])
+            .map((scorer: string) => parseRefMaybe(scorer)?.artifactName)
+            .filter(Boolean) as string[]
+        );
+      }
+    });
+
+    return () => {
+      mounted = false;
+    };
+  }, [client, entity, project, evaluationObjectRef]);
+
+  return scorers;
+};
+
+export const useMetrics = (
+  entity: string,
+  project: string,
+  evaluationObjectRef: string,
+  scorerName: string
+): string[] => {
+  const getClient = useGetTraceServerClientContext();
+  const client = getClient();
+  const [metrics, setMetrics] = useState<string[]>([]);
+
+  useEffect(() => {
+    if (!evaluationObjectRef || !scorerName) {
+      setMetrics([]);
+      return;
+    }
+
+    let mounted = true;
+    client
+      .callsStreamQuery({
+        project_id: projectIdFromParts({entity, project}),
+        filter: {
+          op_names: [
+            opVersionKeyToRefUri({
+              entity,
+              project,
+              opId: EVALUATE_OP_NAME_POST_PYDANTIC,
+              versionHash: ALL_VALUE,
+            }),
+          ],
+          input_refs: [evaluationObjectRef],
+        },
+        limit: 1,
+      })
+      .then(res => {
+        if (mounted) {
+          const calls = res.calls;
+          if (calls.length === 0) {
+            setMetrics([]);
+          } else {
+            const output = calls[0].output ?? {};
+            if (
+              typeof output === 'object' &&
+              output != null &&
+              scorerName in output
+            ) {
+              setMetrics(
+                Object.keys(
+                  flattenObjectPreservingWeaveTypes(
+                    (output as Record<string, any>)[scorerName]
+                  )
+                )
+              );
+            } else {
+              setMetrics([]);
+            }
+          }
+        }
+      });
+
+    return () => {
+      mounted = false;
+    };
+  }, [client, entity, project, evaluationObjectRef, scorerName]);
+
+  return metrics;
+};

From d3e0e0058a3f8e64e27859180e6a230c752098e8 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 13:47:59 -0600
Subject: [PATCH 153/169] Uptake the first of many hooks

---
 .../LeaderboardListingPage.tsx                | 42 ++++++++++---------
 .../wfReactInterface/tsDataModelHooks.ts      |  8 ++--
 .../wfDataModelHooksInterface.ts              |  4 +-
 3 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index ed1c2bbd150..e6c1ff877bb 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -1,8 +1,7 @@
 import {Box} from '@material-ui/core';
 import {Button} from '@wandb/weave/components/Button/Button';
-import {ErrorPanel} from '@wandb/weave/components/ErrorPanel';
 import {Loading} from '@wandb/weave/components/Loading';
-import React, {FC, useCallback} from 'react';
+import React, {FC, useCallback, useMemo} from 'react';
 import {useHistory} from 'react-router-dom';
 import styled from 'styled-components';
 
@@ -11,10 +10,13 @@ import {Empty} from '../common/Empty';
 import {EMPTY_PROPS_LEADERBOARDS} from '../common/EmptyContent';
 import {SimplePageLayout} from '../common/SimplePageLayout';
 import {ObjectVersionsTable} from '../ObjectVersionsPage';
-import {useWFHooks} from '../wfReactInterface/context';
+import {useBaseObjectInstances} from '../wfReactInterface/baseObjectClassQuery';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {sanitizeObjectId} from '../wfReactInterface/traceServerDirectClient';
-import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
+import {
+  convertTraceServerObjectVersionToSchema,
+  projectIdFromParts,
+} from '../wfReactInterface/tsDataModelHooks';
 import {ObjectVersionSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 import {useIsEditor} from './LeaderboardPage';
 
@@ -101,18 +103,22 @@ const LeaderboardTable: React.FC<{
 }> = props => {
   const history = useHistory();
   const {peekingRouter} = useWeaveflowRouteContext();
-  const {useRootObjectVersions} = useWFHooks();
 
   // TODO: Once `useCollectionObjects` lands from the online
   // evals project, switch to that (much more type safe)
-  const leaderboardObjectVersions = useRootObjectVersions(
-    props.entity,
-    props.project,
-    {
-      baseObjectClasses: ['Leaderboard'],
-      latestOnly: true,
-    }
-  );
+  const leaderboardQuery = useBaseObjectInstances('Leaderboard', {
+    project_id: projectIdFromParts({
+      entity: props.entity,
+      project: props.project,
+    }),
+    filter: {latest_only: true},
+  });
+
+  const leaderboardObjectVersions = useMemo(() => {
+    return (leaderboardQuery.result ?? []).map(
+      convertTraceServerObjectVersionToSchema
+    );
+  }, [leaderboardQuery.result]);
   const onClick = useCallback(
     (obj: ObjectVersionSchema) => {
       const to = peekingRouter.leaderboardsUIUrl(
@@ -125,22 +131,18 @@ const LeaderboardTable: React.FC<{
     [history, peekingRouter, props.entity, props.project]
   );
 
-  if (leaderboardObjectVersions.loading) {
+  if (leaderboardQuery.loading) {
     return <Loading centered />;
   }
-  if (leaderboardObjectVersions.error) {
-    return <ErrorPanel />;
-  }
 
-  const objectVersions = leaderboardObjectVersions.result ?? [];
-  const isEmpty = objectVersions.length === 0;
+  const isEmpty = leaderboardObjectVersions.length === 0;
   if (isEmpty) {
     return <Empty {...EMPTY_PROPS_LEADERBOARDS} />;
   }
 
   return (
     <ObjectVersionsTable
-      objectVersions={leaderboardObjectVersions.result ?? []}
+      objectVersions={leaderboardObjectVersions}
       objectTitle="Name"
       hidePropsAsColumns
       hidePeerVersionsColumn
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooks.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooks.ts
index 12c259b68cc..de675a4a839 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooks.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/tsDataModelHooks.ts
@@ -893,9 +893,11 @@ const useObjectVersion = (
   }, [cachedObjectVersion, key, objectVersionRes]);
 };
 
-const convertTraceServerObjectVersionToSchema = (
-  obj: traceServerTypes.TraceObjSchema
-): ObjectVersionSchema => {
+export const convertTraceServerObjectVersionToSchema = <
+  T extends traceServerTypes.TraceObjSchema
+>(
+  obj: T
+): ObjectVersionSchema<T['val']> => {
   const [entity, project] = obj.project_id.split('/');
   return {
     scheme: 'weave' as const,
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts
index 856308cbe8e..0e0e2ce348a 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/wfDataModelHooksInterface.ts
@@ -106,12 +106,12 @@ export type ObjectVersionKey =
   | WandbArtifactObjectVersionKey
   | WeaveObjectVersionKey;
 
-export type ObjectVersionSchema = ObjectVersionKey & {
+export type ObjectVersionSchema<T extends any = any> = ObjectVersionKey & {
   // TODO: Add more fields & FKs
   versionIndex: number;
   baseObjectClass: string | null;
   createdAtMs: number;
-  val: any;
+  val: T;
 };
 
 export type ObjectVersionFilter = {

From c85b6f4ceec7dcba7552d9ac13e8356c9c1e8bb9 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 14:01:36 -0600
Subject: [PATCH 154/169] Fixed generation

---
 weave-js/package.json                         |  3 +-
 weave-js/scripts/generate-schemas.sh          | 41 +++++++++++++++++++
 .../wfReactInterface/baseObjectClassQuery.ts  |  7 ++--
 3 files changed, 46 insertions(+), 5 deletions(-)
 create mode 100644 weave-js/scripts/generate-schemas.sh

diff --git a/weave-js/package.json b/weave-js/package.json
index 2e3fdeaea92..efd9a510032 100644
--- a/weave-js/package.json
+++ b/weave-js/package.json
@@ -19,9 +19,10 @@
     "generate:watch": "graphql-codegen -w",
     "prettier": "prettier --config .prettierrc --check \"src/**/*.ts\" \"src/**/*.tsx\"",
     "prettier-fix": "prettier --loglevel warn --config .prettierrc --write \"src/**/*.ts\" \"src/**/*.tsx\"",
+    "direct-prettier": "prettier",
     "lint": "yarn eslint & yarn tslint & yarn prettier & wait",
     "lint-fix": "yarn eslint-fix & yarn tslint-fix & yarn prettier-fix & wait",
-    "generate-schemas": "yarn quicktype -s schema  ../weave/scripts/generated_base_object_class_schemas.json -o ./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts --lang typescript-zod && prettier --write ./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts"
+    "generate-schemas": "bash scripts/generate-schemas.sh"
   },
   "dependencies": {
     "@apollo/client": "^3.8.4",
diff --git a/weave-js/scripts/generate-schemas.sh b/weave-js/scripts/generate-schemas.sh
new file mode 100644
index 00000000000..a9d6933155a
--- /dev/null
+++ b/weave-js/scripts/generate-schemas.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Exit on error
+set -e
+
+SCHEMA_INPUT_PATH="../weave/scripts/generated_base_object_class_schemas.json"
+SCHEMA_OUTPUT_PATH="./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts"
+
+echo "Generating schemas..."
+
+# Generate TypeScript-Zod types from schema
+yarn quicktype -s schema "$SCHEMA_INPUT_PATH" -o "$SCHEMA_OUTPUT_PATH" --lang typescript-zod
+
+# Transform the schema to extract the type map
+sed -i.bak '
+  # Find the GeneratedBaseObjectClassesZodSchema definition and capture its contents
+  /export const GeneratedBaseObjectClassesZodSchema = z.object({/,/});/ {
+    # Replace the opening line with typeMap declaration
+    s/export const GeneratedBaseObjectClassesZodSchema = z.object({/export const baseObjectClassRegistry = ({/
+    # Store the pattern
+    h
+    # If this is the last line (with closing brace), append the schema definition
+    /});/ {
+      p
+      s/.*//
+      x
+      s/.*//
+      i\
+\
+export const GeneratedBaseObjectClassesZodSchema = z.object(baseObjectClassRegistry)
+    }
+  }
+' "$SCHEMA_OUTPUT_PATH"
+
+# Remove backup file
+rm "${SCHEMA_OUTPUT_PATH}.bak"
+
+# Format the generated file
+yarn direct-prettier --write "$SCHEMA_OUTPUT_PATH"
+
+echo "Schema generation completed successfully" 
\ No newline at end of file
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index d1d4bbaeb48..7250bb0f687 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -2,7 +2,9 @@ import {useDeepMemo} from '@wandb/weave/hookUtils';
 import {useEffect, useRef, useState} from 'react';
 import {z} from 'zod';
 
-import {GeneratedBaseObjectClassesZodSchema} from './generatedBaseObjectClasses.zod';
+import {
+  baseObjectClassRegistry,
+} from './generatedBaseObjectClasses.zod';
 import {TraceServerClient} from './traceServerClient';
 import {useGetTraceServerClientContext} from './traceServerClientContext';
 import {
@@ -13,9 +15,6 @@ import {
 } from './traceServerClientTypes';
 import {Loadable} from './wfDataModelHooksInterface';
 
-const baseObjectClassRegistry: Record<string, z.ZodType> =
-  GeneratedBaseObjectClassesZodSchema.shape;
-
 export const useBaseObjectInstances = <
   C extends keyof typeof baseObjectClassRegistry,
   T = z.infer<(typeof baseObjectClassRegistry)[C]>

From ea49fb125297eb0e0d25a07d8060f5913d6c76e6 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 14:02:13 -0600
Subject: [PATCH 155/169] Fixed generation 2

---
 .../wfReactInterface/generatedBaseObjectClasses.zod.ts   | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
index ba5a38ebf65..fd27f0bc933 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts
@@ -25,10 +25,15 @@ export const TestOnlyExampleSchema = z.object({
 });
 export type TestOnlyExample = z.infer<typeof TestOnlyExampleSchema>;
 
-export const GeneratedBaseObjectClassesZodSchema = z.object({
+export const baseObjectClassRegistry = {
   TestOnlyExample: TestOnlyExampleSchema,
   TestOnlyNestedBaseObject: TestOnlyNestedBaseObjectSchema,
-});
+};
+
+export const GeneratedBaseObjectClassesZodSchema = z.object(
+  baseObjectClassRegistry
+);
+
 export type GeneratedBaseObjectClassesZod = z.infer<
   typeof GeneratedBaseObjectClassesZodSchema
 >;

From d6406eb651e53b7d3e3336499b0f041caaf71af7 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 14:16:29 -0600
Subject: [PATCH 156/169] More code removal

---
 .../LeaderboardListingPage.tsx                | 16 ++--
 .../pages/LeaderboardPage/LeaderboardPage.tsx | 87 ++-----------------
 2 files changed, 14 insertions(+), 89 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
index e6c1ff877bb..52d6795806c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardListingPage.tsx
@@ -10,8 +10,10 @@ import {Empty} from '../common/Empty';
 import {EMPTY_PROPS_LEADERBOARDS} from '../common/EmptyContent';
 import {SimplePageLayout} from '../common/SimplePageLayout';
 import {ObjectVersionsTable} from '../ObjectVersionsPage';
-import {useBaseObjectInstances} from '../wfReactInterface/baseObjectClassQuery';
-import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
+import {
+  useBaseObjectInstances,
+  useCreateBaseObjectInstance,
+} from '../wfReactInterface/baseObjectClassQuery';
 import {sanitizeObjectId} from '../wfReactInterface/traceServerDirectClient';
 import {
   convertTraceServerObjectVersionToSchema,
@@ -160,21 +162,15 @@ const generateLeaderboardId = () => {
 };
 
 const useCreateLeaderboard = (entity: string, project: string) => {
-  const getTsClient = useGetTraceServerClientContext();
-  const client = getTsClient();
+  const createLeaderboardInstance = useCreateBaseObjectInstance('Leaderboard');
 
-  // TODO: Once `useCreateCollectionObject` lands from the online
-  // evals project, switch to that (much more type safe)
   const createLeaderboard = async () => {
     const objectId = sanitizeObjectId(generateLeaderboardId());
-    await client.objCreate({
+    await createLeaderboardInstance({
       obj: {
         project_id: projectIdFromParts({entity, project}),
         object_id: objectId,
         val: {
-          _type: 'Leaderboard',
-          _class_name: 'Leaderboard',
-          _bases: ['Object', 'BaseModel'],
           name: objectId,
           description: '',
           columns: [],
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index cbd4ef8cce6..ed07e029c53 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -2,7 +2,6 @@ import {Box} from '@mui/material';
 import {MOON_250} from '@wandb/weave/common/css/color.styles';
 import {useViewerInfo} from '@wandb/weave/common/hooks/useViewerInfo';
 import {Button} from '@wandb/weave/components/Button';
-import {ErrorPanel} from '@wandb/weave/components/ErrorPanel';
 import {Loading} from '@wandb/weave/components/Loading';
 import _ from 'lodash';
 import React, {
@@ -25,7 +24,7 @@ import {
 import {useSavedLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
 import {LeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {SimplePageLayout} from '../common/SimplePageLayout';
-import {useWFHooks} from '../wfReactInterface/context';
+import {useBaseObjectInstances} from '../wfReactInterface/baseObjectClassQuery';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
 import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
@@ -85,25 +84,19 @@ export const LeaderboardPageContent: React.FC<
     setIsEditing: (isEditing: boolean) => void;
   }
 > = props => {
-  const {useRootObjectVersions} = useWFHooks();
   const {entity, project} = props;
-  const leaderboardObjectVersion = useRootObjectVersions(entity, project, {
-    objectIds: [props.leaderboardName],
-    baseObjectClasses: ['Leaderboard'],
-    latestOnly: true,
+  const leaderboardInstances = useBaseObjectInstances('Leaderboard', {
+    project_id: projectIdFromParts({entity, project}),
+    filter: {object_ids: [props.leaderboardName], latest_only: true},
   });
 
-  if (leaderboardObjectVersion.loading) {
+  if (leaderboardInstances.loading) {
     return <Loading centered />;
   }
 
-  if (leaderboardObjectVersion.error) {
-    return <ErrorPanel />;
-  }
-
   if (
-    leaderboardObjectVersion.result == null ||
-    leaderboardObjectVersion.result.length !== 1
+    leaderboardInstances.result == null ||
+    leaderboardInstances.result.length !== 1
   ) {
     return (
       <NotFoundPanel
@@ -112,9 +105,7 @@ export const LeaderboardPageContent: React.FC<
     );
   }
 
-  const leaderboardVal = parseLeaderboardVal(
-    leaderboardObjectVersion.result[0].val
-  );
+  const leaderboardVal = leaderboardInstances.result[0].val;
 
   if (leaderboardVal == null) {
     return (
@@ -323,68 +314,6 @@ export const ToggleLeaderboardConfig: React.FC<{
   );
 };
 
-const parseLeaderboardVal = (
-  leaderboardVal: any
-): LeaderboardObjectVal | null => {
-  if (typeof leaderboardVal !== 'object' || leaderboardVal == null) {
-    return null;
-  }
-
-  const name = leaderboardVal.name;
-  if (typeof name !== 'string') {
-    return null;
-  }
-
-  const description = leaderboardVal.description;
-  if (typeof description !== 'string') {
-    return null;
-  }
-
-  const columns = leaderboardVal.columns;
-  if (!Array.isArray(columns)) {
-    return null;
-  }
-
-  const finalColumns = columns
-    // .map(column => {
-    //   const evaluationObjectRef = column.evaluation_object_ref;
-    //   if (typeof evaluationObjectRef !== 'string') {
-    //     return null;
-    //   }
-
-    //   const scorerName = column.scorer_name;
-    //   if (typeof scorerName !== 'string') {
-    //     return null;
-    //   }
-
-    //   const shouldMinimize = column.should_minimize;
-    //   if (shouldMinimize != null && typeof shouldMinimize !== 'boolean') {
-    //     return null;
-    //   }
-
-    //   const summaryMetricParts = column.summary_metric_path_parts;
-    //   if (!Array.isArray(summaryMetricParts)) {
-    //     return null;
-    //   } else if (summaryMetricParts.some(part => typeof part !== 'string')) {
-    //     return null;
-    //   }
-
-    //   return {
-    //     evaluation_object_ref: evaluationObjectRef,
-    //     scorer_name: scorerName,
-    //     should_minimize: shouldMinimize,
-    //     summary_metric_path_parts: summaryMetricParts,
-    //   };
-    // })
-    .filter(column => column != null) as LeaderboardObjectVal['columns'];
-
-  return {
-    name,
-    description,
-    columns: finalColumns,
-  };
-};
-
 const EditLeaderboardButton: FC<{
   isEditing: boolean;
   setIsEditing: (isEditing: boolean) => void;

From d84fb36d7b6d36553e67097a14a8c0e801291060 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 14:16:47 -0600
Subject: [PATCH 157/169] Fixed types

---
 .../Browse3/pages/wfReactInterface/baseObjectClassQuery.ts | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index 7250bb0f687..804c0f775f3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -2,9 +2,7 @@ import {useDeepMemo} from '@wandb/weave/hookUtils';
 import {useEffect, useRef, useState} from 'react';
 import {z} from 'zod';
 
-import {
-  baseObjectClassRegistry,
-} from './generatedBaseObjectClasses.zod';
+import {baseObjectClassRegistry} from './generatedBaseObjectClasses.zod';
 import {TraceServerClient} from './traceServerClient';
 import {useGetTraceServerClientContext} from './traceServerClientContext';
 import {
@@ -57,8 +55,7 @@ const getBaseObjectInstances = async <
   baseObjectClassName: C,
   req: TraceObjQueryReq
 ): Promise<Array<TraceObjSchema<T, C>>> => {
-  const knownObjectClass: z.ZodType<T> =
-    baseObjectClassRegistry[baseObjectClassName];
+  const knownObjectClass = baseObjectClassRegistry[baseObjectClassName];
   if (!knownObjectClass) {
     console.warn(`Unknown object class: ${baseObjectClassName}`);
     return [];

From ec9f5f67ea5880c2c143298b4fbd2e9d1c6a2158 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 14:20:09 -0600
Subject: [PATCH 158/169] Refactor complete

---
 .../pages/LeaderboardPage/LeaderboardPage.tsx  | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
index ed07e029c53..6fac8eaa599 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardPage.tsx
@@ -24,8 +24,10 @@ import {
 import {useSavedLeaderboardData} from '../../views/Leaderboard/query/hookAdapters';
 import {LeaderboardObjectVal} from '../../views/Leaderboard/types/leaderboardConfigType';
 import {SimplePageLayout} from '../common/SimplePageLayout';
-import {useBaseObjectInstances} from '../wfReactInterface/baseObjectClassQuery';
-import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
+import {
+  useBaseObjectInstances,
+  useCreateBaseObjectInstance,
+} from '../wfReactInterface/baseObjectClassQuery';
 import {projectIdFromParts} from '../wfReactInterface/tsDataModelHooks';
 import {LeaderboardConfigEditor} from './LeaderboardConfigEditor';
 
@@ -129,20 +131,14 @@ const useUpdateLeaderboard = (
   project: string,
   objectId: string
 ) => {
-  const getTsClient = useGetTraceServerClientContext();
-  const client = getTsClient();
+  const createLeaderboard = useCreateBaseObjectInstance('Leaderboard');
 
   const updateLeaderboard = async (leaderboardVal: LeaderboardObjectVal) => {
-    return await client.objCreate({
+    return await createLeaderboard({
       obj: {
         project_id: projectIdFromParts({entity, project}),
         object_id: objectId,
-        val: {
-          _type: 'Leaderboard',
-          _class_name: 'Leaderboard',
-          _bases: ['Object', 'BaseModel'],
-          ...leaderboardVal,
-        },
+        val: leaderboardVal,
       },
     });
   };

From e0ab86cdb7abd6ee3b0115a145e5743776c1b2ed Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 14:37:23 -0600
Subject: [PATCH 159/169] Small ts error

---
 .../Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 9b100e9e200..38521292ca2 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -265,7 +265,7 @@ const ColumnEditor: React.FC<{
           handleColumnChange(
             index,
             'summary_metric_path',
-            newVal?.val.split('.')
+            newVal?.val ?? ''
           )
         }
         options={metrics.map(metric => ({val: metric}))}

From 1532ffd91f1c9d4831921c3b06fac302c6ea6b16 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 14:43:03 -0600
Subject: [PATCH 160/169] Release Candidate

---
 tests/trace/test_leaderboard.py                    | 11 +++++------
 weave/flow/{leaderboard_util.py => leaderboard.py} |  4 ++++
 2 files changed, 9 insertions(+), 6 deletions(-)
 rename weave/flow/{leaderboard_util.py => leaderboard.py} (96%)

diff --git a/tests/trace/test_leaderboard.py b/tests/trace/test_leaderboard.py
index 242891e76d5..a25fccbdd35 100644
--- a/tests/trace/test_leaderboard.py
+++ b/tests/trace/test_leaderboard.py
@@ -1,9 +1,8 @@
 import pytest
 
 import weave
-from weave.flow.leaderboard_util import get_leaderboard_results
+from weave.flow import leaderboard
 from weave.trace.weave_client import get_ref
-from weave.trace_server.interface.base_object_classes import leaderboard
 
 
 def test_leaderboard_empty(client):
@@ -32,7 +31,7 @@ def test_leaderboard_empty(client):
     # Overriding spec to show that this works
     spec = ref.get()
 
-    results = get_leaderboard_results(spec, client)
+    results = leaderboard.get_leaderboard_results(spec, client)
     assert len(results) == 0
 
 
@@ -54,7 +53,7 @@ def test_leaderboard_mis_configured(client):
     # Overriding spec to show that this works
     spec = ref.get()
 
-    results = get_leaderboard_results(spec, client)
+    results = leaderboard.get_leaderboard_results(spec, client)
     assert len(results) == 0
 
 
@@ -123,7 +122,7 @@ async def test_leaderboard_with_results(client):
     # Overriding spec to show that this works
     spec = ref.get()
 
-    results = get_leaderboard_results(spec, client)
+    results = leaderboard.get_leaderboard_results(spec, client)
     assert len(results) == 1
     assert results[0].model_ref == get_ref(simple_model).uri()
     assert results[0].column_scores[0].scores[0].value == 1.0
@@ -173,7 +172,7 @@ async def test_leaderboard_with_results(client):
     # Overriding spec to show that this works
     spec = ref.get()
 
-    results = get_leaderboard_results(spec, client)
+    results = leaderboard.get_leaderboard_results(spec, client)
     assert len(results) == 3
     assert results[0].model_ref == get_ref(simple_model).uri()
     assert len(results[0].column_scores) == 3
diff --git a/weave/flow/leaderboard_util.py b/weave/flow/leaderboard.py
similarity index 96%
rename from weave/flow/leaderboard_util.py
rename to weave/flow/leaderboard.py
index 4084597d751..95bda82a45f 100644
--- a/weave/flow/leaderboard_util.py
+++ b/weave/flow/leaderboard.py
@@ -80,3 +80,7 @@ def get_leaderboard_results(
                 )
             )
     return list(res_map.values())
+
+# Re-export:
+Leaderboard = leaderboard.Leaderboard
+LeaderboardColumn = leaderboard.LeaderboardColumn
\ No newline at end of file

From 99f58d17a13050755e0c5d892e1eb3740930b5e1 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 15:03:20 -0600
Subject: [PATCH 161/169] Type fixes

---
 .../wfReactInterface/baseObjectClassQuery.ts  | 40 +++++++++++--------
 1 file changed, 23 insertions(+), 17 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
index 804c0f775f3..5906d52ad6f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/baseObjectClassQuery.ts
@@ -13,14 +13,19 @@ import {
 } from './traceServerClientTypes';
 import {Loadable} from './wfDataModelHooksInterface';
 
-export const useBaseObjectInstances = <
-  C extends keyof typeof baseObjectClassRegistry,
-  T = z.infer<(typeof baseObjectClassRegistry)[C]>
->(
+type BaseObjectClassRegistry = typeof baseObjectClassRegistry;
+type BaseObjectClassRegistryKeys = keyof BaseObjectClassRegistry;
+type BaseObjectClassType<C extends BaseObjectClassRegistryKeys> = z.infer<
+  BaseObjectClassRegistry[C]
+>;
+
+export const useBaseObjectInstances = <C extends BaseObjectClassRegistryKeys>(
   baseObjectClassName: C,
   req: TraceObjQueryReq
-): Loadable<Array<TraceObjSchema<T, C>>> => {
-  const [objects, setObjects] = useState<Array<TraceObjSchema<T, C>>>([]);
+): Loadable<Array<TraceObjSchema<BaseObjectClassType<C>, C>>> => {
+  const [objects, setObjects] = useState<
+    Array<TraceObjSchema<BaseObjectClassType<C>, C>>
+  >([]);
   const getTsClient = useGetTraceServerClientContext();
   const client = getTsClient();
   const deepReq = useDeepMemo(req);
@@ -34,7 +39,7 @@ export const useBaseObjectInstances = <
     getBaseObjectInstances(client, baseObjectClassName, deepReq).then(
       collectionObjects => {
         if (isMounted && currReq.current === deepReq) {
-          setObjects(collectionObjects as Array<TraceObjSchema<T, C>>);
+          setObjects(collectionObjects);
           setLoading(false);
         }
       }
@@ -47,14 +52,11 @@ export const useBaseObjectInstances = <
   return {result: objects, loading};
 };
 
-const getBaseObjectInstances = async <
-  C extends keyof typeof baseObjectClassRegistry,
-  T = z.infer<(typeof baseObjectClassRegistry)[C]>
->(
+const getBaseObjectInstances = async <C extends BaseObjectClassRegistryKeys>(
   client: TraceServerClient,
   baseObjectClassName: C,
   req: TraceObjQueryReq
-): Promise<Array<TraceObjSchema<T, C>>> => {
+): Promise<Array<TraceObjSchema<BaseObjectClassType<C>, C>>> => {
   const knownObjectClass = baseObjectClassRegistry[baseObjectClassName];
   if (!knownObjectClass) {
     console.warn(`Unknown object class: ${baseObjectClassName}`);
@@ -78,13 +80,17 @@ const getBaseObjectInstances = async <
     .filter(({parsed}) => parsed.success)
     .filter(({obj}) => obj.base_object_class === baseObjectClassName)
     .map(
-      ({obj, parsed}) => ({...obj, val: parsed.data} as TraceObjSchema<T, C>)
+      ({obj, parsed}) =>
+        ({...obj, val: parsed.data} as TraceObjSchema<
+          BaseObjectClassType<C>,
+          C
+        >)
     );
 };
 
 export const useCreateBaseObjectInstance = <
-  C extends keyof typeof baseObjectClassRegistry,
-  T = z.infer<(typeof baseObjectClassRegistry)[C]>
+  C extends BaseObjectClassRegistryKeys,
+  T = BaseObjectClassType<C>
 >(
   baseObjectClassName: C
 ): ((req: TraceObjCreateReq<T>) => Promise<TraceObjCreateRes>) => {
@@ -95,8 +101,8 @@ export const useCreateBaseObjectInstance = <
 };
 
 const createBaseObjectInstance = async <
-  C extends keyof typeof baseObjectClassRegistry,
-  T = z.infer<(typeof baseObjectClassRegistry)[C]>
+  C extends BaseObjectClassRegistryKeys,
+  T = BaseObjectClassType<C>
 >(
   client: TraceServerClient,
   baseObjectClassName: C,

From 7b76d3ea9117253674388ceed36764fd09270aa2 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 15:05:34 -0600
Subject: [PATCH 162/169] Lint

---
 .../pages/LeaderboardPage/LeaderboardConfigEditor.tsx       | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
index 38521292ca2..e1e1f3b5ffd 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/LeaderboardPage/LeaderboardConfigEditor.tsx
@@ -262,11 +262,7 @@ const ColumnEditor: React.FC<{
       <Select<{val: string}>
         value={selectedMetricPath}
         onChange={newVal =>
-          handleColumnChange(
-            index,
-            'summary_metric_path',
-            newVal?.val ?? ''
-          )
+          handleColumnChange(index, 'summary_metric_path', newVal?.val ?? '')
         }
         options={metrics.map(metric => ({val: metric}))}
         isDisabled={!column.evaluation_object_ref || !column.scorer_name}

From 7f95417dc6331b6f6740c0d32f053ce3633143e4 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 15:11:43 -0600
Subject: [PATCH 163/169] Moved to generated

---
 dev_docs/BaseObjectClasses.md                               | 2 +-
 weave-js/scripts/generate-schemas.sh                        | 2 +-
 weave/flow/leaderboard.py                                   | 3 ++-
 .../generated_base_object_class_schemas.json                | 0
 weave/scripts/generate_base_object_schemas.py               | 6 +++++-
 5 files changed, 9 insertions(+), 4 deletions(-)
 rename weave/{scripts => generated}/generated_base_object_class_schemas.json (100%)

diff --git a/dev_docs/BaseObjectClasses.md b/dev_docs/BaseObjectClasses.md
index fe3024c5a20..af170a95dce 100644
--- a/dev_docs/BaseObjectClasses.md
+++ b/dev_docs/BaseObjectClasses.md
@@ -157,7 +157,7 @@ Run `make synchronize-base-object-schemas` to ensure the frontend TypeScript typ
 1. Define your schema in a python file in the `weave/trace_server/interface/base_object_classes/test_only_example.py` directory. See `weave/trace_server/interface/base_object_classes/test_only_example.py` as an example.
 2. Make sure to register your schemas in `weave/trace_server/interface/base_object_classes/base_object_registry.py` by calling `register_base_object`.
 3. Run `make synchronize-base-object-schemas` to generate the frontend types.
-    * The first step (`make generate_base_object_schemas`) will run `weave/scripts/generate_base_object_schemas.py` to generate a JSON schema in `weave/scripts/generated_base_object_class_schemas.json`.
+    * The first step (`make generate_base_object_schemas`) will run `weave/scripts/generate_base_object_schemas.py` to generate a JSON schema in `weave/generated/generated_base_object_class_schemas.json`.
     * The second step (yarn `generate-schemas`) will read this file and use it to generate the frontend types located in `weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts`.
 4. Now, each use case uses different parts:
     1. `Python Writing`. Users can directly import these classes and use them as normal Pydantic models, which get published with `weave.publish`. The python client correct builds the requisite payload.
diff --git a/weave-js/scripts/generate-schemas.sh b/weave-js/scripts/generate-schemas.sh
index a9d6933155a..6edc00e5207 100644
--- a/weave-js/scripts/generate-schemas.sh
+++ b/weave-js/scripts/generate-schemas.sh
@@ -3,7 +3,7 @@
 # Exit on error
 set -e
 
-SCHEMA_INPUT_PATH="../weave/scripts/generated_base_object_class_schemas.json"
+SCHEMA_INPUT_PATH="../weave/generated/generated_base_object_class_schemas.json"
 SCHEMA_OUTPUT_PATH="./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts"
 
 echo "Generating schemas..."
diff --git a/weave/flow/leaderboard.py b/weave/flow/leaderboard.py
index 95bda82a45f..14fce2cf5e9 100644
--- a/weave/flow/leaderboard.py
+++ b/weave/flow/leaderboard.py
@@ -81,6 +81,7 @@ def get_leaderboard_results(
             )
     return list(res_map.values())
 
+
 # Re-export:
 Leaderboard = leaderboard.Leaderboard
-LeaderboardColumn = leaderboard.LeaderboardColumn
\ No newline at end of file
+LeaderboardColumn = leaderboard.LeaderboardColumn
diff --git a/weave/scripts/generated_base_object_class_schemas.json b/weave/generated/generated_base_object_class_schemas.json
similarity index 100%
rename from weave/scripts/generated_base_object_class_schemas.json
rename to weave/generated/generated_base_object_class_schemas.json
diff --git a/weave/scripts/generate_base_object_schemas.py b/weave/scripts/generate_base_object_schemas.py
index fc6ae2a5d1f..74881fe62f1 100644
--- a/weave/scripts/generate_base_object_schemas.py
+++ b/weave/scripts/generate_base_object_schemas.py
@@ -7,7 +7,8 @@
     BASE_OBJECT_REGISTRY,
 )
 
-OUTPUT_PATH = Path(__file__).parent / "generated_base_object_class_schemas.json"
+OUTPUT_DIR = Path(__file__).parent.parent / "generated"
+OUTPUT_PATH = OUTPUT_DIR / "generated_base_object_class_schemas.json"
 
 
 def generate_schemas() -> None:
@@ -25,6 +26,9 @@ def generate_schemas() -> None:
     # Generate the schema using the composite model
     top_level_schema = CompositeModel.model_json_schema(mode="validation")
 
+    # make sure the output directory exists
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
     # Write schema to file
     with OUTPUT_PATH.open("w") as f:
         json.dump(top_level_schema, f, indent=2)

From 9d35f13adbf406f01ccfdf8bd5ef2fc0f26c947e Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 15:16:16 -0600
Subject: [PATCH 164/169] Moved to gen

---
 dev_docs/BaseObjectClasses.md                             | 2 +-
 weave-js/scripts/generate-schemas.sh                      | 2 +-
 weave/scripts/generate_base_object_schemas.py             | 8 +++++++-
 .../generated/generated_base_object_class_schemas.json    | 0
 4 files changed, 9 insertions(+), 3 deletions(-)
 rename weave/{ => trace_server/interface/base_object_classes}/generated/generated_base_object_class_schemas.json (100%)

diff --git a/dev_docs/BaseObjectClasses.md b/dev_docs/BaseObjectClasses.md
index af170a95dce..a571af49755 100644
--- a/dev_docs/BaseObjectClasses.md
+++ b/dev_docs/BaseObjectClasses.md
@@ -157,7 +157,7 @@ Run `make synchronize-base-object-schemas` to ensure the frontend TypeScript typ
 1. Define your schema in a python file in the `weave/trace_server/interface/base_object_classes/test_only_example.py` directory. See `weave/trace_server/interface/base_object_classes/test_only_example.py` as an example.
 2. Make sure to register your schemas in `weave/trace_server/interface/base_object_classes/base_object_registry.py` by calling `register_base_object`.
 3. Run `make synchronize-base-object-schemas` to generate the frontend types.
-    * The first step (`make generate_base_object_schemas`) will run `weave/scripts/generate_base_object_schemas.py` to generate a JSON schema in `weave/generated/generated_base_object_class_schemas.json`.
+    * The first step (`make generate_base_object_schemas`) will run `weave/scripts/generate_base_object_schemas.py` to generate a JSON schema in `weave/trace_server/interface/base_object_classes/generated/generated_base_object_class_schemas.json`.
     * The second step (yarn `generate-schemas`) will read this file and use it to generate the frontend types located in `weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts`.
 4. Now, each use case uses different parts:
     1. `Python Writing`. Users can directly import these classes and use them as normal Pydantic models, which get published with `weave.publish`. The python client correct builds the requisite payload.
diff --git a/weave-js/scripts/generate-schemas.sh b/weave-js/scripts/generate-schemas.sh
index 6edc00e5207..545d81e17af 100644
--- a/weave-js/scripts/generate-schemas.sh
+++ b/weave-js/scripts/generate-schemas.sh
@@ -3,7 +3,7 @@
 # Exit on error
 set -e
 
-SCHEMA_INPUT_PATH="../weave/generated/generated_base_object_class_schemas.json"
+SCHEMA_INPUT_PATH="../weave/trace_server/interface/base_object_classes/generated/generated_base_object_class_schemas.json"
 SCHEMA_OUTPUT_PATH="./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts"
 
 echo "Generating schemas..."
diff --git a/weave/scripts/generate_base_object_schemas.py b/weave/scripts/generate_base_object_schemas.py
index 74881fe62f1..06b958a9c33 100644
--- a/weave/scripts/generate_base_object_schemas.py
+++ b/weave/scripts/generate_base_object_schemas.py
@@ -7,7 +7,13 @@
     BASE_OBJECT_REGISTRY,
 )
 
-OUTPUT_DIR = Path(__file__).parent.parent / "generated"
+OUTPUT_DIR = (
+    Path(__file__).parent.parent
+    / "trace_server"
+    / "interface"
+    / "base_object_classes"
+    / "generated"
+)
 OUTPUT_PATH = OUTPUT_DIR / "generated_base_object_class_schemas.json"
 
 
diff --git a/weave/generated/generated_base_object_class_schemas.json b/weave/trace_server/interface/base_object_classes/generated/generated_base_object_class_schemas.json
similarity index 100%
rename from weave/generated/generated_base_object_class_schemas.json
rename to weave/trace_server/interface/base_object_classes/generated/generated_base_object_class_schemas.json

From 5e74e5bfc328149ce08f55bee540bea6f36e5ddc Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 15:22:02 -0600
Subject: [PATCH 165/169] Addressed comments

---
 weave/trace_server/base_object_class_util.py | 31 ++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/weave/trace_server/base_object_class_util.py b/weave/trace_server/base_object_class_util.py
index 698b422aad1..ac5e572313d 100644
--- a/weave/trace_server/base_object_class_util.py
+++ b/weave/trace_server/base_object_class_util.py
@@ -6,6 +6,12 @@
     BASE_OBJECT_REGISTRY,
 )
 
+"""
+There are two standard base object classes: BaseObject and Object
+
+`Object` is the base class for the more advanced object-oriented `weave.Object` use cases.
+`BaseObject` is the more simple schema-based base object class.
+"""
 base_object_class_names = ["BaseObject", "Object"]
 
 
@@ -26,6 +32,27 @@ def get_base_object_class(val: Any) -> Optional[str]:
 def process_incoming_object(
     val: Any, req_base_object_class: Optional[str] = None
 ) -> Tuple[dict, Optional[str]]:
+    """
+    This method is responsible for accepting an incoming object from the user, validating it
+    against the base object class, and returning the object with the base object class
+    set. It does not mutate the original object, but returns a new object with values set if needed.
+
+    Specifically,:
+
+    1. If the object is not a dict, it is returned as is, and the base object class is set to None.
+    2. There are 2 ways to specify the base object class:
+        a. The `req_base_object_class` argument.
+            * used by non-pythonic writers of weave objects
+        b. The `_bases` & `_class_name` attributes of the object, which is a list of base class names.
+            * used by pythonic weave object writers (legacy)
+    3. If the object has a base object class that does not match the requested base object class,
+        an error is thrown.
+    4. if the object contains a base object class inside the payload, then we simply validate
+        the object against the base object class (if a match is found in BASE_OBJECT_REGISTRY)
+    5. If the object does not have a base object class and a requested base object class is
+        provided, we require a match in BASE_OBJECT_REGISTRY and validate the object against
+        the requested base object class. Finally, we set the correct feilds.
+    """
     if not isinstance(val, dict):
         if req_base_object_class is not None:
             raise ValueError(
@@ -86,5 +113,9 @@ def _general_dump(val: Any) -> Any:
         return {k: _general_dump(v) for k, v in val.items()}
     elif isinstance(val, list):
         return [_general_dump(v) for v in val]
+    elif isinstance(val, tuple):
+        return tuple(_general_dump(v) for v in val)
+    elif isinstance(val, set):
+        return {_general_dump(v) for v in val}
     else:
         return val

From a1556295fa304671f4f89817dcd6581cff935546 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 15:33:25 -0600
Subject: [PATCH 166/169] Addressed comments

---
 dev_docs/BaseObjectClasses.md                 |  2 +-
 weave-js/scripts/generate-schemas.sh          |  2 +-
 weave/scripts/generate_base_object_schemas.py | 12 ++++++-
 weave/trace_server/base_object_class_util.py  | 31 +++++++++++++++++++
 .../generated_base_object_class_schemas.json  |  0
 5 files changed, 44 insertions(+), 3 deletions(-)
 rename weave/{scripts => trace_server/interface/base_object_classes/generated}/generated_base_object_class_schemas.json (100%)

diff --git a/dev_docs/BaseObjectClasses.md b/dev_docs/BaseObjectClasses.md
index fe3024c5a20..a571af49755 100644
--- a/dev_docs/BaseObjectClasses.md
+++ b/dev_docs/BaseObjectClasses.md
@@ -157,7 +157,7 @@ Run `make synchronize-base-object-schemas` to ensure the frontend TypeScript typ
 1. Define your schema in a python file in the `weave/trace_server/interface/base_object_classes/test_only_example.py` directory. See `weave/trace_server/interface/base_object_classes/test_only_example.py` as an example.
 2. Make sure to register your schemas in `weave/trace_server/interface/base_object_classes/base_object_registry.py` by calling `register_base_object`.
 3. Run `make synchronize-base-object-schemas` to generate the frontend types.
-    * The first step (`make generate_base_object_schemas`) will run `weave/scripts/generate_base_object_schemas.py` to generate a JSON schema in `weave/scripts/generated_base_object_class_schemas.json`.
+    * The first step (`make generate_base_object_schemas`) will run `weave/scripts/generate_base_object_schemas.py` to generate a JSON schema in `weave/trace_server/interface/base_object_classes/generated/generated_base_object_class_schemas.json`.
     * The second step (yarn `generate-schemas`) will read this file and use it to generate the frontend types located in `weave-js/src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts`.
 4. Now, each use case uses different parts:
     1. `Python Writing`. Users can directly import these classes and use them as normal Pydantic models, which get published with `weave.publish`. The python client correct builds the requisite payload.
diff --git a/weave-js/scripts/generate-schemas.sh b/weave-js/scripts/generate-schemas.sh
index a9d6933155a..545d81e17af 100644
--- a/weave-js/scripts/generate-schemas.sh
+++ b/weave-js/scripts/generate-schemas.sh
@@ -3,7 +3,7 @@
 # Exit on error
 set -e
 
-SCHEMA_INPUT_PATH="../weave/scripts/generated_base_object_class_schemas.json"
+SCHEMA_INPUT_PATH="../weave/trace_server/interface/base_object_classes/generated/generated_base_object_class_schemas.json"
 SCHEMA_OUTPUT_PATH="./src/components/PagePanelComponents/Home/Browse3/pages/wfReactInterface/generatedBaseObjectClasses.zod.ts"
 
 echo "Generating schemas..."
diff --git a/weave/scripts/generate_base_object_schemas.py b/weave/scripts/generate_base_object_schemas.py
index fc6ae2a5d1f..06b958a9c33 100644
--- a/weave/scripts/generate_base_object_schemas.py
+++ b/weave/scripts/generate_base_object_schemas.py
@@ -7,7 +7,14 @@
     BASE_OBJECT_REGISTRY,
 )
 
-OUTPUT_PATH = Path(__file__).parent / "generated_base_object_class_schemas.json"
+OUTPUT_DIR = (
+    Path(__file__).parent.parent
+    / "trace_server"
+    / "interface"
+    / "base_object_classes"
+    / "generated"
+)
+OUTPUT_PATH = OUTPUT_DIR / "generated_base_object_class_schemas.json"
 
 
 def generate_schemas() -> None:
@@ -25,6 +32,9 @@ def generate_schemas() -> None:
     # Generate the schema using the composite model
     top_level_schema = CompositeModel.model_json_schema(mode="validation")
 
+    # make sure the output directory exists
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
     # Write schema to file
     with OUTPUT_PATH.open("w") as f:
         json.dump(top_level_schema, f, indent=2)
diff --git a/weave/trace_server/base_object_class_util.py b/weave/trace_server/base_object_class_util.py
index 698b422aad1..ac5e572313d 100644
--- a/weave/trace_server/base_object_class_util.py
+++ b/weave/trace_server/base_object_class_util.py
@@ -6,6 +6,12 @@
     BASE_OBJECT_REGISTRY,
 )
 
+"""
+There are two standard base object classes: BaseObject and Object
+
+`Object` is the base class for the more advanced object-oriented `weave.Object` use cases.
+`BaseObject` is the more simple schema-based base object class.
+"""
 base_object_class_names = ["BaseObject", "Object"]
 
 
@@ -26,6 +32,27 @@ def get_base_object_class(val: Any) -> Optional[str]:
 def process_incoming_object(
     val: Any, req_base_object_class: Optional[str] = None
 ) -> Tuple[dict, Optional[str]]:
+    """
+    This method is responsible for accepting an incoming object from the user, validating it
+    against the base object class, and returning the object with the base object class
+    set. It does not mutate the original object, but returns a new object with values set if needed.
+
+    Specifically,:
+
+    1. If the object is not a dict, it is returned as is, and the base object class is set to None.
+    2. There are 2 ways to specify the base object class:
+        a. The `req_base_object_class` argument.
+            * used by non-pythonic writers of weave objects
+        b. The `_bases` & `_class_name` attributes of the object, which is a list of base class names.
+            * used by pythonic weave object writers (legacy)
+    3. If the object has a base object class that does not match the requested base object class,
+        an error is thrown.
+    4. if the object contains a base object class inside the payload, then we simply validate
+        the object against the base object class (if a match is found in BASE_OBJECT_REGISTRY)
+    5. If the object does not have a base object class and a requested base object class is
+        provided, we require a match in BASE_OBJECT_REGISTRY and validate the object against
+        the requested base object class. Finally, we set the correct feilds.
+    """
     if not isinstance(val, dict):
         if req_base_object_class is not None:
             raise ValueError(
@@ -86,5 +113,9 @@ def _general_dump(val: Any) -> Any:
         return {k: _general_dump(v) for k, v in val.items()}
     elif isinstance(val, list):
         return [_general_dump(v) for v in val]
+    elif isinstance(val, tuple):
+        return tuple(_general_dump(v) for v in val)
+    elif isinstance(val, set):
+        return {_general_dump(v) for v in val}
     else:
         return val
diff --git a/weave/scripts/generated_base_object_class_schemas.json b/weave/trace_server/interface/base_object_classes/generated/generated_base_object_class_schemas.json
similarity index 100%
rename from weave/scripts/generated_base_object_class_schemas.json
rename to weave/trace_server/interface/base_object_classes/generated/generated_base_object_class_schemas.json

From 9ccdcd0fea4f2757adbe2658295e1b563e82e0de Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 17:44:01 -0600
Subject: [PATCH 167/169] empty state

---
 .../Browse3/pages/common/EmptyContent.tsx     | 19 +++++++++++++++++++
 .../views/Leaderboard/LeaderboardGrid.tsx     | 16 ++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx
index 78ef9cd7f46..f2db0da100b 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/common/EmptyContent.tsx
@@ -43,6 +43,25 @@ export const EMPTY_PROPS_EVALUATIONS: EmptyProps = {
   ),
 };
 
+export const EMPTY_PROPS_LEADERBOARD: EmptyProps = {
+  icon: 'benchmark-square' as const,
+  heading: 'No leaderboard submissions found.',
+  description: 'Create leaderboard submissions by running evaluations.',
+  moreInformation: (
+    <>
+      Learn{' '}
+      <TargetBlank href="https://wandb.me/weave_evals">
+        evaluation basics
+      </TargetBlank>{' '}
+      or follow our tutorial to{' '}
+      <TargetBlank href="http://wandb.me/weave_eval_tut">
+        set up an evaluation pipeline
+      </TargetBlank>
+      .
+    </>
+  ),
+};
+
 export const EMPTY_PROPS_LEADERBOARDS: EmptyProps = {
   icon: 'benchmark-square' as const,
   heading: 'No leaderboards yet',
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index 94a9d2c5cc4..ba581874a06 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -15,6 +15,8 @@ import {NotApplicable} from '../../../Browse2/NotApplicable';
 import {parseRefMaybe, SmallRef} from '../../../Browse2/SmallRef';
 import {useWeaveflowRouteContext} from '../../context';
 import {PaginationButtons} from '../../pages/CallsPage/CallsTableButtons';
+import { Empty } from '../../pages/common/Empty';
+import { EMPTY_PROPS_LEADERBOARD } from '../../pages/common/EmptyContent';
 import {StyledDataGrid} from '../../StyledDataGrid';
 import {
   GroupedLeaderboardData,
@@ -331,6 +333,20 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     }
   }, [columns, defaultSortModel, loading]);
 
+
+  if (rows.length === 0) {
+    return <Box
+    sx={{
+      height: '100%',
+      width: '100%',
+      display: 'flex',
+      flexDirection: 'column',
+        overflow: 'hidden',
+      }}>
+        <Empty {...EMPTY_PROPS_LEADERBOARD} />
+      </Box>
+  }
+
   return (
     <Box
       sx={{

From 42fcea808614ce6f0761cac3f38e7e9e169a49aa Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 17:59:02 -0600
Subject: [PATCH 168/169] Lint

---
 .../views/Leaderboard/LeaderboardGrid.tsx     | 23 ++++++++++---------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index ba581874a06..47c44b1d23c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -15,8 +15,8 @@ import {NotApplicable} from '../../../Browse2/NotApplicable';
 import {parseRefMaybe, SmallRef} from '../../../Browse2/SmallRef';
 import {useWeaveflowRouteContext} from '../../context';
 import {PaginationButtons} from '../../pages/CallsPage/CallsTableButtons';
-import { Empty } from '../../pages/common/Empty';
-import { EMPTY_PROPS_LEADERBOARD } from '../../pages/common/EmptyContent';
+import {Empty} from '../../pages/common/Empty';
+import {EMPTY_PROPS_LEADERBOARD} from '../../pages/common/EmptyContent';
 import {StyledDataGrid} from '../../StyledDataGrid';
 import {
   GroupedLeaderboardData,
@@ -333,18 +333,19 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     }
   }, [columns, defaultSortModel, loading]);
 
-
   if (rows.length === 0) {
-    return <Box
-    sx={{
-      height: '100%',
-      width: '100%',
-      display: 'flex',
-      flexDirection: 'column',
-        overflow: 'hidden',
-      }}>
+    return (
+      <Box
+        sx={{
+          height: '100%',
+          width: '100%',
+          display: 'flex',
+          flexDirection: 'column',
+          overflow: 'hidden',
+        }}>
         <Empty {...EMPTY_PROPS_LEADERBOARD} />
       </Box>
+    );
   }
 
   return (

From 0e65c6b9414af1d9387a4b55a297dc33270f8f38 Mon Sep 17 00:00:00 2001
From: Tim Sweeney <tim@wandb.com>
Date: Thu, 31 Oct 2024 18:09:40 -0600
Subject: [PATCH 169/169] Lint

---
 .../Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx       | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
index 47c44b1d23c..334b681335f 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/views/Leaderboard/LeaderboardGrid.tsx
@@ -7,6 +7,7 @@ import {
   GridSortDirection,
   GridSortItem,
 } from '@mui/x-data-grid-pro';
+import {Loading} from '@wandb/weave/components/Loading';
 import {Timestamp} from '@wandb/weave/components/Timestamp';
 import React, {useCallback, useEffect, useMemo, useState} from 'react';
 import {useHistory} from 'react-router-dom';
@@ -333,6 +334,10 @@ export const LeaderboardGrid: React.FC<LeaderboardGridProps> = ({
     }
   }, [columns, defaultSortModel, loading]);
 
+  if (loading) {
+    return <Loading centered />;
+  }
+
   if (rows.length === 0) {
     return (
       <Box