Merge branch 'master' into ui-metadata-update

docs/docs/guides/tools/playground.md

@@ -0,0 +1,48 @@
+# Playground
+
+Evaluating LLM prompts and responses is challenging. The Playground tool enables you to quickly iterate on prompts: editing, retrying, and deleting messages. The LLM Playground is currently in preview.
+
+There are two ways to access the Playground:
+
+1. From the sidebar, click **Playground**. This will open a fresh Playground page with a simple system prompt.
+2. From the Call page, click the **Open chat in playground** button from the call page's chat view.
+
+![Screenshot of Open in Playground button](imgs/open_chat_in_playground.png)
+
+## Retry, edit, and delete messages
+
+Once in the Playground, you can see the chat history.
+When hovering over a message, you will see three buttons: **Edit**, **Retry**, and **Delete**.
+
+![Screenshot of Playground message buttons](imgs/playground_message_buttons.png)
+
+1. **Retry**: Deletes all subsequent messages and retries the chat from the selected message.
+2. **Delete**: Removes the message from the chat.
+3. **Edit**: Allows you to modify the message content.
+
+![Screenshot of Playground editing](imgs/playground_message_editor.png)
+
+## Adding new messages
+
+To add a new message to the chat without sending it to the LLM, select the role (e.g., **User**) and click **Add**.
+To send a new message to the LLM, click the **Send** button or press **Command + Enter**.
+
+![Screenshot of Playground sending a message](imgs/playground_chat_input.png)
+
+## Configuring the LLM
+
+We currently support 4 LLM providers.
+To use each LLM, your team admin needs to add the relevant API key to your team's settings (found at **wandb.ai/[team-name]/settings**):
+
+- OpenAI: `OPENAI_API_KEY`
+- Anthropic: `ANTHROPIC_API_KEY`
+- Gemini: `GOOGLE_API_KEY`
+- Groq: `GEMMA_API_KEY`
+
+### Choosing the LLM and its settings
+
+Click the **Settings** button to open the settings drawer.
+
+![Screenshot of Playground settings](imgs/playground_settings.png)
+
+You can also switch the LLM using the dropdown menu in the top left.

docs/sidebars.ts

@@ -6,7 +6,7 @@ const CATEGORY_SECTION_HEADER_MIXIN: SidebarItemCategoryBase = {
   collapsible: false,
   collapsed: false,
   className: "sidebar-section-title",
-}
+};
 
 const sidebars: SidebarsConfig = {
   documentationSidebar: [
@@ -16,25 +16,25 @@ const sidebars: SidebarsConfig = {
       items: [
         "introduction",
         {
-          type: 'doc',
-          label: 'Trace LLMs',
-          id: "quickstart"
+          type: "doc",
+          label: "Trace LLMs",
+          id: "quickstart",
         },
         {
-          type: 'doc',
-          label: 'Trace Applications',
-          id: "tutorial-tracing_2"
+          type: "doc",
+          label: "Trace Applications",
+          id: "tutorial-tracing_2",
         },
         "tutorial-weave_models",
         {
-          type: 'doc',
-          label: 'Build an Evaluation',
-          id: "tutorial-eval"
+          type: "doc",
+          label: "Build an Evaluation",
+          id: "tutorial-eval",
         },
         {
-          type: 'doc',
-          label: 'Evaluate a RAG App',
-          id: "tutorial-rag"
+          type: "doc",
+          label: "Evaluate a RAG App",
+          id: "tutorial-rag",
         },
       ],
     },
@@ -59,16 +59,15 @@ const sidebars: SidebarsConfig = {
           collapsible: true,
           collapsed: false,
           label: "Evaluation",
-          link: { type: "doc", id: "guides/core-types/evaluations"},
-          items: [
-            "guides/evaluation/scorers",
-          ],
+          link: { type: "doc", id: "guides/core-types/evaluations" },
+          items: ["guides/evaluation/scorers"],
         },
         "guides/core-types/prompts",
         "guides/core-types/models",
         "guides/core-types/datasets",
         "guides/tracking/feedback",
         "guides/tracking/costs",
+        "guides/tools/playground",
         "guides/core-types/media",
         {
           type: "category",
@@ -109,7 +108,8 @@ const sidebars: SidebarsConfig = {
           collapsible: true,
           collapsed: true,
           label: "Frameworks",
-          items: [,
+          items: [
+            ,
             "guides/integrations/langchain",
             "guides/integrations/llamaindex",
             "guides/integrations/dspy",
@@ -130,34 +130,42 @@ const sidebars: SidebarsConfig = {
     },
   ],
   // TODO: add the actual ts-sdk sidebar
-  typescriptSdkSidebar: [{ type: "autogenerated", dirName: "reference/typescript-sdk" }],
-  pythonSdkSidebar: [{ type: "autogenerated", dirName: "reference/python-sdk" }],
-  serviceApiSidebar: require("./docs/reference/service-api/sidebar.ts").filter((row) => {
-    if (row.id == "reference/service-api/fastapi") {
-      // Remove FastAPI from the sidebar - this is a default homepage that is not useful for us
-      return false;
-    }
+  typescriptSdkSidebar: [
+    { type: "autogenerated", dirName: "reference/typescript-sdk" },
+  ],
+  pythonSdkSidebar: [
+    { type: "autogenerated", dirName: "reference/python-sdk" },
+  ],
+  serviceApiSidebar: require("./docs/reference/service-api/sidebar.ts")
+    .filter((row) => {
+      if (row.id == "reference/service-api/fastapi") {
+        // Remove FastAPI from the sidebar - this is a default homepage that is not useful for us
+        return false;
+      }
 
-    // Hide the `Service` category from the sidebar
-    if (row.label == "Service") {
-      return false;
-    }
+      // Hide the `Service` category from the sidebar
+      if (row.label == "Service") {
+        return false;
+      }
 
-    return true;
-  }).map((row) => {
-    // This makes each section nicely formatted.
-    // Totally up for debate if we want to keep this or not.
-    if (row.type === "category") {
-      return {
-        ...row,
-        ...CATEGORY_SECTION_HEADER_MIXIN,
-      };
-    }
+      return true;
+    })
+    .map((row) => {
+      // This makes each section nicely formatted.
+      // Totally up for debate if we want to keep this or not.
+      if (row.type === "category") {
+        return {
+          ...row,
+          ...CATEGORY_SECTION_HEADER_MIXIN,
+        };
+      }
 
-    return row;
-  }),
+      return row;
+    }),
   // This will probably need to be customized in the future
-  notebookSidebar: [{ type: "autogenerated", dirName: "reference/gen_notebooks" }],
+  notebookSidebar: [
+    { type: "autogenerated", dirName: "reference/gen_notebooks" },
+  ],
 };
 
 export default sidebars;

weave-js/src/components/FancyPage/useProjectSidebar.ts

@@ -212,6 +212,13 @@ export const useProjectSidebar = (
             isShown: isWeaveOnly,
             iconName: IconNames.CubeContainer,
           },
+          {
+            type: 'button' as const,
+            name: 'Playground',
+            slug: 'weave/playground',
+            isShown: isWeaveOnly,
+            iconName: IconNames.RobotServiceMember,
+          },
           {
             type: 'menuPlaceholder' as const,
             // name: 'More',
@@ -224,6 +231,7 @@ export const useProjectSidebar = (
               // 'weave/scorers', // Hiding until we want to release
               'weave/operations',
               'weave/objects',
+              'weave/playground',
             ],
           },
         ];

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx

@@ -1,6 +1,7 @@
 import Box from '@mui/material/Box';
 import {useViewerInfo} from '@wandb/weave/common/hooks/useViewerInfo';
 import {Loading} from '@wandb/weave/components/Loading';
+import {urlPrefixed} from '@wandb/weave/config';
 import {useViewTraceEvent} from '@wandb/weave/integrations/analytics/useViewEvents';
 import React, {FC, useCallback, useContext, useEffect, useState} from 'react';
 import {useHistory} from 'react-router-dom';
@@ -75,6 +76,14 @@ const useCallTabs = (call: CallSchema) => {
   const codeURI = call.opVersionRef;
   const {entity, project, callId} = call;
   const weaveRef = makeRefCall(entity, project, callId);
+
+  const handleOpenInPlayground = () => {
+    window.open(
+      urlPrefixed(`/${entity}/${project}/weave/playground/${callId}`),
+      '_blank'
+    );
+  };
+
   return [
     // Disabling Evaluation tab until it's better for single evaluation
     ...(false && isEvaluateOp(call.spanName)
@@ -101,11 +110,20 @@ const useCallTabs = (call: CallSchema) => {
           {
             label: 'Chat',
             content: (
-              <ScrollableTabContent>
-                <Tailwind>
-                  <CallChat call={call.traceCall!} />
-                </Tailwind>
-              </ScrollableTabContent>
+              <>
+                <Button
+                  variant="secondary"
+                  startIcon="sandbox-playground"
+                  className="m-16 mb-8"
+                  onClick={handleOpenInPlayground}>
+                  Open chat in playground
+                </Button>
+                <ScrollableTabContent>
+                  <Tailwind>
+                    <CallChat call={call.traceCall!} />
+                  </Tailwind>
+                </ScrollableTabContent>
+              </>
             ),
           },
         ]