diff --git a/docs/docs/guides/tools/imgs/open_chat_in_playground.png b/docs/docs/guides/tools/imgs/open_chat_in_playground.png new file mode 100644 index 00000000000..37260b50d7d Binary files /dev/null and b/docs/docs/guides/tools/imgs/open_chat_in_playground.png differ diff --git a/docs/docs/guides/tools/imgs/playground_chat_input.png b/docs/docs/guides/tools/imgs/playground_chat_input.png new file mode 100644 index 00000000000..4886d0a99f2 Binary files /dev/null and b/docs/docs/guides/tools/imgs/playground_chat_input.png differ diff --git a/docs/docs/guides/tools/imgs/playground_message_buttons.png b/docs/docs/guides/tools/imgs/playground_message_buttons.png new file mode 100644 index 00000000000..a4aa2eb19f5 Binary files /dev/null and b/docs/docs/guides/tools/imgs/playground_message_buttons.png differ diff --git a/docs/docs/guides/tools/imgs/playground_message_editor.png b/docs/docs/guides/tools/imgs/playground_message_editor.png new file mode 100644 index 00000000000..ae194525a99 Binary files /dev/null and b/docs/docs/guides/tools/imgs/playground_message_editor.png differ diff --git a/docs/docs/guides/tools/imgs/playground_settings.png b/docs/docs/guides/tools/imgs/playground_settings.png new file mode 100644 index 00000000000..c0d408e4b0d Binary files /dev/null and b/docs/docs/guides/tools/imgs/playground_settings.png differ diff --git a/docs/docs/guides/tools/playground.md b/docs/docs/guides/tools/playground.md new file mode 100644 index 00000000000..db222c12c6b --- /dev/null +++ b/docs/docs/guides/tools/playground.md @@ -0,0 +1,48 @@ +# Playground + +Evaluating LLM prompts and responses is challenging. The Playground tool enables you to quickly iterate on prompts: editing, retrying, and deleting messages. The LLM Playground is currently in preview. + +There are two ways to access the Playground: + +1. From the sidebar, click **Playground**. This will open a fresh Playground page with a simple system prompt. +2. From the Call page, click the **Open chat in playground** button from the call page's chat view. + +![Screenshot of Open in Playground button](imgs/open_chat_in_playground.png) + +## Retry, edit, and delete messages + +Once in the Playground, you can see the chat history. +When hovering over a message, you will see three buttons: **Edit**, **Retry**, and **Delete**. + +![Screenshot of Playground message buttons](imgs/playground_message_buttons.png) + +1. **Retry**: Deletes all subsequent messages and retries the chat from the selected message. +2. **Delete**: Removes the message from the chat. +3. **Edit**: Allows you to modify the message content. + +![Screenshot of Playground editing](imgs/playground_message_editor.png) + +## Adding new messages + +To add a new message to the chat without sending it to the LLM, select the role (e.g., **User**) and click **Add**. +To send a new message to the LLM, click the **Send** button or press **Command + Enter**. + +![Screenshot of Playground sending a message](imgs/playground_chat_input.png) + +## Configuring the LLM + +We currently support 4 LLM providers. +To use each LLM, your team admin needs to add the relevant API key to your team's settings (found at **wandb.ai/[team-name]/settings**): + +- OpenAI: `OPENAI_API_KEY` +- Anthropic: `ANTHROPIC_API_KEY` +- Gemini: `GOOGLE_API_KEY` +- Groq: `GEMMA_API_KEY` + +### Choosing the LLM and its settings + +Click the **Settings** button to open the settings drawer. + +![Screenshot of Playground settings](imgs/playground_settings.png) + +You can also switch the LLM using the dropdown menu in the top left. diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 9848c6b8398..ede80ed6c50 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -6,7 +6,7 @@ const CATEGORY_SECTION_HEADER_MIXIN: SidebarItemCategoryBase = { collapsible: false, collapsed: false, className: "sidebar-section-title", -} +}; const sidebars: SidebarsConfig = { documentationSidebar: [ @@ -16,25 +16,25 @@ const sidebars: SidebarsConfig = { items: [ "introduction", { - type: 'doc', - label: 'Trace LLMs', - id: "quickstart" + type: "doc", + label: "Trace LLMs", + id: "quickstart", }, { - type: 'doc', - label: 'Trace Applications', - id: "tutorial-tracing_2" + type: "doc", + label: "Trace Applications", + id: "tutorial-tracing_2", }, "tutorial-weave_models", { - type: 'doc', - label: 'Build an Evaluation', - id: "tutorial-eval" + type: "doc", + label: "Build an Evaluation", + id: "tutorial-eval", }, { - type: 'doc', - label: 'Evaluate a RAG App', - id: "tutorial-rag" + type: "doc", + label: "Evaluate a RAG App", + id: "tutorial-rag", }, ], }, @@ -59,16 +59,15 @@ const sidebars: SidebarsConfig = { collapsible: true, collapsed: false, label: "Evaluation", - link: { type: "doc", id: "guides/core-types/evaluations"}, - items: [ - "guides/evaluation/scorers", - ], + link: { type: "doc", id: "guides/core-types/evaluations" }, + items: ["guides/evaluation/scorers"], }, "guides/core-types/prompts", "guides/core-types/models", "guides/core-types/datasets", "guides/tracking/feedback", "guides/tracking/costs", + "guides/tools/playground", "guides/core-types/media", { type: "category", @@ -109,7 +108,8 @@ const sidebars: SidebarsConfig = { collapsible: true, collapsed: true, label: "Frameworks", - items: [, + items: [ + , "guides/integrations/langchain", "guides/integrations/llamaindex", "guides/integrations/dspy", @@ -130,34 +130,42 @@ const sidebars: SidebarsConfig = { }, ], // TODO: add the actual ts-sdk sidebar - typescriptSdkSidebar: [{ type: "autogenerated", dirName: "reference/typescript-sdk" }], - pythonSdkSidebar: [{ type: "autogenerated", dirName: "reference/python-sdk" }], - serviceApiSidebar: require("./docs/reference/service-api/sidebar.ts").filter((row) => { - if (row.id == "reference/service-api/fastapi") { - // Remove FastAPI from the sidebar - this is a default homepage that is not useful for us - return false; - } + typescriptSdkSidebar: [ + { type: "autogenerated", dirName: "reference/typescript-sdk" }, + ], + pythonSdkSidebar: [ + { type: "autogenerated", dirName: "reference/python-sdk" }, + ], + serviceApiSidebar: require("./docs/reference/service-api/sidebar.ts") + .filter((row) => { + if (row.id == "reference/service-api/fastapi") { + // Remove FastAPI from the sidebar - this is a default homepage that is not useful for us + return false; + } - // Hide the `Service` category from the sidebar - if (row.label == "Service") { - return false; - } + // Hide the `Service` category from the sidebar + if (row.label == "Service") { + return false; + } - return true; - }).map((row) => { - // This makes each section nicely formatted. - // Totally up for debate if we want to keep this or not. - if (row.type === "category") { - return { - ...row, - ...CATEGORY_SECTION_HEADER_MIXIN, - }; - } + return true; + }) + .map((row) => { + // This makes each section nicely formatted. + // Totally up for debate if we want to keep this or not. + if (row.type === "category") { + return { + ...row, + ...CATEGORY_SECTION_HEADER_MIXIN, + }; + } - return row; - }), + return row; + }), // This will probably need to be customized in the future - notebookSidebar: [{ type: "autogenerated", dirName: "reference/gen_notebooks" }], + notebookSidebar: [ + { type: "autogenerated", dirName: "reference/gen_notebooks" }, + ], }; export default sidebars; diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts index ddc17c100bc..65bad90278d 100644 --- a/weave-js/src/components/FancyPage/useProjectSidebar.ts +++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts @@ -212,6 +212,13 @@ export const useProjectSidebar = ( isShown: isWeaveOnly, iconName: IconNames.CubeContainer, }, + { + type: 'button' as const, + name: 'Playground', + slug: 'weave/playground', + isShown: isWeaveOnly, + iconName: IconNames.RobotServiceMember, + }, { type: 'menuPlaceholder' as const, // name: 'More', @@ -224,6 +231,7 @@ export const useProjectSidebar = ( // 'weave/scorers', // Hiding until we want to release 'weave/operations', 'weave/objects', + 'weave/playground', ], }, ]; diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx index 7f38ff247e1..600d94ac07d 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx @@ -1,6 +1,7 @@ import Box from '@mui/material/Box'; import {useViewerInfo} from '@wandb/weave/common/hooks/useViewerInfo'; import {Loading} from '@wandb/weave/components/Loading'; +import {urlPrefixed} from '@wandb/weave/config'; import {useViewTraceEvent} from '@wandb/weave/integrations/analytics/useViewEvents'; import React, {FC, useCallback, useContext, useEffect, useState} from 'react'; import {useHistory} from 'react-router-dom'; @@ -75,6 +76,14 @@ const useCallTabs = (call: CallSchema) => { const codeURI = call.opVersionRef; const {entity, project, callId} = call; const weaveRef = makeRefCall(entity, project, callId); + + const handleOpenInPlayground = () => { + window.open( + urlPrefixed(`/${entity}/${project}/weave/playground/${callId}`), + '_blank' + ); + }; + return [ // Disabling Evaluation tab until it's better for single evaluation ...(false && isEvaluateOp(call.spanName) @@ -101,11 +110,20 @@ const useCallTabs = (call: CallSchema) => { { label: 'Chat', content: ( - - - - - + <> + + + + + + + ), }, ]