diff --git a/docs/docs/guides/core-types/imgs/prompt-comparison.png b/docs/docs/guides/core-types/imgs/prompt-comparison.png new file mode 100644 index 000000000000..90b78c5f2118 Binary files /dev/null and b/docs/docs/guides/core-types/imgs/prompt-comparison.png differ diff --git a/docs/docs/guides/core-types/imgs/prompt-object.png b/docs/docs/guides/core-types/imgs/prompt-object.png new file mode 100644 index 000000000000..5a2c059f7859 Binary files /dev/null and b/docs/docs/guides/core-types/imgs/prompt-object.png differ diff --git a/docs/docs/guides/core-types/prompts.md b/docs/docs/guides/core-types/prompts.md index 9a2d50ecf2bc..600e5dfed9f9 100644 --- a/docs/docs/guides/core-types/prompts.md +++ b/docs/docs/guides/core-types/prompts.md @@ -3,371 +3,164 @@ Creating, evaluating, and refining prompts is a core activity for AI engineers. Small changes to a prompt can have big impacts on your application's behavior. Weave lets you create prompts, save and retrieve them, and evolve them over time. -Some of the benefits of Weave's prompt management system are: -- Unopinionated core, with a batteries-included option for rapid development -- Versioning that shows you how a prompt has evolved over time -- The ability to update a prompt in production without redeploying your application -- The ability to evaluate a prompt against many inputs to evaluate performance +Weave is unopinionated about how a Prompt is constructed. If your needs are simple you can use our built-in `weave.StringPrompt` or `weave.MessagesPrompt` classes. If your needs are more complex you can subclass those or our base class `weave.Prompt` and override the +`format` method. -## Getting started +When you publish one of these objects with `weave.publish`, it will appear in your Weave project on the "Prompts" page. -If you want complete control over how a Prompt is constructed, you can subclass the base class, `weave.Prompt`, `weave.StringPrompt`, or `weave.MessagesPrompt` and implement the corresponding `format` method. When you publish one of these objects with `weave.publish`, it will appear in your Weave project on the "Prompts" page. - -``` -class Prompt(Object): - def format(self, **kwargs: Any) -> Any: - ... - -class StringPrompt(Prompt): - def format(self, **kwargs: Any) -> str: - ... - -class MessagesPrompt(Prompt): - def format(self, **kwargs: Any) -> list: - ... -``` - -Weave also includes a "batteries-included" class called `EasyPrompt` that can be simpler to start with, especially if you are working with APIs that are similar to OpenAI. This document highlights the features you get with EasyPrompt. - -## Constructing prompts - -You can think of the EasyPrompt object as a list of messages with associated roles, optional -placeholder variables, and an optional model configuration. -But constructing a prompt can be as simple as providing a single string: - -```python -import weave - -prompt = weave.EasyPrompt("What's 23 * 42?") -assert prompt[0] == {"role": "user", "content": "What's 23 * 42?"} -``` - -For terseness, the weave library aliases the `EasyPrompt` class to `P`. - -```python -from weave import P -p = P("What's 23 * 42?") -``` - -It is common for a prompt to consist of multiple messages. Each message has an associated `role`. -If the role is omitted, it defaults to `"user"`. - -**Some common roles** - -| Role | Description | -| --------- | -------------------------------------------------------------------------------------------------------------------- | -| system | System prompts provide high level instructions and can be used to set the behavior, knowledge, or persona of the AI. | -| user | Represents input from a human user. (This is the default role.) | -| assistant | Represents the AI's generated replies. Can be used for historical completions or to show examples. | - -For convenience, you can prefix a message string with one of these known roles: +## StringPrompt ```python import weave +weave.init('intro-example') -prompt = weave.EasyPrompt("system: Talk like a pirate") -assert prompt[0] == {"role": "system", "content": "Talk like a pirate"} - -# An explicit role parameter takes precedence -prompt = weave.EasyPrompt("system: Talk like a pirate", role="user") -assert prompt[0] == {"role": "user", "content": "system: Talk like a pirate"} - -``` - -Messages can be appended to a prompt one-by-one: +# highlight-next-line +system_prompt = weave.StringPrompt("You are a pirate") +# highlight-next-line +weave.publish(system_prompt, name="pirate_prompt") -```python -import weave +from openai import OpenAI +client = OpenAI() -prompt = weave.EasyPrompt() -prompt.append("You are an expert travel consultant.", role="system") -prompt.append("Give me five ideas for top kid-friendly attractions in New Zealand.") +response = client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "system", + # highlight-next-line + "content": system_prompt.format() + }, + { + "role": "user", + "content": "Explain general relativity in one paragraph." + } + ], +) ``` -Or you can append multiple messages at once, either with the `append` method or with the `Prompt` -constructor, which is convenient for constructing a prompt from existing messages. +Perhaps this prompt does not yield the desired effect, so we modify the prompt to be more +clearly instructive. ```python import weave +weave.init('intro-example') -prompt = weave.EasyPrompt() -prompt.append([ - {"role": "system", "content": "You are an expert travel consultant."}, - "Give me five ideas for top kid-friendly attractions in New Zealand." -]) - -# Same -prompt = weave.EasyPrompt([ - {"role": "system", "content": "You are an expert travel consultant."}, - "Give me five ideas for top kid-friendly attractions in New Zealand." -]) -``` - -The Prompt class is designed to be easily inserted into existing code. -For example, you can quickly wrap it around all of the arguments to the -OpenAI chat completion `create` call including its messages and model -configuration. If you don't wrap the inputs, Weave's integration would still -track all of the call's inputs, but it would not extract them as a separate -versioned object. Having a separate Prompt object allows you to version -the prompt, easily filter calls by that version, etc. +# highlight-next-line +system_prompt = weave.StringPrompt("Talk like a pirate. I need to know I'm listening to a pirate.") +weave.publish(system_prompt, name="pirate_prompt") -```python -from weave import init, P from openai import OpenAI client = OpenAI() -# Must specify a target project, otherwise the Weave code is a no-op -# highlight-next-line -init("intro-example") - -# highlight-next-line -response = client.chat.completions.create(P( - model="gpt-4o-mini", +response = client.chat.completions.create( + model="gpt-4o", messages=[ - {"role": "user", "content": "What's 23 * 42?"} + { + "role": "system", + # highlight-next-line + "content": system_prompt.format() + }, + { + "role": "user", + "content": "Explain general relativity in one paragraph." + } ], - temperature=0.7, - max_tokens=64, - top_p=1 -# highlight-next-line -)) -``` - -:::note -Why this works: Weave's OpenAI integration wraps the OpenAI `create` method to make it a Weave Op. -When the Op is executed, the Prompt object in the input will get saved and associated with the Call. -However, it will be replaced with the structure the `create` method expects for the execution of the -underlying function. -::: - -## Parameterizing prompts - -When specifying a prompt, you can include placeholders for values you want to fill in later. These placeholders are called "Parameters". -Parameters are indicated with curly braces. Here's a simple example: - -```python -import weave - -prompt = weave.EasyPrompt("What's {A} + {B}?") +) ``` -You will specify values for all of the parameters or "bind" them, when you [use the prompt](#using-prompts). - -The `require` method of Prompt allows you to associate parameters with restrictions that will be checked at bind time to detect programming errors. - -```python -import weave +When viewing this prompt object, I can see that it has two versions. -prompt = weave.EasyPrompt("What's {A} + 42?") -prompt.require("A", type="int", min=0, max=100) - -prompt = weave.EasyPrompt("system: You are a {profession}") -prompt.require("profession", oneof=('pirate', 'cartoon mouse', 'hungry dragon'), default='pirate') -``` +![Screenshot of viewing a prompt object](imgs/prompt-object.png) -## Using prompts +I can also select them for comparison to see exactly what changed. -You use a Prompt by converting it into a list of messages where all template placeholders have been filled in. You can bind a prompt to parameter values with the `bind` method or by simply calling it as a function. Here's an example where the prompt has zero parameters. +![Screenshot of prompt comparison](imgs/prompt-comparison.png) -```python -import weave -prompt = weave.EasyPrompt("What's 23 * 42?") -assert prompt() == prompt.bind() == [ - {"role": "user", "content": "What's 23 * 42?"} -] -``` +## MessagesPrompt -If a prompt has parameters, you would specify values for them when you use the prompt. -Parameter values can be passed in as a dictionary or as keyword arguments. +The `MessagesPrompt` can be used to replace an array of Message objects. ```python import weave -prompt = weave.EasyPrompt("What's {A} + {B}?") -assert prompt(A=5, B="10") == prompt({"A": 5, "B": "10"}) -``` +weave.init('intro-example') -If any parameters are missing, they will be left unsubstituted in the output. - -Here's a complete example of using a prompt with OpenAI. This example also uses [Weave's OpenAI integration](../integrations/openai.md) to automatically log the prompt and response. +# highlight-next-line +prompt = weave.MessagesPrompt([ + { + "role": "system", + "content": "You are a stegosaurus, but don't be too obvious about it." + }, + { + "role": "user", + "content": "What's good to eat around here?" + } +]) +weave.publish(prompt, name="dino_prompt") -```python -import weave from openai import OpenAI client = OpenAI() -weave.init("intro-example") -prompt = weave.EasyPrompt() -prompt.append("You will be provided with a tweet, and your task is to classify its sentiment as positive, neutral, or negative.", role="system") -prompt.append("I love {this_thing}!") - response = client.chat.completions.create( - model="gpt-4o-mini", - messages=prompt(this_thing="Weave"), - temperature=0.7, - max_tokens=64, - top_p=1 + model="gpt-4o", + # highlight-next-line + messages=prompt.format(), ) ``` -## Publishing to server +## Parameterizing prompts -Prompt are a type of [Weave object](../tracking/objects.md), and use the same methods for publishing to the Weave server. -You must specify a destination project name with `weave.init` before you can publish a prompt. +As the `format` method's name suggests, you can pass arguments to +fill in template placeholders in the content string. ```python import weave +weave.init('intro-example') -prompt = weave.EasyPrompt() -prompt.append("What's 23 * 42?") - -weave.init("intro-example") # Use entity/project format if not targeting your default entity -weave.publish(prompt, name="calculation-prompt") -``` - -Weave will automatically determine if the object has changed and only publish a new version if it has. -You can also specify a name or description for the Prompt as part of its constructor. +# highlight-next-line +prompt = weave.StringPrompt("Solve the equation {equation}") +weave.publish(prompt, name="calculator_prompt") -```python -import weave +from openai import OpenAI +client = OpenAI() -prompt = weave.EasyPrompt( - "What's 23 * 42?", - name="calculation-prompt", - description="A prompt for calculating the product of two numbers.", +response = client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "user", + # highlight-next-line + "content": prompt.format(equation="1 + 1 = ?") + } + ], ) - -weave.init("intro-example") -weave.publish(prompt) -``` - -## Retrieving from server - -Prompt are a type of [Weave object](../tracking/objects.md), and use the same methods for retrieval from the Weave server. -You must specify a source project name with `weave.init` before you can retrieve a prompt. - -```python -import weave - -weave.init("intro-example") -prompt = weave.ref("calculation-prompt").get() -``` - -By default, the latest version of the prompt is returned. You can make this explicit or select a specific version by providing its version id. - -```python -import weave - -weave.init("intro-example") -prompt = weave.ref("calculation-prompt:latest").get() -# ":", for example: -prompt = weave.ref("calculation-prompt:QSLzr96CTzFwLWgFFi3EuawCI4oODz4Uax98SxIY79E").get() -``` - -It is also possible to retrieve a Prompt without calling `init` if you pass a fully qualified URI to `weave.ref`. - -## Loading and saving from files - -Prompts can be saved to files and loaded from files. This can be convenient if you want your Prompt to be versioned through -a mechanism other than Weave such as git, or as a fallback if Weave is not available. - -To save a prompt to a file, you can use the `dump_file` method. - -```python -import weave - -prompt = weave.EasyPrompt("What's 23 * 42?") -prompt.dump_file("~/prompt.json") ``` -and load it again later with `Prompt.load_file`. - -```python -import weave - -prompt = weave.EasyPrompt.load_file("~/prompt.json") -``` - -You can also use the lower level `dump` and `Prompt.load` methods for custom (de)serialization. - -## Evaluating prompts - -The [Parameter feature of prompts](#parameterizing-prompts) can be used to execute or evaluate variations of a prompt. - -You can bind each row of a [Dataset](./datasets.md) to generate N variations of a prompt. +This also works with `MessagesPrompt`. ```python import weave +weave.init('intro-example') -# Create a dataset -dataset = weave.Dataset(name='countries', rows=[ - {'id': '0', 'country': "Argentina"}, - {'id': '1', 'country': "Belize"}, - {'id': '2', 'country': "Canada"}, - {'id': '3', 'country': "New Zealand"}, +# highlight-next-line +prompt = weave.MessagesPrompt([ +{ + "role": "system", + "content": "You will be provided with a description of a scene and your task is to provide a single word that best describes an associated emotion." +}, +{ + "role": "user", + "content": "{scene}" +} ]) +weave.publish(prompt, name="emotion_prompt") -prompt = weave.EasyPrompt(name='travel_agent') -prompt.append("You are an expert travel consultant.", role="system") -prompt.append("Tell me the capital of {country} and about five kid-friendly attractions there.") - - -prompts = prompt.bind_rows(dataset) -assert prompts[2][1]["content"] == "Tell me the capital of Canada and about five kid-friendly attractions there." -``` - -You can extend this into an [Evaluation](./evaluations.md): - -```python -import asyncio - -import openai -import weave - -weave.init("intro-example") - -# Create a dataset -dataset = weave.Dataset(name='countries', rows=[ - {'id': '0', 'country': "Argentina", 'capital': "Buenos Aires"}, - {'id': '1', 'country': "Belize", 'capital': "Belmopan"}, - {'id': '2', 'country': "Canada", 'capital': "Ottawa"}, - {'id': '3', 'country': "New Zealand", 'capital': "Wellington"}, -]) +from openai import OpenAI +client = OpenAI() -# Create a prompt -prompt = weave.EasyPrompt(name='travel_agent') -prompt.append("You are an expert travel consultant.", role="system") -prompt.append("Tell me the capital of {country} and about five kid-friendly attractions there.") - -# Create a model, combining a prompt with model configuration -class TravelAgentModel(weave.Model): - - model_name: str - prompt: weave.EasyPrompt - - @weave.op - async def predict(self, country: str) -> dict: - client = openai.AsyncClient() - - response = await client.chat.completions.create( - model=self.model_name, - messages=self.prompt(country=country), - ) - result = response.choices[0].message.content - if result is None: - raise ValueError("No response from model") - return result - -# Define and run the evaluation -@weave.op -def mentions_capital_scorer(capital: str, model_output: str) -> dict: - return {'correct': capital in model_output} - -model = TravelAgentModel(model_name="gpt-4o-mini", prompt=prompt) -evaluation = weave.Evaluation( - dataset=dataset, - scorers=[mentions_capital_scorer], +response = client.chat.completions.create( + model="gpt-4o", + # highlight-next-line + messages=prompt.format(scene="A dog is lying on a dock next to a fisherman."), ) -asyncio.run(evaluation.evaluate(model)) - ``` diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 29e0a09ccd9d..9848c6b8398d 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -64,7 +64,7 @@ const sidebars: SidebarsConfig = { "guides/evaluation/scorers", ], }, - // "guides/core-types/prompts", + "guides/core-types/prompts", "guides/core-types/models", "guides/core-types/datasets", "guides/tracking/feedback", diff --git a/tests/trace/test_prompt.py b/tests/trace/test_prompt.py index 98bb731d076e..d32a5236a946 100644 --- a/tests/trace/test_prompt.py +++ b/tests/trace/test_prompt.py @@ -2,22 +2,21 @@ def test_stringprompt_format(): - class MyPrompt(StringPrompt): - def format(self, **kwargs) -> str: - return "Imagine a lot of complicated logic build this string." - - prompt = MyPrompt() - assert prompt.format() == "Imagine a lot of complicated logic build this string." + prompt = StringPrompt("You are a pirate. Tell us your thoughts on {topic}.") + assert ( + prompt.format(topic="airplanes") + == "You are a pirate. Tell us your thoughts on airplanes." + ) def test_messagesprompt_format(): - class MyPrompt(MessagesPrompt): - def format(self, **kwargs) -> list: - return [ - {"role": "user", "content": "What's 23 * 42"}, - ] - - prompt = MyPrompt() - assert prompt.format() == [ - {"role": "user", "content": "What's 23 * 42"}, + prompt = MessagesPrompt( + [ + {"role": "system", "content": "You are a pirate."}, + {"role": "user", "content": "Tell us your thoughts on {topic}."}, + ] + ) + assert prompt.format(topic="airplanes") == [ + {"role": "system", "content": "You are a pirate."}, + {"role": "user", "content": "Tell us your thoughts on airplanes."}, ] diff --git a/weave-js/src/components/FancyPage/useProjectSidebar.ts b/weave-js/src/components/FancyPage/useProjectSidebar.ts index 92d8d7a7a505..ddc17c100bc1 100644 --- a/weave-js/src/components/FancyPage/useProjectSidebar.ts +++ b/weave-js/src/components/FancyPage/useProjectSidebar.ts @@ -170,13 +170,13 @@ export const useProjectSidebar = ( key: 'dividerWithinWeave-2', isShown: isWeaveOnly, }, - // { - // type: 'button' as const, - // name: 'Prompts', - // slug: 'weave/prompts', - // isShown: showWeaveSidebarItems || isShowAll, - // iconName: IconNames.ForumChatBubble, - // }, + { + type: 'button' as const, + name: 'Prompts', + slug: 'weave/prompts', + isShown: showWeaveSidebarItems || isShowAll, + iconName: IconNames.ForumChatBubble, + }, { type: 'button' as const, name: 'Models', diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/TabUsePrompt.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/TabUsePrompt.tsx index 6d00af48bc61..1d3c54dba46c 100644 --- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/TabUsePrompt.tsx +++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/TabUsePrompt.tsx @@ -30,37 +30,6 @@ export const TabUsePrompt = ({ const isParentObject = !ref.artifactRefExtra; const label = isParentObject ? 'prompt version' : 'prompt'; - // TODO: Simplify if no params. - const longExample = `import weave -from openai import OpenAI - -weave.init("${projectName}") - -${pythonName} = weave.ref("${uri}").get() - -class MyModel(weave.Model): - model_name: str - prompt: weave.Prompt - - @weave.op - def predict(self, params: dict) -> dict: - client = OpenAI() - response = client.chat.completions.create( - model=self.model_name, - messages=self.prompt.bind(params), - ) - result = response.choices[0].message.content - if result is None: - raise ValueError("No response from model") - return result - -mymodel = MyModel(model_name="gpt-3.5-turbo", prompt=${pythonName}) - -# Replace with desired parameter values -params = ${JSON.stringify({}, null, 2)} -print(mymodel.predict(params)) -`; - return ( @@ -85,15 +54,6 @@ print(mymodel.predict(params)) tooltipText="Click to copy unabridged string" /> - - A more complete example: - - - ); }; diff --git a/weave/flow/prompt/prompt.py b/weave/flow/prompt/prompt.py index fbf96ac12cac..86e971e00c5d 100644 --- a/weave/flow/prompt/prompt.py +++ b/weave/flow/prompt/prompt.py @@ -73,17 +73,52 @@ def color_content(content: str, values: dict) -> str: class Prompt(Object): def format(self, **kwargs: Any) -> Any: - raise NotImplemented + raise NotImplementedError("Subclasses must implement format()") -class MessagesPrompt(Prompt): - def format(self, **kwargs: Any) -> list: - raise NotImplemented +class StringPrompt(Prompt): + content: str = "" + def __init__(self, content: str): + super().__init__() + self.content = content -class StringPrompt(Prompt): def format(self, **kwargs: Any) -> str: - raise NotImplemented + return self.content.format(**kwargs) + + @staticmethod + def from_obj(obj: Any) -> "StringPrompt": + prompt = StringPrompt(content=obj.content) + prompt.name = obj.name + prompt.description = obj.description + return prompt + + +class MessagesPrompt(Prompt): + messages: list[dict] = Field(default_factory=list) + + def __init__(self, messages: list[dict]): + super().__init__() + self.messages = messages + + def format_message(self, message: dict, **kwargs: Any) -> dict: + m = {} + for k, v in message.items(): + if isinstance(v, str): + m[k] = v.format(**kwargs) + else: + m[k] = v + return m + + def format(self, **kwargs: Any) -> list: + return [self.format_message(m, **kwargs) for m in self.messages] + + @staticmethod + def from_obj(obj: Any) -> "MessagesPrompt": + prompt = MessagesPrompt(messages=obj.messages) + prompt.name = obj.name + prompt.description = obj.description + return prompt class EasyPrompt(UserList, Prompt): diff --git a/weave/trace/refs.py b/weave/trace/refs.py index 7557a404f9de..9190368b5926 100644 --- a/weave/trace/refs.py +++ b/weave/trace/refs.py @@ -173,6 +173,18 @@ def objectify(self, obj: Any) -> Any: # version number or latest alias resolved to a specific digest. prompt.__dict__["ref"] = obj.ref return prompt + if "StringPrompt" == class_name: + from weave.flow.prompt.prompt import StringPrompt + + prompt = StringPrompt.from_obj(obj) + prompt.__dict__["ref"] = obj.ref + return prompt + if "MessagesPrompt" == class_name: + from weave.flow.prompt.prompt import MessagesPrompt + + prompt = MessagesPrompt.from_obj(obj) + prompt.__dict__["ref"] = obj.ref + return prompt return obj def get(self) -> Any: