From 2d101a8216543b2cc9956ce44a3de6e894ba9b7d Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 14:45:57 -0700
Subject: [PATCH 01/15] Adds GH action to trigger generate_llmstxt.py, testing
action
---
docs/scripts/generate_llmstxt.py | 24 +-
docs/static/llms.md | 4256 ------------------------------
docs/static/llms.txt | 111 -
3 files changed, 20 insertions(+), 4371 deletions(-)
delete mode 100644 docs/static/llms.md
delete mode 100644 docs/static/llms.txt
diff --git a/docs/scripts/generate_llmstxt.py b/docs/scripts/generate_llmstxt.py
index 24ccb39c697..422b1f6c3c9 100644
--- a/docs/scripts/generate_llmstxt.py
+++ b/docs/scripts/generate_llmstxt.py
@@ -1,6 +1,7 @@
from pathlib import Path
import yaml
import re
+import os
def is_optional(file_path: Path) -> bool:
"""Check if file should be in Optional section."""
@@ -67,8 +68,23 @@ def generate_llms_txt(docs_dir: Path, output_file: Path):
output_file.write_text('\n'.join(content))
+def main():
+ # Get the root directory of the project (adjust as needed)
+ script_dir = Path(__file__).resolve().parent
+ root_dir = script_dir.parent # Assuming the script is in "docs/scripts"
+
+ docs_dir = root_dir / "docs"
+ output_file = root_dir / "static" / "llms.txt"
+
+ # Debugging output
+ print(f"Docs directory: {docs_dir}")
+ print(f"Output file: {output_file}")
+
+ # Ensure the static directory exists
+ output_file.parent.mkdir(parents=True, exist_ok=True)
+
+ # Call the function
+ generate_llms_txt(docs_dir=docs_dir, output_file=output_file)
+
if __name__ == "__main__":
- generate_llms_txt(
- docs_dir=Path("./docs"),
- output_file=Path("./static/llms.txt")
- )
\ No newline at end of file
+ main()
\ No newline at end of file
diff --git a/docs/static/llms.md b/docs/static/llms.md
deleted file mode 100644
index db7d123a009..00000000000
--- a/docs/static/llms.md
+++ /dev/null
@@ -1,4256 +0,0 @@
-This document contains links to all the documentation for Weave.---
-slug: /
----
-
-# Introduction
-
-**Weave** is a lightweight toolkit for tracking and evaluating LLM applications, built by Weights & Biases.
-
-Our goal is to bring rigor, best-practices, and composability to the inherently experimental process of developing AI applications, without introducing cognitive overhead.
-
-**[Get started](/quickstart)** by decorating Python functions with `@weave.op()`.
-
-
-
-Seriously, try the 🍪 **[quickstart](/quickstart)** 🍪 or {window.analytics?.track("Weave Docs: Quickstart colab clicked")}}>
-
-You can use Weave to:
-
-- Log and debug language model inputs, outputs, and traces
-- Build rigorous, apples-to-apples evaluations for language model use cases
-- Organize all the information generated across the LLM workflow, from experimentation to evaluations to production
-
-## What's next?
-
-Try the [Quickstart](/quickstart) to see Weave in action.import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Track LLM inputs & outputs
-
-
-Follow these steps to track your first call or
-
-## 1. Install Weave and create an API Key
-
-**Install weave**
-
-First install the weave library:
-
-
-
- ```bash
- pip install weave
- ```
-
-
- ```bash
- pnpm install weave
- ```
-
-
-
-**Get your API key**
-
-Then, create a Weights & Biases (W&B) account at https://wandb.ai and copy your API key from https://wandb.ai/authorize
-
-## 2. Log a trace to a new project
-
-To get started with tracking your first project with Weave:
-
-- Import the `weave` library
-- Call `weave.init('project-name')` to start tracking
- - You will be prompted to log in with your API key if you are not yet logged in on your machine.
- - To log to a specific W&B Team name, replace `project-name` with `team-name/project-name`
- - **NOTE:** In automated environments, you can define the environment variable `WANDB_API_KEY` with your API key to login without prompting.
-- Add the `@weave.op()` decorator to the python functions you want to track
-
-_In this example, we're using openai so you will need to add an OpenAI [API key](https://platform.openai.com/docs/quickstart/step-2-setup-your-api-key)._
-
-
-
- ```python
- # highlight-next-line
- import weave
- from openai import OpenAI
-
- client = OpenAI(api_key="...")
-
- # Weave will track the inputs, outputs and code of this function
- # highlight-next-line
- @weave.op()
- def extract_dinos(sentence: str) -> dict:
- response = client.chat.completions.create(
- model="gpt-4o",
- messages=[
- {
- "role": "system",
- "content": """In JSON format extract a list of `dinosaurs`, with their `name`,
- their `common_name`, and whether its `diet` is a herbivore or carnivore"""
- },
- {
- "role": "user",
- "content": sentence
- }
- ],
- response_format={ "type": "json_object" }
- )
- return response.choices[0].message.content
-
-
- # Initialise the weave project
- # highlight-next-line
- weave.init('jurassic-park')
-
- sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \
- both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \
- Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below."""
-
- result = extract_dinos(sentence)
- print(result)
- ```
- When you call the `extract_dinos` function Weave will output a link to view your trace.
-
-
-
- ```typescript
- import OpenAI from 'openai';
- // highlight-next-line
- import * as weave from 'weave';
-
- // highlight-next-line
- const openai = weave.wrapOpenAI(new OpenAI());
-
- async function extractDinos(input: string) {
- const response = await openai.chat.completions.create({
- model: 'gpt-4o',
- messages: [
- {
- role: 'user',
- content: `In JSON format extract a list of 'dinosaurs', with their 'name', their 'common_name', and whether its 'diet' is a herbivore or carnivore: ${input}`,
- },
- ],
- });
- return response.choices[0].message.content;
- }
- // highlight-next-line
- const extractDinosOp = weave.op(extractDinos);
-
- async function main() {
- // highlight-next-line
- await weave.init('examples');
- const result = await extractDinosOp(
- 'I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.'
- );
- console.log(result);
- }
-
- main();
-
- ```
- When you call the `extractDinos` function Weave will output a link to view your trace.
-
-
-
-
-## 3. Automated LLM library logging
-
-Calls made to OpenAI, Anthropic and [many more LLM libraries](guides/integrations/) are automatically tracked with Weave, with **LLM metadata**, **token usage** and **cost** being logged automatically. If your LLM library isn't currently one of our integrations you can track calls to other LLMs libraries or frameworks easily by wrapping them with `@weave.op()`.
-
-## 4. See traces of your application in your project
-
-🎉 Congrats! Now, every time you call this function, weave will automatically capture the input & output data and log any changes made to the code.
-
-
-
-## What's next?
-
-- Follow the [Tracking flows and app metadata](/tutorial-tracing_2) to start tracking and the data flowing through your app.import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Tutorial: Build an Evaluation pipeline
-
-To iterate on an application, we need a way to evaluate if it's improving. To do so, a common practice is to test it against the same set of examples when there is a change. Weave has a first-class way to track evaluations with `Model` & `Evaluation` classes. We have built the APIs to make minimal assumptions to allow for the flexibility to support a wide array of use-cases.
-
-
-
-## 1. Build a `Model`
-
-
-
-
-`Model`s store and version information about your system, such as prompts, temperatures, and more.
-Weave automatically captures when they are used and updates the version when there are changes.
-
-`Model`s are declared by subclassing `Model` and implementing a `predict` function definition, which takes one example and returns the response.
-
-:::warning
-
-**Known Issue**: If you are using Google Colab, remove `async` from the following examples.
-
-:::
-
- ```python
- import json
- import openai
- import weave
-
- # highlight-next-line
- class ExtractFruitsModel(weave.Model):
- model_name: str
- prompt_template: str
-
- # highlight-next-line
- @weave.op()
- # highlight-next-line
- async def predict(self, sentence: str) -> dict:
- client = openai.AsyncClient()
-
- response = await client.chat.completions.create(
- model=self.model_name,
- messages=[
- {"role": "user", "content": self.prompt_template.format(sentence=sentence)}
- ],
- )
- result = response.choices[0].message.content
- if result is None:
- raise ValueError("No response from model")
- parsed = json.loads(result)
- return parsed
- ```
-
- You can instantiate `Model` objects as normal like this:
-
- ```python
- import asyncio
- import weave
-
- weave.init('intro-example')
-
- model = ExtractFruitsModel(model_name='gpt-3.5-turbo-1106',
- prompt_template='Extract fields ("fruit": , "color": , "flavor": ) from the following text, as json: {sentence}')
- sentence = "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy."
- print(asyncio.run(model.predict(sentence)))
- # if you're in a Jupyter Notebook, run:
- # await model.predict(sentence)
- ```
-
-:::note
-Checkout the [Models](/guides/core-types/models) guide to learn more.
-:::
-
-
-
-
- `weave.Model` is not supported in TypeScript yet. Instead, you can just wrap your model-like function with `weave.op`
-
- ```typescript
- // highlight-next-line
- const model = weave.op(async function myModel({datasetRow}) {
- const prompt = `Extract fields ("fruit": , "color": , "flavor") from the following text, as json: ${datasetRow.sentence}`;
- const response = await openaiClient.chat.completions.create({
- model: 'gpt-3.5-turbo',
- messages: [{role: 'user', content: prompt}],
- response_format: {type: 'json_object'},
- });
- const result = response?.choices?.[0]?.message?.content;
- if (result == null) {
- throw new Error('No response from model');
- }
- return JSON.parse(result);
- });
- ```
-
-
-
-
-## 2. Collect some examples
-
-
-
-
- ```python
- sentences = [
- "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.",
- "Pounits are a bright green color and are more savory than sweet.",
- "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."
- ]
- labels = [
- {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'},
- {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'},
- {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'}
- ]
- examples = [
- {'id': '0', 'sentence': sentences[0], 'target': labels[0]},
- {'id': '1', 'sentence': sentences[1], 'target': labels[1]},
- {'id': '2', 'sentence': sentences[2], 'target': labels[2]}
- ]
- ```
-
-
-
-
- ```typescript
- const sentences = [
- 'There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.',
- 'Pounits are a bright green color and are more savory than sweet.',
- 'Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them.',
- ];
- const labels = [
- {fruit: 'neoskizzles', color: 'purple', flavor: 'candy'},
- {fruit: 'pounits', color: 'bright green', flavor: 'savory'},
- {fruit: 'glowls', color: 'pale orange', flavor: 'sour and bitter'},
- ];
- const examples = [
- {id: '0', sentence: sentences[0], target: labels[0]},
- {id: '1', sentence: sentences[1], target: labels[1]},
- {id: '2', sentence: sentences[2], target: labels[2]},
- ];
- const dataset = new weave.Dataset({
- id: 'Fruit Dataset',
- rows: examples,
- });
- ```
-
-
-
-## 3. Evaluate a `Model`
-
-
-
-
-`Evaluation`s assess a `Model`s performance on a set of examples using a list of specified scoring functions or `weave.scorer.Scorer` classes.
-
-Here, we'll use a default scoring class `MultiTaskBinaryClassificationF1` and we'll also define our own `fruit_name_score` scoring function.
-
-Here `sentence` is passed to the model's predict function, and `target` is used in the scoring function, these are inferred based on the argument names of the `predict` and scoring functions. The `fruit` key needs to be outputted by the model's predict function and must also be existing as a column in the dataset (or outputted by the `preprocess_model_input` function if defined).
-
- ```python
- import weave
- from weave.scorers import MultiTaskBinaryClassificationF1
-
- weave.init('intro-example')
-
- @weave.op()
- def fruit_name_score(target: dict, output: dict) -> dict:
- return {'correct': target['fruit'] == output['fruit']}
-
- # highlight-next-line
- evaluation = weave.Evaluation(
- # highlight-next-line
- dataset=examples,
- # highlight-next-line
- scorers=[
- # highlight-next-line
- MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]),
- # highlight-next-line
- fruit_name_score
- # highlight-next-line
- ],
- # highlight-next-line
- )
- # highlight-next-line
- print(asyncio.run(evaluation.evaluate(model)))
- # if you're in a Jupyter Notebook, run:
- # await evaluation.evaluate(model)
- ```
-
-
-
-`Evaluation`s assess a model's performance on a set of examples using a list of specified scoring functions.
-
-For this example, we'll define a few simple scoring functions.
-
-Here, `sentence` is passed to the model and `...` is used in the scoring function. These are defined...
-
- ```typescript
- import * as weave from 'weave';
- import {OpenAI} from 'openai';
-
- const client = await weave.init('intro-example');
- const openaiClient = weave.wrapOpenAI(new OpenAI());
-
- const fruitNameScorer = weave.op(
- ({modelOutput, datasetRow}) => datasetRow.target.fruit == modelOutput.fruit,
- {name: 'fruitNameScore'}
- );
-
- const evaluation = new weave.Evaluation({
- dataset: ds,
- scorers: [fruitNameScorer],
- });
-
- const results = await evaluation.evaluate(model);
- console.log(JSON.stringify(results, null, 2));
- ```
-
-
-
-
-In some applications we want to create custom `Scorer` classes - where for example a standardized `LLMJudge` class should be created with specific parameters (e.g. chat model, prompt), specific scoring of each row, and specific calculation of an aggregate score. See the tutorial on defining a `Scorer` class in the next chapter on [Model-Based Evaluation of RAG applications](/tutorial-rag#optional-defining-a-scorer-class) for more information.
-
-## 4. Pulling it all together
-
-
-
-
- ```python
- import json
- import asyncio
- # highlight-next-line
- import weave
- # highlight-next-line
- from weave.scorers import MultiTaskBinaryClassificationF1
- import openai
-
- # We create a model class with one predict function.
- # All inputs, predictions and parameters are automatically captured for easy inspection.
-
- # highlight-next-line
- class ExtractFruitsModel(weave.Model):
- model_name: str
- prompt_template: str
-
- # highlight-next-line
- @weave.op()
- # highlight-next-line
- async def predict(self, sentence: str) -> dict:
- client = openai.AsyncClient()
-
- response = await client.chat.completions.create(
- model=self.model_name,
- messages=[
- {"role": "user", "content": self.prompt_template.format(sentence=sentence)}
- ],
- response_format={ "type": "json_object" }
- )
- result = response.choices[0].message.content
- if result is None:
- raise ValueError("No response from model")
- parsed = json.loads(result)
- return parsed
-
- # We call init to begin capturing data in the project, intro-example.
- weave.init('intro-example')
-
- # We create our model with our system prompt.
- model = ExtractFruitsModel(name='gpt4',
- model_name='gpt-4-0125-preview',
- prompt_template='Extract fields ("fruit": , "color": , "flavor") from the following text, as json: {sentence}')
- sentences = ["There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.",
- "Pounits are a bright green color and are more savory than sweet.",
- "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."]
- labels = [
- {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'},
- {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'},
- {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'}
- ]
- examples = [
- {'id': '0', 'sentence': sentences[0], 'target': labels[0]},
- {'id': '1', 'sentence': sentences[1], 'target': labels[1]},
- {'id': '2', 'sentence': sentences[2], 'target': labels[2]}
- ]
- # If you have already published the Dataset, you can run:
- # dataset = weave.ref('example_labels').get()
-
- # We define a scoring function to compare our model predictions with a ground truth label.
- @weave.op()
- def fruit_name_score(target: dict, output: dict) -> dict:
- return {'correct': target['fruit'] == output['fruit']}
-
- # Finally, we run an evaluation of this model.
- # This will generate a prediction for each input example, and then score it with each scoring function.
- # highlight-next-line
- evaluation = weave.Evaluation(
- # highlight-next-line
- name='fruit_eval',
- # highlight-next-line
- dataset=examples, scorers=[MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]), fruit_name_score],
- # highlight-next-line
- )
- print(asyncio.run(evaluation.evaluate(model)))
- # if you're in a Jupyter Notebook, run:
- # await evaluation.evaluate(model)
- ```
-
-
-
-
- ```typescript
- import {OpenAI} from 'openai';
- import 'source-map-support/register';
- import * as weave from 'weave';
-
- const sentences = [
- 'There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.',
- 'Pounits are a bright green color and are more savory than sweet.',
- 'Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them.',
- 'There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.',
- ];
- const labels = [
- {fruit: 'neoskizzles', color: 'purple', flavor: 'candy'},
- {fruit: 'pounits', color: 'bright green', flavor: 'savory'},
- {fruit: 'glowls', color: 'pale orange', flavor: 'sour and bitter'},
- ];
- const examples = [
- {id: '0', sentence: sentences[0], target: labels[0]},
- {id: '1', sentence: sentences[1], target: labels[1]},
- {id: '2', sentence: sentences[2], target: labels[2]},
- ];
- const dataset = new weave.Dataset({
- id: 'Fruit Dataset',
- rows: examples,
- });
-
- const openaiClient = weave.wrapOpenAI(new OpenAI());
-
- const model = weave.op(async function myModel({datasetRow}) {
- const prompt = `Extract fields ("fruit": , "color": , "flavor") from the following text, as json: ${datasetRow.sentence}`;
- const response = await openaiClient.chat.completions.create({
- model: 'gpt-3.5-turbo',
- messages: [{role: 'user', content: prompt}],
- response_format: {type: 'json_object'},
- });
- const result = response?.choices?.[0]?.message?.content;
- if (result == null) {
- throw new Error('No response from model');
- }
- return JSON.parse(result);
- });
-
- const fruitNameScorer = weave.op(
- ({modelOutput, datasetRow}) => datasetRow.target.fruit == modelOutput.fruit,
- {name: 'fruitNameScore'}
- );
-
- async function main() {
- await weave.init('examples');
- const evaluation = new weave.Evaluation({
- dataset,
- scorers: [fruitNameScorer],
- });
-
- const results = await evaluation.evaluate({model});
- console.log(JSON.stringify(results, null, 2));
- }
-
- main();
-
- ```
-
-
-
-
-## What's next?
-
-- Follow the [Model-Based Evaluation of RAG applications](/tutorial-rag) to evaluate a RAG app using an LLM judge.import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Tutorial: Model-Based Evaluation of RAG applications
-
-:::warning
-
-This tutorial is currently only available for Python!
-
-:::
-
-Retrieval Augmented Generation (RAG) is a common way of building Generative AI applications that have access to custom knowledge bases.
-
-In this example, we'll show an example that has a retrieval step to get documents. By tracking this, you can debug your app and see what documents were pulled into the LLM context.
-We'll also show how to evaluate it using an LLM judge.
-
-
-
-Check out the [RAG++ course](https://www.wandb.courses/courses/rag-in-production?utm_source=wandb_docs&utm_medium=code&utm_campaign=weave_docs) for a more advanced dive into practical RAG techniques for engineers, where you'll learn production-ready solutions from Weights & Biases, Cohere and Weaviate to optimize performance, cut costs, and enhance the accuracy and relevance of your applications.
-
-## 1. Build a knowledge base
-
-First, we compute the embeddings for our articles. You would typically do this once with your articles and put the embeddings & metadata in a database, but here we're doing it every time we run our script for simplicity.
-
-
-
- ```python
- from openai import OpenAI
- import weave
- from weave import Model
- import numpy as np
- import json
- import asyncio
-
- articles = [
- "Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial “tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too,” one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.",
- "Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.",
- "Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if its stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities",
- "Rivian and Lucid shares plunge after weak EV earnings reports Shares of electric vehicle makers Rivian and Lucid fell Thursday after the companies reported stagnant production in their fourth-quarter earnings after the bell Wednesday. Rivian shares sank about 25 percent, and Lucids stock dropped around 17 percent. Rivian forecast it will make 57,000 vehicles in 2024, slightly less than the 57,232 vehicles it produced in 2023. Lucid said it expects to make 9,000 vehicles in 2024, more than the 8,428 vehicles it made in 2023.",
- "Mauritius blocks Norwegian cruise ship over fears of a potential cholera outbreak Local authorities on Sunday denied permission for the Norwegian Dawn ship, which has 2,184 passengers and 1,026 crew on board, to access the Mauritius capital of Port Louis, citing “potential health risks.” The Mauritius Ports Authority said Sunday that samples were taken from at least 15 passengers on board the cruise ship. A spokesperson for the U.S.-headquartered Norwegian Cruise Line Holdings said Sunday that 'a small number of guests experienced mild symptoms of a stomach-related illness' during Norwegian Dawns South Africa voyage.",
- "Intuitive Machines lands on the moon in historic first for a U.S. company Intuitive Machines Nova-C cargo lander, named Odysseus after the mythological Greek hero, is the first U.S. spacecraft to soft land on the lunar surface since 1972. Intuitive Machines is the first company to pull off a moon landing — government agencies have carried out all previously successful missions. The company's stock surged in extended trading Thursday, after falling 11 percent in regular trading.",
- "Lunar landing photos: Intuitive Machines Odysseus sends back first images from the moon Intuitive Machines cargo moon lander Odysseus returned its first images from the surface. Company executives believe the lander caught its landing gear sideways on the moon's surface while touching down and tipped over. Despite resting on its side, the company's historic IM-1 mission is still operating on the moon.",
- ]
-
- def docs_to_embeddings(docs: list) -> list:
- openai = OpenAI()
- document_embeddings = []
- for doc in docs:
- response = (
- openai.embeddings.create(input=doc, model="text-embedding-3-small")
- .data[0]
- .embedding
- )
- document_embeddings.append(response)
- return document_embeddings
-
- article_embeddings = docs_to_embeddings(articles) # Note: you would typically do this once with your articles and put the embeddings & metadata in a database
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-## 2. Create a RAG app
-
-Next, we wrap our retrieval function `get_most_relevant_document` with a `weave.op()` decorator and we create our `Model` class. We call `weave.init('rag-qa')` to begin tracking all the inputs and outputs of our functions for later inspection.
-
-
-
- ```python
- from openai import OpenAI
- import weave
- from weave import Model
- import numpy as np
- import asyncio
-
- # highlight-next-line
- @weave.op()
- def get_most_relevant_document(query):
- openai = OpenAI()
- query_embedding = (
- openai.embeddings.create(input=query, model="text-embedding-3-small")
- .data[0]
- .embedding
- )
- similarities = [
- np.dot(query_embedding, doc_emb)
- / (np.linalg.norm(query_embedding) * np.linalg.norm(doc_emb))
- for doc_emb in article_embeddings
- ]
- # Get the index of the most similar document
- most_relevant_doc_index = np.argmax(similarities)
- return articles[most_relevant_doc_index]
-
- # highlight-next-line
- class RAGModel(Model):
- system_message: str
- model_name: str = "gpt-3.5-turbo-1106"
-
- # highlight-next-line
- @weave.op()
- def predict(self, question: str) -> dict: # note: `question` will be used later to select data from our evaluation rows
- from openai import OpenAI
- context = get_most_relevant_document(question)
- client = OpenAI()
- query = f"""Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know."
- Context:
- \"\"\"
- {context}
- \"\"\"
- Question: {question}"""
- response = client.chat.completions.create(
- model=self.model_name,
- messages=[
- {"role": "system", "content": self.system_message},
- {"role": "user", "content": query},
- ],
- temperature=0.0,
- response_format={"type": "text"},
- )
- answer = response.choices[0].message.content
- return {'answer': answer, 'context': context}
-
- # highlight-next-line
- weave.init('rag-qa')
- model = RAGModel(
- system_message="You are an expert in finance and answer questions related to finance, financial services, and financial markets. When responding based on provided information, be sure to cite the source."
- )
- model.predict("What significant result was reported about Zealand Pharma's obesity trial?")
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-## 3. Evaluating with an LLM Judge
-
-When there aren't simple ways to evaluate your application, one approach is to use an LLM to evaluate aspects of it. Here is an example of using an LLM judge to try to measure the context precision by prompting it to verify if the context was useful in arriving at the given answer. This prompt was augmented from the popular [RAGAS framework](https://docs.ragas.io/).
-
-### Defining a scoring function
-
-As we did in the [Build an Evaluation pipeline tutorial](/tutorial-eval), we'll define a set of example rows to test our app against and a scoring function. Our scoring function will take one row and evaluate it. The input arguments should match with the corresponding keys in our row, so `question` here will be taken from the row dictionary. `output` is the output of the model. The input to the model will be taken from the example based on its input argument, so `question` here too. We're using `async` functions so they run fast in parallel. If you need a quick introduction to async, you can find one [here](https://docs.python.org/3/library/asyncio.html).
-
-
-
- ```python
- from openai import OpenAI
- import weave
- import asyncio
-
- # highlight-next-line
- @weave.op()
- async def context_precision_score(question, output):
- context_precision_prompt = """Given question, answer and context verify if the context was useful in arriving at the given answer. Give verdict as "1" if useful and "0" if not with json output.
- Output in only valid JSON format.
-
- question: {question}
- context: {context}
- answer: {answer}
- verdict: """
- client = OpenAI()
-
- prompt = context_precision_prompt.format(
- question=question,
- context=output['context'],
- answer=output['answer'],
- )
-
- response = client.chat.completions.create(
- model="gpt-4-turbo-preview",
- messages=[{"role": "user", "content": prompt}],
- response_format={ "type": "json_object" }
- )
- response_message = response.choices[0].message
- response = json.loads(response_message.content)
- return {
- "verdict": int(response["verdict"]) == 1,
- }
-
- questions = [
- {"question": "What significant result was reported about Zealand Pharma's obesity trial?"},
- {"question": "How much did Berkshire Hathaway's cash levels increase in the fourth quarter?"},
- {"question": "What is the goal of Highmark Health's integration of Google Cloud and Epic Systems technology?"},
- {"question": "What were Rivian and Lucid's vehicle production forecasts for 2024?"},
- {"question": "Why was the Norwegian Dawn cruise ship denied access to Mauritius?"},
- {"question": "Which company achieved the first U.S. moon landing since 1972?"},
- {"question": "What issue did Intuitive Machines' lunar lander encounter upon landing on the moon?"}
- ]
- # highlight-next-line
- evaluation = weave.Evaluation(dataset=questions, scorers=[context_precision_score])
- # highlight-next-line
- asyncio.run(evaluation.evaluate(model)) # note: you'll need to define a model to evaluate
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-### Optional: Defining a `Scorer` class
-
-In some applications we want to create custom evaluation classes - where for example a standardized `LLMJudge` class should be created with specific parameters (e.g. chat model, prompt), specific scoring of each row, and specific calculation of an aggregate score. In order to do that Weave defines a list of ready-to-use `Scorer` classes and also makes it easy to create a custom `Scorer` - in the following we'll see how to create a custom `class CorrectnessLLMJudge(Scorer)`.
-
-On a high-level the steps to create custom Scorer are quite simple:
-
-1. Define a custom class that inherits from `weave.flow.scorer.Scorer`
-2. Overwrite the `score` function and add a `@weave.op()` if you want to track each call of the function
- - this function has to define an `output` argument where the prediction of the model will be passed to. We define it as type `Optional[dict]` in case the mode might return "None".
- - the rest of the arguments can either be a general `Any` or `dict` or can select specific columns from the dataset that is used to evaluate the model using the `weave.Evaluate` class - they have to have the exact same names as the column names or keys of a single row after being passed to `preprocess_model_input` if that is used.
-3. _Optional:_ Overwrite the `summarize` function to customize the calculation of the aggregate score. By default Weave uses the `weave.flow.scorer.auto_summarize` function if you don't define a custom function.
- - this function has to have a `@weave.op()` decorator.
-
-
-
- ```python
- from weave.scorers import Scorer
- from weave import WeaveList
-
- class CorrectnessLLMJudge(Scorer):
- prompt: str
- model_name: str
- device: str
-
- @weave.op()
- async def score(self, output: Optional[dict], query: str, answer: str) -> Any:
- """Score the correctness of the predictions by comparing the pred, query, target.
- Args:
- - output: the dict that will be provided by the model that is evaluated
- - query: the question asked - as defined in the dataset
- - answer: the target answer - as defined in the dataset
- Returns:
- - single dict {metric name: single evaluation value}"""
-
- # get_model is defined as general model getter based on provided params (OpenAI,HF...)
- eval_model = get_model(
- model_name = self.model_name,
- prompt = self.prompt
- device = self.device,
- )
- # async evaluation to speed up evaluation - this doesn't have to be async
- grade = await eval_model.async_predict(
- {
- "query": query,
- "answer": answer,
- "result": output.get("result"),
- }
- )
- # output parsing - could be done more reobustly with pydantic
- evaluation = "incorrect" not in grade["text"].strip().lower()
-
- # the column name displayed in Weave
- return {"correct": evaluation}
-
- @weave.op()
- def summarize(self, score_rows: WeaveList) -> Optional[dict]:
- """Aggregate all the scores that are calculated for each row by the scoring function.
- Args:
- - score_rows: a WeaveList object, nested dict of metrics and scores
- Returns:
- - nested dict with the same structure as the input"""
-
- # if nothing is provided the weave.flow.scorer.auto_summarize function is used
- # return auto_summarize(score_rows)
-
- valid_data = [x.get("correct") for x in score_rows if x.get("correct") is not None]
- count_true = list(valid_data).count(True)
- int_data = [int(x) for x in valid_data]
-
- sample_mean = np.mean(int_data) if int_data else 0
- sample_variance = np.var(int_data) if int_data else 0
- sample_error = np.sqrt(sample_variance / len(int_data)) if int_data else 0
-
- # the extra "correct" layer is not necessary but adds structure in the UI
- return {
- "correct": {
- "true_count": count_true,
- "true_fraction": sample_mean,
- "stderr": sample_error,
- }
- }
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-To use this as a scorer, you would initialize it and pass it to `scorers` argument in your `Evaluation like this:
-
-
-
- ```python
- evaluation = weave.Evaluation(dataset=questions, scorers=[CorrectnessLLMJudge()])
- ```
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-## 4. Pulling it all together
-
-To get the same result for your RAG apps:
-
-- Wrap LLM calls & retrieval step functions with `weave.op()`
-- (optional) Create a `Model` subclass with `predict` function and app details
-- Collect examples to evaluate
-- Create scoring functions that score one example
-- Use `Evaluation` class to run evaluations on your examples
-
-**NOTE:** Sometimes the async execution of Evaluations will trigger a rate limit on the models of OpenAI, Anthropic, etc. To prevent that you can set an environment variable to limit the amount of parallel workers e.g. `WEAVE_PARALLELISM=3`.
-
-Here, we show the code in it's entirety.
-
-
-
- ```python
- from openai import OpenAI
- import weave
- from weave import Model
- import numpy as np
- import json
- import asyncio
-
- # Examples we've gathered that we want to use for evaluations
- articles = [
- "Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial “tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too,” one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.",
- "Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.",
- "Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if it's stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities",
- "Rivian and Lucid shares plunge after weak EV earnings reports Shares of electric vehicle makers Rivian and Lucid fell Thursday after the companies reported stagnant production in their fourth-quarter earnings after the bell Wednesday. Rivian shares sank about 25 percent, and Lucids stock dropped around 17 percent. Rivian forecast it will make 57,000 vehicles in 2024, slightly less than the 57,232 vehicles it produced in 2023. Lucid said it expects to make 9,000 vehicles in 2024, more than the 8,428 vehicles it made in 2023.",
- "Mauritius blocks Norwegian cruise ship over fears of a potential cholera outbreak Local authorities on Sunday denied permission for the Norwegian Dawn ship, which has 2,184 passengers and 1,026 crew on board, to access the Mauritius capital of Port Louis, citing “potential health risks.” The Mauritius Ports Authority said Sunday that samples were taken from at least 15 passengers on board the cruise ship. A spokesperson for the U.S.-headquartered Norwegian Cruise Line Holdings said Sunday that 'a small number of guests experienced mild symptoms of a stomach-related illness' during Norwegian Dawns South Africa voyage.",
- "Intuitive Machines lands on the moon in historic first for a U.S. company Intuitive Machines Nova-C cargo lander, named Odysseus after the mythological Greek hero, is the first U.S. spacecraft to soft land on the lunar surface since 1972. Intuitive Machines is the first company to pull off a moon landing — government agencies have carried out all previously successful missions. The company's stock surged in extended trading Thursday, after falling 11 percent in regular trading.",
- "Lunar landing photos: Intuitive Machines Odysseus sends back first images from the moon Intuitive Machines cargo moon lander Odysseus returned its first images from the surface. Company executives believe the lander caught its landing gear sideways on the surface of the moon while touching down and tipped over. Despite resting on its side, the company's historic IM-1 mission is still operating on the moon.",
- ]
-
- def docs_to_embeddings(docs: list) -> list:
- openai = OpenAI()
- document_embeddings = []
- for doc in docs:
- response = (
- openai.embeddings.create(input=doc, model="text-embedding-3-small")
- .data[0]
- .embedding
- )
- document_embeddings.append(response)
- return document_embeddings
-
- article_embeddings = docs_to_embeddings(articles) # Note: you would typically do this once with your articles and put the embeddings & metadata in a database
-
- # We've added a decorator to our retrieval step
- # highlight-next-line
- @weave.op()
- def get_most_relevant_document(query):
- openai = OpenAI()
- query_embedding = (
- openai.embeddings.create(input=query, model="text-embedding-3-small")
- .data[0]
- .embedding
- )
- similarities = [
- np.dot(query_embedding, doc_emb)
- / (np.linalg.norm(query_embedding) * np.linalg.norm(doc_emb))
- for doc_emb in article_embeddings
- ]
- # Get the index of the most similar document
- most_relevant_doc_index = np.argmax(similarities)
- return articles[most_relevant_doc_index]
-
- # We create a Model subclass with some details about our app, along with a predict function that produces a response
- # highlight-next-line
- class RAGModel(Model):
- system_message: str
- model_name: str = "gpt-3.5-turbo-1106"
-
- # highlight-next-line
- @weave.op()
- # highlight-next-line
- def predict(self, question: str) -> dict: # note: `question` will be used later to select data from our evaluation rows
- from openai import OpenAI
- context = get_most_relevant_document(question)
- client = OpenAI()
- query = f"""Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know."
- Context:
- \"\"\"
- {context}
- \"\"\"
- Question: {question}"""
- response = client.chat.completions.create(
- model=self.model_name,
- messages=[
- {"role": "system", "content": self.system_message},
- {"role": "user", "content": query},
- ],
- temperature=0.0,
- response_format={"type": "text"},
- )
- answer = response.choices[0].message.content
- return {'answer': answer, 'context': context}
-
- # highlight-next-line
- weave.init('rag-qa')
- # highlight-next-line
- model = RAGModel(
- system_message="You are an expert in finance and answer questions related to finance, financial services, and financial markets. When responding based on provided information, be sure to cite the source."
- )
-
- # Here is our scoring function uses our question and output to product a score
- # highlight-next-line
- @weave.op()
- # highlight-next-line
- async def context_precision_score(question, output):
- context_precision_prompt = """Given question, answer and context verify if the context was useful in arriving at the given answer. Give verdict as "1" if useful and "0" if not with json output.
- Output in only valid JSON format.
-
- question: {question}
- context: {context}
- answer: {answer}
- verdict: """
- client = OpenAI()
-
- prompt = context_precision_prompt.format(
- question=question,
- context=output['context'],
- answer=output['answer'],
- )
-
- response = client.chat.completions.create(
- model="gpt-4-turbo-preview",
- messages=[{"role": "user", "content": prompt}],
- response_format={ "type": "json_object" }
- )
- response_message = response.choices[0].message
- response = json.loads(response_message.content)
- return {
- "verdict": int(response["verdict"]) == 1,
- }
-
- questions = [
- {"question": "What significant result was reported about Zealand Pharma's obesity trial?"},
- {"question": "How much did Berkshire Hathaway's cash levels increase in the fourth quarter?"},
- {"question": "What is the goal of Highmark Health's integration of Google Cloud and Epic Systems technology?"},
- {"question": "What were Rivian and Lucid's vehicle production forecasts for 2024?"},
- {"question": "Why was the Norwegian Dawn cruise ship denied access to Mauritius?"},
- {"question": "Which company achieved the first U.S. moon landing since 1972?"},
- {"question": "What issue did Intuitive Machines' lunar lander encounter upon landing on the moon?"}
- ]
-
- # We define an Evaluation object and pass our example questions along with scoring functions
- # highlight-next-line
- evaluation = weave.Evaluation(dataset=questions, scorers=[context_precision_score])
- # highlight-next-line
- asyncio.run(evaluation.evaluate(model))
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-## Conclusion
-
-We've learned how to build observability into different steps of our applications, like the retrieval step in this example.
-We've also learned how to build more complex scoring functions, like an LLM judge, for doing automatic evaluation of application responses.
-
-```
-
-```import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Track data flows and app metadata
-
-In the [Track LLM inputs & outputs](/quickstart) tutorial, the basics of tracking the inputs and outputs of your LLMs was covered.
-
-In this tutorial you will learn how to:
-
-- **Track data** as it flows through your application
-- **Track metadata** at call time
-
-## Tracking nested function calls
-
-LLM-powered applications can contain multiple LLMs calls and additional data processing and validation logic that is important to monitor. Even deep nested call structures common in many apps, Weave will keep track of the parent-child relationships in nested functions as long as `weave.op()` is added to every function you'd like to track.
-
-Building on our [basic tracing example](/quickstart), we will now add additional logic to count the returned items from our LLM and wrap them all in a higher level function. We'll then add `weave.op()` to trace every function, its call order and its parent-child relationship:
-
-
-
-
- ```python
- import weave
- import json
- from openai import OpenAI
-
- client = OpenAI(api_key="...")
-
- # highlight-next-line
- @weave.op()
- def extract_dinos(sentence: str) -> dict:
- response = client.chat.completions.create(
- model="gpt-4o",
- messages=[
- {
- "role": "system",
- "content": """Extract any dinosaur `name`, their `common_name`, \
- names and whether its `diet` is a herbivore or carnivore, in JSON format."""
- },
- {
- "role": "user",
- "content": sentence
- }
- ],
- response_format={ "type": "json_object" }
- )
- return response.choices[0].message.content
-
- # highlight-next-line
- @weave.op()
- def count_dinos(dino_data: dict) -> int:
- # count the number of items in the returned list
- k = list(dino_data.keys())[0]
- return len(dino_data[k])
-
- # highlight-next-line
- @weave.op()
- def dino_tracker(sentence: str) -> dict:
- # extract dinosaurs using a LLM
- dino_data = extract_dinos(sentence)
-
- # count the number of dinosaurs returned
- dino_data = json.loads(dino_data)
- n_dinos = count_dinos(dino_data)
- return {"n_dinosaurs": n_dinos, "dinosaurs": dino_data}
-
- # highlight-next-line
- weave.init('jurassic-park')
-
- sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \
- both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \
- Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below."""
-
- result = dino_tracker(sentence)
- print(result)
- ```
- **Nested functions**
-
- When you run the above code you will see the the inputs and outputs from the two nested functions (`extract_dinos` and `count_dinos`), as well as the automatically-logged OpenAI trace.
-
- 
-
-
-
-
- ```typescript
- import OpenAI from 'openai';
- import * as weave from 'weave';
-
- const openai = weave.wrapOpenAI(new OpenAI());
-
- const extractDinos = weave.op(async (sentence: string) => {
- const response = await openai.chat.completions.create({
- model: 'gpt-4o',
- messages: [
- {
- role: 'system',
- content:
- 'Extract any dinosaur `name`, their `common_name`, names and whether its `diet` is a herbivore or carnivore, in JSON format.',
- },
- {role: 'user', content: sentence},
- ],
- response_format: {type: 'json_object'},
- });
- return response.choices[0].message.content;
- });
-
- const countDinos = weave.op(async (dinoData: string) => {
- const parsed = JSON.parse(dinoData);
- return Object.keys(parsed).length;
- });
-
- const dinoTracker = weave.op(async (sentence: string) => {
- const dinoData = await extractDinos(sentence);
- const nDinos = await countDinos(dinoData);
- return {nDinos, dinoData};
- });
-
- async function main() {
- await weave.init('jurassic-park');
-
- const sentence = `I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike),
- both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant
- Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below.`;
-
- const result = await dinoTracker(sentence);
- console.log(result);
- }
-
- main();
-
- ```
-
- **Nested functions**
-
- When you run the above code you will see the the inputs and outputs from the two nested functions (`extractDinos` and `countDinos`), as well as the automatically-logged OpenAI trace.
-
-
- 
-
-
-
-
-## Tracking metadata
-
-Tracking metadata can be done easily by using the `weave.attributes` context manager and passing it a dictionary of the metadata to track at call time.
-
-Continuing our example from above:
-
-
-
- ```python
- import weave
-
- weave.init('jurassic-park')
-
- sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \
- both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \
- Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below."""
-
- # track metadata alongside our previously defined function
- # highlight-next-line
- with weave.attributes({'user_id': 'lukas', 'env': 'production'}):
- result = dino_tracker(sentence)
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-:::note
-It's recommended to use metadata tracking to track metadata at run time, e.g. user ids or whether or not the call is part of the development process or is in production etc.
-
-To track system attributes, such as a System Prompt, we recommend using [weave Models](guides/core-types/models)
-:::
-
-## What's next?
-
-- Follow the [App Versioning tutorial](/tutorial-weave_models) to capture, version and organize ad-hoc prompt, model, and application changes.import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# App versioning
-
-Tracking the [inputs, outputs, metadata](/quickstart) as well as [data flowing through your app](/tutorial-tracing_2) is critical to understanding the performance of your system. However **versioning your app over time** is also critical to understand how modifications to your code or app attributes change your outputs. Weave's `Model` class is how these changes can be tracked in Weave.
-
-In this tutorial you'll learn:
-
-- How to use Weave `Model` to track and version your app and its attributes.
-- How to export, modify and re-use a Weave `Model` already logged.
-
-## Using `weave.Model`
-
-:::warning
-
-The `weave.Model` class is currently only supported in Python!
-
-:::
-
-Using Weave `Model`s means that attributes such as model vendor ids, prompts, temperature, and more are stored and versioned when they change.
-
-To create a `Model` in Weave, you need the following:
-
-- a class that inherits from `weave.Model`
-- type definitions on all class attributes
-- a typed `invoke` function with the `@weave.op()` decorator
-
-When you change the class attributes or the code that defines your model, **these changes will be logged and the version will be updated**. This ensures that you can compare the generations across different versions of your app.
-
-In the example below, the **model name, temperature and system prompt will be tracked and versioned**:
-
-
-
- ```python
- import json
- from openai import OpenAI
-
- import weave
-
- @weave.op()
- def extract_dinos(wmodel: weave.Model, sentence: str) -> dict:
- response = wmodel.client.chat.completions.create(
- model=wmodel.model_name,
- temperature=wmodel.temperature,
- messages=[
- {
- "role": "system",
- "content": wmodel.system_prompt
- },
- {
- "role": "user",
- "content": sentence
- }
- ],
- response_format={ "type": "json_object" }
- )
- return response.choices[0].message.content
-
- # Sub-class with a weave.Model
- # highlight-next-line
- class ExtractDinos(weave.Model):
- client: OpenAI = None
- model_name: str
- temperature: float
- system_prompt: str
-
- # Ensure your function is called `invoke` or `predict`
- # highlight-next-line
- @weave.op()
- # highlight-next-line
- def invoke(self, sentence: str) -> dict:
- dino_data = extract_dinos(self, sentence)
- return json.loads(dino_data)
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-Now you can instantiate and call the model with `invoke`:
-
-
-
- ```python
- weave.init('jurassic-park')
- client = OpenAI()
-
- system_prompt = """Extract any dinosaur `name`, their `common_name`, \
- names and whether its `diet` is a herbivore or carnivore, in JSON format."""
-
- # highlight-next-line
- dinos = ExtractDinos(
- client=client,
- model_name='gpt-4o',
- temperature=0.4,
- system_prompt=system_prompt
- )
-
- sentence = """I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), \
- both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant \
- Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below."""
-
- # highlight-next-line
- result = dinos.invoke(sentence)
- print(result)
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-Now after calling `.invoke` you can see the trace in Weave **now tracks the model attributes as well as the code** for the model functions that have been decorated with `weave.op()`. You can see the model is also versioned, "v21" in this case, and if you click on the model **you can see all of the calls** that have used that version of the model
-
-
-
-**A note on using `weave.Model`:**
-
-- You can use `predict` instead of `invoke` for the name of the function in your Weave `Model` if you prefer.
-- If you want other class methods to be tracked by weave they need to be wrapped in `weave.op()`
-- Attributes starting with an underscore are ignored by weave and won't be logged
-
-## Exporting and re-using a logged `weave.Model`
-
-Because Weave stores and versions Models that have been invoked, it is possible to export and re-use these models.
-
-**Get the Model ref**
-In the Weave UI you can get the Model ref for a particular version
-
-**Using the Model**
-Once you have the URI of the Model object, you can export and re-use it. Note that the exported model is already initialised and ready to use:
-
-
-
- ```python
- # the exported weave model is already initialised and ready to be called
- # highlight-next-line
- new_dinos = weave.ref("weave:///morgan/jurassic-park/object/ExtractDinos:ey4udBU2MU23heQFJenkVxLBX4bmDsFk7vsGcOWPjY4").get()
-
- # set the client to the openai client again
- new_dinos.client = client
-
- new_sentence = """I also saw an Ankylosaurus grazing on giant ferns"""
- new_result = new_dinos.invoke(new_sentence)
- print(new_result)
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-Here you can now see the name Model version (v21) was used with the new input:
-
-
-
-## What's next?
-
-- Follow the [Build an Evaluation pipeline tutorial](/tutorial-eval) to start iteratively improving your applications.import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Costs
-
-## Adding a custom cost
-
-
-
- You can add a custom cost by using the [`add_cost`](/reference/python-sdk/weave/trace/weave.trace.weave_client#method-add_cost) method.
- The three required fields are `llm_id`, `prompt_token_cost`, and `completion_token_cost`.
- `llm_id` is the name of the LLM (e.g. `gpt-4o`). `prompt_token_cost` and `completion_token_cost` are cost per token for the LLM (if the LLM prices were specified inper million tokens, make sure to convert the value).
- You can also set `effective_date` to a datetime, to make the cost effective at a specific date, this defaults to the current date.
-
- ```python
- import weave
- from datetime import datetime
-
- client = weave.init("my_custom_cost_model")
-
- client.add_cost(
- llm_id="your_model_name",
- prompt_token_cost=0.01,
- completion_token_cost=0.02
- )
-
- client.add_costs(
- llm_id="your_model_name",
- prompt_token_cost=10,
- completion_token_cost=20,
- # If for example I want to raise the price of the model after a certain date
- effective_date=datetime(2025, 4, 22),
- )
- ```
-
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-
-## Querying for costs
-
-
-
- You can query for costs by using the [`query_costs`](/reference/python-sdk/weave/trace/weave.trace.weave_client#method-query_costs) method.
- There are a few ways to query for costs, you can pass in a singular cost id, or a list of LLM model names.
-
- ```python
- import weave
-
- client = weave.init("my_custom_cost_model")
-
- costs = client.query_costs(llm_ids=["your_model_name"])
-
- cost = client.query_costs(costs[0].id)
- ```
-
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-
-## Purging a custom cost
-
-
-
- You can purge a custom cost by using the [`purge_costs`](/reference/python-sdk/weave/trace/weave.trace.weave_client#method-purge_costs) method. You pass in a list of cost ids, and the costs with those ids are purged.
-
- ```python
- import weave
-
- client = weave.init("my_custom_cost_model")
-
- costs = client.query_costs(llm_ids=["your_model_name"])
- client.purge_costs([cost.id for cost in costs])
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-## Calculating costs for a Project
-
-
-
- You can calculate costs for a project by using our `calls_query` and adding `include_costs=True` with a little bit of setup.
-
- ```python
- import weave
-
- weave.init("project_costs")
- @weave.op()
- def get_costs_for_project(project_name: str):
- total_cost = 0
- requests = 0
-
- client = weave.init(project_name)
- # Fetch all the calls in the project
- calls = list(
- client.get_calls(filter={"trace_roots_only": True}, include_costs=True)
- )
-
- for call in calls:
- # If the call has costs, we add them to the total cost
- if call.summary["weave"] is not None and call.summary["weave"].get("costs", None) is not None:
- for k, cost in call.summary["weave"]["costs"].items():
- requests += cost["requests"]
- total_cost += cost["prompt_tokens_total_cost"]
- total_cost += cost["completion_tokens_total_cost"]
-
- # We return the total cost, requests, and calls
- return {
- "total_cost": total_cost,
- "requests": requests,
- "calls": len(calls),
- }
-
- # Since we decorated our function with @weave.op(),
- # our totals are stored in weave for historic cost total calculations
- get_costs_for_project("my_custom_cost_model")
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-## Setting up a custom model with custom costs
-
-Try our cookbook for a [Setting up costs with a custom model](/reference/gen_notebooks/custom_model_cost) or 
Open in Colab
[
-```
-
-to deploy a gcp cloud function that serves your model. The last line of the deployment will look like `Service URL: `. Visit `/docs` to interact with your model!
-
-Run
-
-```
-weave deploy gcp --help
-```
-
-to see all command line options.]# Evaluations
-
-Evaluation-driven development helps you reliably iterate on an application. The `Evaluation` class is designed to assess the performance of a `Model` on a given `Dataset` or set of examples using scoring functions.
-
-
-
-```python
-import weave
-from weave import Evaluation
-import asyncio
-
-# Collect your examples
-examples = [
- {"question": "What is the capital of France?", "expected": "Paris"},
- {"question": "Who wrote 'To Kill a Mockingbird'?", "expected": "Harper Lee"},
- {"question": "What is the square root of 64?", "expected": "8"},
-]
-
-# Define any custom scoring function
-@weave.op()
-def match_score1(expected: str, model_output: dict) -> dict:
- # Here is where you'd define the logic to score the model output
- return {'match': expected == model_output['generated_text']}
-
-@weave.op()
-def function_to_evaluate(question: str):
- # here's where you would add your LLM call and return the output
- return {'generated_text': 'Paris'}
-
-# Score your examples using scoring functions
-evaluation = Evaluation(
- dataset=examples, scorers=[match_score1]
-)
-
-# Start tracking the evaluation
-weave.init('intro-example')
-# Run the evaluation
-asyncio.run(evaluation.evaluate(function_to_evaluate))
-```
-
-## Create an Evaluation
-
-To systematically improve your application, it's helpful to test your changes against a consistent dataset of potential inputs so that you catch regressions and can inspect your apps behaviour under different conditions. Using the `Evaluation` class, you can be sure you're comparing apples-to-apples by keeping track of all of the details that you're experimenting and evaluating with.
-
-Weave will take each example, pass it through your application and score the output on multiple custom scoring functions. By doing this, you'll have a view of the performance of your application, and a rich UI to drill into individual outputs and scores.
-
-### Define an evaluation dataset
-
-First, define a [Dataset](/guides/core-types/datasets) or list of dictionaries with a collection of examples to be evaluated. These examples are often failure cases that you want to test for, these are similar to unit tests in Test-Driven Development (TDD).
-
-### Defining scoring functions
-
-Then, create a list of scoring functions. These are used to score each example. Each function should have a `model_output` and optionally, other inputs from your examples, and return a dictionary with the scores.
-
-Scoring functions need to have a `model_output` keyword argument, but the other arguments are user defined and are taken from the dataset examples. It will only take the necessary keys by using a dictionary key based on the argument name.
-
-This will take `expected` from the dictionary for scoring.
-
-```python
-import weave
-
-# Collect your examples
-examples = [
- {"question": "What is the capital of France?", "expected": "Paris"},
- {"question": "Who wrote 'To Kill a Mockingbird'?", "expected": "Harper Lee"},
- {"question": "What is the square root of 64?", "expected": "8"},
-]
-
-# Define any custom scoring function
-@weave.op()
-def match_score1(expected: str, model_output: dict) -> dict:
- # Here is where you'd define the logic to score the model output
- return {'match': expected == model_output['generated_text']}
-```
-
-### Optional: Define a custom `Scorer` class
-
-In some applications we want to create custom `Scorer` classes - where for example a standardized `LLMJudge` class should be created with specific parameters (e.g. chat model, prompt), specific scoring of each row, and specific calculation of an aggregate score.
-
-See the tutorial on defining a `Scorer` class in the next chapter on [Model-Based Evaluation of RAG applications](/tutorial-rag#optional-defining-a-scorer-class) for more information.
-
-### Define a Model to evaluate
-
-To evaluate a `Model`, call `evaluate` on it using an `Evaluation`. `Models` are used when you have attributes that you want to experiment with and capture in weave.
-
-```python
-from weave import Model, Evaluation
-import asyncio
-
-class MyModel(Model):
- prompt: str
-
- @weave.op()
- def predict(self, question: str):
- # here's where you would add your LLM call and return the output
- return {'generated_text': 'Hello, ' + self.prompt}
-
-model = MyModel(prompt='World')
-
-evaluation = Evaluation(
- dataset=examples, scorers=[match_score1]
-)
-weave.init('intro-example') # begin tracking results with weave
-asyncio.run(evaluation.evaluate(model))
-```
-
-This will run `predict` on each example and score the output with each scoring functions.
-
-#### Custom Naming
-
-You can change the name of the Evaluation itself by passing a `name` parameter to the `Evaluation` class.
-
-```python
-evaluation = Evaluation(
- dataset=examples, scorers=[match_score1], name="My Evaluation"
-)
-```
-
-You can also change the name of individual evaluations by setting the `display_name` key of the `__weave` dictionary.
-
-:::note
-
-Using the `__weave` dictionary sets the call display name which is distinct from the Evaluation object name. In the
-UI, you will see the display name if set, otherwise the Evaluation object name will be used.
-
-:::
-
-```python
-evaluation = Evaluation(
- dataset=examples, scorers=[match_score1]
-)
-evaluation.evaluate(model, __weave={"display_name": "My Evaluation Run"})
-```
-
-### Define a function to evaluate
-
-Alternatively, you can also evaluate a function that is wrapped in a `@weave.op()`.
-
-```python
-@weave.op()
-def function_to_evaluate(question: str):
- # here's where you would add your LLM call and return the output
- return {'generated_text': 'some response'}
-
-asyncio.run(evaluation.evaluate(function_to_evaluate))
-```
-
-### Pulling it all together
-
-```python
-from weave import Evaluation, Model
-import weave
-import asyncio
-weave.init('intro-example')
-examples = [
- {"question": "What is the capital of France?", "expected": "Paris"},
- {"question": "Who wrote 'To Kill a Mockingbird'?", "expected": "Harper Lee"},
- {"question": "What is the square root of 64?", "expected": "8"},
-]
-
-@weave.op()
-def match_score1(expected: str, model_output: dict) -> dict:
- return {'match': expected == model_output['generated_text']}
-
-@weave.op()
-def match_score2(expected: dict, model_output: dict) -> dict:
- return {'match': expected == model_output['generated_text']}
-
-class MyModel(Model):
- prompt: str
-
- @weave.op()
- def predict(self, question: str):
- # here's where you would add your LLM call and return the output
- return {'generated_text': 'Hello, ' + question + self.prompt}
-
-model = MyModel(prompt='World')
-evaluation = Evaluation(dataset=examples, scorers=[match_score1, match_score2])
-
-asyncio.run(evaluation.evaluate(model))
-
-@weave.op()
-def function_to_evaluate(question: str):
- # here's where you would add your LLM call and return the output
- return {'generated_text': 'some response' + question}
-
-asyncio.run(evaluation.evaluate(function_to_evaluate))
-```import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Feedback
-
-Evaluating LLM applications automatically is challenging. Teams often rely on direct user feedback, particularly from domain experts, who assess the content quality using simple indicators such as thumbs up or down. Developers also actively identify and resolve content issues.
-
-Weave's feedback feature enables users to provide feedback directly within the Weave UI or through the API. You can add emoji reactions, textual notes, and structured data to calls. This feedback helps compile evaluation datasets, monitor application performance, and collect examples for advanced tasks like fine-tuning.
-
-## View and Add Feedback within UI
-
-Reactions and notes are displayed in a column in the calls grid. Hovering over these indicators provides more detail. Use these buttons to add reactions or notes.
-
-
-
-View and edit feedback in the header of the call details page.
-
-
-
-View the feedback for a call on the "Feedback" tab of the call details page.
-
-
-
-Access copy-and-paste examples on the "Use" tab of the call details page to manipulate the feedback for that call using the SDK.
-
-
-
-## SDK
-
-Use the Weave SDK to programmatically add, remove, and query feedback on calls.
-
-### Querying a project's feedback
-
-
-
- ```python
- import weave
- client = weave.init('intro-example')
-
- # Get all feedback in a project
- all_feedback = client.get_feedback()
-
- # Fetch a specific feedback object by id.
- # Note that the API still returns a collection, which is expected
- # to contain zero or one item(s).
- one_feedback = client.get_feedback("")[0]
-
- # Find all feedback objects with a specific reaction. You can specify offset and limit.
- thumbs_up = client.get_feedback(reaction="👍", limit=10)
-
- # After retrieval you can view the details of individual feedback objects.
- for f in client.get_feedback():
- print(f.id)
- print(f.created_at)
- print(f.feedback_type)
- print(f.payload)
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-### Adding feedback to a call
-
-
-
- ```python
- import weave
- client = weave.init('intro-example')
-
- call = client.get_call("")
-
- # Adding an emoji reaction
- call.feedback.add_reaction("👍")
-
- # Adding a note
- call.feedback.add_note("this is a note")
-
- # Adding custom key/value pairs.
- # The first argument is a user-defined "type" string.
- # Feedback must be JSON serializable and less than 1kb when serialized.
- call.feedback.add("correctness", { "value": 5 })
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-### Retrieving the Call UUID
-
-For scenarios where you need to add feedback immediately after a call, you can retrieve the call UUID programmatically during or after the call execution. Here is how to get the UUID of the call from within the operation:
-
-
-
- ```python
-
- import weave
- weave.init("uuid")
-
- @weave.op()
- def simple_operation(input_value):
- # Perform some simple operation
- output = f"Processed {input_value}"
- # Get the current call ID
- current_call = weave.require_current_call()
- call_id = current_call.id
- return output, call_id
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-Additionally, you can use call() method to execute the operation and retrieve the call ID after execution of the function:
-
-
-
- ```python
- import weave
- weave.init("uuid")
-
- @weave.op()
- def simple_operation(input_value):
- return f"Processed {input_value}"
-
- # Execute the operation and retrieve the result and call ID
- result, call = simple_operation.call("example input")
- call_id = call.id
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-### Querying feedback on a call
-
-
-
- ```python
- for f in call.feedback:
- print(f.id)
- print(f.feedback_type)
- print(f.payload)
- ```
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-### Deleting feedback from a call
-
-
-
- ```python
- call.feedback.purge("")
- ```
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Logging media
-
-Weave supports logging and displaying multiple first class media types. Log images with `PIL.Image.Image` and audio with `wave.Wave_read` either directly with the object API, or as the inputs or output of an op.
-
-## Images
-
-Logging type: `PIL.Image.Image`. Here is an example of logging an image with the OpenAI DALL-E API:
-
-
-
-
- ```python
- import weave
- from openai import OpenAI
- import requests
- from PIL import Image
-
- weave.init('image-example')
- client = OpenAI()
-
- @weave.op
- def generate_image(prompt: str) -> Image:
- response = client.images.generate(
- model="dall-e-3",
- prompt=prompt,
- size="1024x1024",
- quality="standard",
- n=1,
- )
- image_url = response.data[0].url
- image_response = requests.get(image_url, stream=True)
- image = Image.open(image_response.raw)
-
- # return a PIL.Image.Image object to be logged as an image
- return image
-
- generate_image("a cat with a pumpkin hat")
- ```
-
-
-
-
- ```typescript
- import {OpenAI} from 'openai';
- import * as weave from 'weave';
-
- async function main() {
- const client = await weave.init('image-example');
- const openai = new OpenAI();
-
- const generateImage = weave.op(async (prompt: string) => {
- const response = await openai.images.generate({
- model: 'dall-e-3',
- prompt: prompt,
- size: '1024x1024',
- quality: 'standard',
- n: 1,
- });
- const imageUrl = response.data[0].url;
- const imgResponse = await fetch(imageUrl);
- const data = Buffer.from(await imgResponse.arrayBuffer());
-
- return weave.weaveImage({data});
- });
-
- generateImage('a cat with a pumpkin hat');
- }
-
- main();
- ```
-
-
-
-
-This image will be logged to weave and automatically displayed in the UI. The following is the trace view for above.
-
-
-
-## Audio
-
-Logging type: `wave.Wave_read`. Here is an example of logging an audio file using openai's speech generation API.
-
-
-
-
- ```python
- import weave
- from openai import OpenAI
- import wave
-
- weave.init("audio-example")
- client = OpenAI()
-
-
- @weave.op
- def make_audio_file_streaming(text: str) -> wave.Wave_read:
- with client.audio.speech.with_streaming_response.create(
- model="tts-1",
- voice="alloy",
- input=text,
- response_format="wav",
- ) as res:
- res.stream_to_file("output.wav")
-
- # return a wave.Wave_read object to be logged as audio
- return wave.open("output.wav")
-
- make_audio_file_streaming("Hello, how are you?")
- ```
-
-
-
-
- ```typescript
- import {OpenAI} from 'openai';
- import * as weave from 'weave';
-
- async function main() {
- await weave.init('audio-example');
- const openai = new OpenAI();
-
- const makeAudioFileStreaming = weave.op(async function audio(text: string) {
- const response = await openai.audio.speech.create({
- model: 'tts-1',
- voice: 'alloy',
- input: text,
- response_format: 'wav',
- });
-
- const chunks: Uint8Array[] = [];
- for await (const chunk of response.body) {
- chunks.push(chunk);
- }
- return weave.weaveAudio({data: Buffer.concat(chunks)});
- });
-
- await makeAudioFileStreaming('Hello, how are you?');
- }
-
- main();
- ```
-
-
-
-
-This audio will be logged to weave and automatically displayed in the UI, with an audio player. The player can be expanded to view the raw audio waveform, in addition to a download button.
-
-
-
-Try our cookbook for [Audio Logging](/reference/gen_notebooks/audio_with_weave) or Open in Colab
[
-```
-
-to run a FastAPI server for that model. Visit [http://0.0.0.0:9996/docs](http://0.0.0.0:9996/docs) to query the model interactively.
-
-## Install FastAPI
-
-```bash
-pip install fastapi uvicorn
-```
-
-## Serve Model
-
-In a terminal, call:
-
-```bash
-weave serve
-```
-
-Get your model ref by navigating to the model and copying it from the UI. It should look like:
-`weave:///your_entity/project-name/YourModel:`
-
-To use it, navigate to the Swagger UI link, click the predict endpoint and then click "Try it out!".]# Tools & Utilities
-
-Weave is developing a set of tools and utilities to help with your workflow and deployment process for AI applications. These are currently in early alpha stages and subject to change. Here's an overview of what we're working on:
-
-## Serve (experimental)
-
-[Serve](/guides/tools/serve) is a feature to expose your Weave ops and models as API endpoints. We're exploring possibilities such as:
-
-- Creating web services for your Weave components
-- Integrating Weave components into existing applications
-- Providing a way to test models in a more production-like setting
-
-## Deploy (experimental)
-
-[Deploy](/guides/tools/deploy) is another alpha-stage utility we're developing to help with deploying Weave ops and models. Some potential features we're considering include:
-
-- Pushing models to cloud platforms
-- Managing different deployment environments
-- Exploring ways to automate parts of the deployment process
-
-Please note that these tools are still in very early stages of development. They may not be fully functional, could change significantly, or might be discontinued. We recommend using them for experimental purposes only at this time.import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-import { DesktopWindow } from '../../../src/components/DesktopImage'
-import TracingCallsMacroImage from '@site/static/img/screenshots/calls_macro.png';
-import TracingCallsFilterImage from '@site/static/img/screenshots/calls_filter.png';
-import BasicCallImage from '@site/static/img/screenshots/basic_call.png';
-
-# Calls
-
-
-
-:::info[Calls]
-Calls are the fundamental building block in Weave. They represent a single execution of a function, including:
-- Inputs (arguments)
-- Outputs (return value)
-- Metadata (duration, exceptions, LLM usage, etc.)
-
-Calls are similar to spans in the [OpenTelemetry](https://opentelemetry.io) data model. A Call can:
-- Belong to a Trace (a collection of calls in the same execution context)
-- Have parent and child Calls, forming a tree structure
-:::
-
-## Creating Calls
-
-There are three main ways to create Calls in Weave:
-
-### 1. Automatic Tracking of LLM Libraries
-
-
-
-
- Weave automatically tracks [calls to common LLM libraries](../integrations/index.md) like `openai`, `anthropic`, `cohere`, and `mistral`. Simply call [`weave.init('project_name')`](../../reference/python-sdk/weave/index.md#function-init) at the start of your program:
-
- ```python showLineNumbers
- import weave
-
- from openai import OpenAI
- client = OpenAI()
-
- # Initialize Weave Tracing
- weave.init('intro-example')
-
- response = client.chat.completions.create(
- model="gpt-4",
- messages=[
- {
- "role": "user",
- "content": "How are you?"
- }
- ],
- temperature=0.8,
- max_tokens=64,
- top_p=1,
- )
- ```
-
-
-
- Weave automatically tracks [calls to common LLM libraries](../integrations/index.md) like `openai`. Simply call [`await weave.init('project_name')`](../../reference/typescript-sdk/weave/functions/init.md) and wrap your OpenAI client with [`weave.wrapOpenAI`](../../reference/typescript-sdk/weave/functions/wrapOpenAI.md) at the start of your program:
-
- ```typescript showLineNumbers
- import OpenAI from 'openai'
- import * as weave from 'weave'
-
- const client = weave.wrapOpenAI(new OpenAI())
-
- // Initialize Weave Tracing
- await weave.init('intro-example')
-
- const response = await client.chat.completions.create({
- model: 'gpt-4',
- messages: [
- {
- role: 'user',
- content: 'How are you?',
- },
- ],
- temperature: 0.8,
- max_tokens: 64,
- top_p: 1,
- });
- ```
-
-
-
-
-
-
-### 2. Decorating/Wrapping Functions
-
-However, often LLM applications have additional logic (such as pre/post processing, prompts, etc.) that you want to track.
-
-
-
- Weave allows you to manually track these calls using the [`@weave.op`](../../reference/python-sdk/weave/index.md#function-op) decorator. For example:
-
- ```python showLineNumbers
- import weave
-
- # Initialize Weave Tracing
- weave.init('intro-example')
-
- # Decorate your function
- @weave.op
- def my_function(name: str):
- return f"Hello, {name}!"
-
- # Call your function -- Weave will automatically track inputs and outputs
- print(my_function("World"))
- ```
-
- This works for both functions as well as methods on classes:
-
-
- ```python showLineNumbers
- import weave
-
- # Initialize Weave Tracing
- weave.init("intro-example")
-
- class MyClass:
- # Decorate your method
- @weave.op
- def my_method(self, name: str):
- return f"Hello, {name}!"
-
- instance = MyClass()
-
- # Call your method -- Weave will automatically track inputs and outputs
- print(instance.my_method("World"))
- ```
-
-
- Weave allows you to manually track these calls by wrapping your function with [`weave.op`](../../reference/typescript-sdk/weave/functions/op.md). For example:
-
- ```typescript showLineNumbers
- import * as weave from 'weave'
-
- await weave.init('intro-example')
-
- function myFunction(name: string) {
- return `Hello, ${name}!`
- }
-
- const myFunctionOp = weave.op(myFunction)
- ```
-
- You can also define the wrapping inline:
-
- ```typescript
- const myFunctionOp = weave.op((name: string) => `Hello, ${name}!`)
- ```
-
- This works for both functions as well as methods on classes:
-
- ```typescript
- class MyClass {
- constructor() {
- this.myMethod = weave.op(this.myMethod)
- }
-
- myMethod(name: string) {
- return `Hello, ${name}!`
- }
- }
- ```
-
-
-
-
-#### Getting a handle to the Call object during execution
-
-
-
- Sometimes it is useful to get a handle to the `Call` object itself. You can do this by calling the `op.call` method, which returns both the result and the `Call` object. For example:
-
- ```python showLineNumbers
- result, call = my_function.call("World")
- ```
-
- Then, `call` can be used to set / update / fetch additional properties (most commonly used to get the ID of the call to be used for feedback).
-
- :::note
- If your op is a method on a class, you need to pass the instance as the first argument to the op (see example below).
- :::
-
- ```python showLineNumbers
- # Notice that we pass the `instance` as the first argument.
- # highlight-next-line
- print(instance.my_method.call(instance, "World"))
- ```
-
-
- ```python showLineNumbers
- import weave
-
- # Initialize Weave Tracing
- weave.init("intro-example")
-
- class MyClass:
- # Decorate your method
- @weave.op
- def my_method(self, name: str):
- return f"Hello, {name}!"
-
- instance = MyClass()
-
- # Call your method -- Weave will automatically track inputs and outputs
- instance.my_method.call(instance, "World")
- ```
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-
-
-#### Call Display Name
-
-
-
- Sometimes you may want to override the display name of a call. You can achieve this in one of four ways:
-
- 1. Change the display name at the time of calling the op:
-
- ```python showLineNumbers
- result = my_function("World", __weave={"display_name": "My Custom Display Name"})
- ```
-
- :::note
-
- Using the `__weave` dictionary sets the call display name which will take precedence over the Op display name.
-
- :::
-
- 2. Change the display name on a per-call basis. This uses the [`Op.call`](../../reference/python-sdk/weave/trace/weave.trace.op.md#function-call) method to return a `Call` object, which you can then use to set the display name using [`Call.set_display_name`](../../reference/python-sdk/weave/trace/weave.trace.weave_client.md#method-set_display_name).
- ```python showLineNumbers
- result, call = my_function.call("World")
- call.set_display_name("My Custom Display Name")
- ```
-
- 3. Change the display name for all Calls of a given Op:
-
- ```python showLineNumbers
- @weave.op(call_display_name="My Custom Display Name")
- def my_function(name: str):
- return f"Hello, {name}!"
- ```
-
- 4. The `call_display_name` can also be a function that takes in a `Call` object and returns a string. The `Call` object will be passed automatically when the function is called, so you can use it to dynamically generate names based on the function's name, call inputs, attributes, etc.
-
- 1. One common use case is just appending a timestamp to the function's name.
-
- ```py
- from datetime import datetime
-
- @weave.op(call_display_name=lambda call: f"{call.func_name}__{datetime.now()}")
- def func():
- return ...
- ```
-
- 2. You can also get creative with custom attributes
-
- ```py
- def custom_attribute_name(call):
- model = call.attributes["model"]
- revision = call.attributes["revision"]
- now = call.attributes["date"]
-
- return f"{model}__{revision}__{now}"
-
- @weave.op(call_display_name=custom_attribute_name)
- def func():
- return ...
-
- with weave.attributes(
- {
- "model": "finetuned-llama-3.1-8b",
- "revision": "v0.1.2",
- "date": "2024-08-01",
- }
- ):
- func() # the display name will be "finetuned-llama-3.1-8b__v0.1.2__2024-08-01"
-
-
- with weave.attributes(
- {
- "model": "finetuned-gpt-4o",
- "revision": "v0.1.3",
- "date": "2024-08-02",
- }
- ):
- func() # the display name will be "finetuned-gpt-4o__v0.1.3__2024-08-02"
- ```
-
-
- **Technical Note:** "Calls" are produced by "Ops". An Op is a function or method that is decorated with `@weave.op`.
- By default, the Op's name is the function name, and the associated calls will have the same display name. The above example shows how to override the display name for all Calls of a given Op. Sometimes, users wish to override the name of the Op itself. This can be achieved in one of two ways:
-
- 1. Set the `name` property of the Op before any calls are logged
- ```python showLineNumbers
- my_function.name = "My Custom Op Name"
- ```
-
- 2. Set the `name` option on the op decorator
- ```python showLineNumbers
- @weave.op(name="My Custom Op Name)
- ```
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-
-
-
-
-#### Attributes
-
-
-
- When calling tracked functions, you can add additional metadata to the call by using the [`weave.attributes`](../../reference/python-sdk/weave/index.md#function-attributes) context manager. In the example below, we add an `env` attribute to the call specified as `'production'`.
-
- ```python showLineNumbers
- # ... continued from above ...
-
- # Add additional "attributes" to the call
- with weave.attributes({'env': 'production'}):
- print(my_function.call("World"))
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-### 3. Manual Call Tracking
-
-You can also manually create Calls using the API directly.
-
-
-
-
- ```python showLineNumbers
- import weave
-
- # Initialize Weave Tracing
- weave.init('intro-example')
-
- def my_function(name: str):
- # Start a call
- call = weave.create_call(op="my_function", inputs={"name": name})
-
- # ... your function code ...
-
- # End a call
- weave.finish_call(call, output="Hello, World!")
-
- # Call your function
- print(my_function("World"))
- ```
-
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
-
-
- * Start a call: [POST `/call/start`](../../reference/service-api/call-start-call-start-post.api.mdx)
- * End a call: [POST `/call/end`](../../reference/service-api/call-end-call-end-post.api.mdx)
- ```bash
- curl -L 'https://trace.wandb.ai/call/start' \
- -H 'Content-Type: application/json' \
- -H 'Accept: application/json' \
- -d '{
- "start": {
- "project_id": "string",
- "id": "string",
- "op_name": "string",
- "display_name": "string",
- "trace_id": "string",
- "parent_id": "string",
- "started_at": "2024-09-08T20:07:34.849Z",
- "attributes": {},
- "inputs": {},
- "wb_run_id": "string"
- }
- }
- ```
-
-
-
-
-
-
-## Viewing Calls
-
-
- To view a call in the web app:
- 1. Navigate to your project's "Traces" tab
- 2. Find the call you want to view in the list
- 3. Click on the call to open its details page
-
- The details page will show the call's inputs, outputs, runtime, and any additional attributes or metadata.
-
- 
-
-
- To view a call using the Python API, you can use the [`get_call`](../../reference/python-sdk/weave/trace/weave.trace.weave_client#method-get_call) method:
-
- ```python
- import weave
-
- # Initialize the client
- client = weave.init("your-project-name")
-
- # Get a specific call by its ID
- call = client.get_call("call-uuid-here")
-
- print(call)
- ```
-
-
-
- ```typescript showLineNumbers
- import * as weave from 'weave'
-
- // Initialize the client
- const client = await weave.init('intro-example')
-
- // Get a specific call by its ID
- const call = await client.getCall('call-uuid-here')
-
- console.log(call)
- ```
-
-
-
- To view a call using the Service API, you can make a request to the [`/call/read`](../../reference/service-api/call-read-call-read-post.api.mdx) endpoint.
-
- ```bash
- curl -L 'https://trace.wandb.ai/call/read' \
- -H 'Content-Type: application/json' \
- -H 'Accept: application/json' \
- -d '{
- "project_id": "string",
- "id": "string",
- }'
- ```
-
-
-
-
-## Updating Calls
-
-Calls are mostly immutable once created, however, there are a few mutations which are supported:
-* [Set Display Name](#set-display-name)
-* [Add Feedback](#add-feedback)
-* [Delete a Call](#delete-a-call)
-
-All of these mutations can be performed from the UI by navigating to the call detail page:
-
-
-
-### Set Display Name
-
-
-
- In order to set the display name of a call, you can use the [`Call.set_display_name`](../../reference/python-sdk/weave/trace/weave.trace.weave_client.md#method-set_display_name) method.
-
- ```python showLineNumbers
- import weave
-
- # Initialize the client
- client = weave.init("your-project-name")
-
- # Get a specific call by its ID
- call = client.get_call("call-uuid-here")
-
- # Set the display name of the call
- call.set_display_name("My Custom Display Name")
- ```
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
- To set the display name of a call using the Service API, you can make a request to the [`/call/update`](../../reference/service-api/call-update-call-update-post.api.mdx) endpoint.
-
- ```bash
- curl -L 'https://trace.wandb.ai/call/update' \
- -H 'Content-Type: application/json' \
- -H 'Accept: application/json' \
- -d '{
- "project_id": "string",
- "call_id": "string",
- "display_name": "string",
- }'
- ```
-
-
-
-### Add Feedback
-
-Please see the [Feedback Documentation](./feedback.md) for more details.
-
-### Delete a Call
-
-
-
- To delete a call using the Python API, you can use the [`Call.delete`](../../reference/python-sdk/weave/trace/weave.trace.weave_client.md#method-delete) method.
-
- ```python showLineNumbers
- import weave
-
- # Initialize the client
- client = weave.init("your-project-name")
-
- # Get a specific call by its ID
- call = client.get_call("call-uuid-here")
-
- # Delete the call
- call.delete()
- ```
-
-
-
- ```plaintext
- This feature is not available in TypeScript yet. Stay tuned!
- ```
-
-
- To delete a call using the Service API, you can make a request to the [`/calls/delete`](../../reference/service-api/calls-delete-calls-delete-post.api.mdx) endpoint.
-
- ```bash
- curl -L 'https://trace.wandb.ai/calls/delete' \
- -H 'Content-Type: application/json' \
- -H 'Accept: application/json' \
- -d '{
- "project_id": "string",
- "call_ids": [
- "string"
- ],
- }'
- ```
-
-
-
-## Querying & Exporting Calls
-
-
-
-The `/calls` page of your project ("Traces" tab) contains a table view of all the Calls in your project. From there, you can:
-* Sort
-* Filter
-* Export
-
-
-
-The Export Modal (shown above) allows you to export your data in a number of formats, as well as shows the Python & CURL equivalents for the selected calls!
-The easiest way to get started is to construct a view in the UI, then learn more about the export API via the generated code snippets.
-
-
-
-
- To fetch calls using the Python API, you can use the [`client.calls`](../../reference/python-sdk/weave/trace/weave.trace.weave_client.md#method-calls) method:
-
- ```python
- import weave
-
- # Initialize the client
- client = weave.init("your-project-name")
-
- # Fetch calls
- calls = client.get_calls(filter=...)
- ```
-
- :::info[Notice: Evolving APIs]
- Currently, it is easier to use the lower-level [`calls_query_stream`](../../reference/python-sdk/weave/trace_server_bindings/weave.trace_server_bindings.remote_http_trace_server#method-calls_query_stream) API as it is more flexible and powerful.
- In the near future, we will move all functionality to the above client API.
-
- ```python
- import weave
-
- # Initialize the client
- client = weave.init("your-project-name")
-
- calls = client.server.calls_query_stream({
- "project_id": "",
- "filter": {},
- "query": {},
- "sort_by": [],
- })
- ```
- :::
-
-
- To fetch calls using the TypeScript API, you can use the [`client.getCalls`](../../reference/typescript-sdk/weave/classes/WeaveClient#getcalls) method.
- ```typescript
- import * as weave from 'weave'
-
- // Initialize the client
- const client = await weave.init('intro-example')
-
- // Fetch calls
- const calls = await client.getCalls(filter=...)
- ```
-
-
- The most powerful query layer is at the Service API. To fetch calls using the Service API, you can make a request to the [`/calls/stream_query`](../../reference/service-api/calls-query-stream-calls-stream-query-post.api.mdx) endpoint.
-
- ```bash
- curl -L 'https://trace.wandb.ai/calls/stream_query' \
- -H 'Content-Type: application/json' \
- -H 'Accept: application/json' \
- -d '{
- "project_id": "string",
- "filter": {
- "op_names": [
- "string"
- ],
- "input_refs": [
- "string"
- ],
- "output_refs": [
- "string"
- ],
- "parent_ids": [
- "string"
- ],
- "trace_ids": [
- "string"
- ],
- "call_ids": [
- "string"
- ],
- "trace_roots_only": true,
- "wb_user_ids": [
- "string"
- ],
- "wb_run_ids": [
- "string"
- ]
- },
- "limit": 100,
- "offset": 0,
- "sort_by": [
- {
- "field": "string",
- "direction": "asc"
- }
- ],
- "query": {
- "$expr": {}
- },
- "include_costs": true,
- "include_feedback": true,
- "columns": [
- "string"
- ],
- "expand_columns": [
- "string"
- ]
- }'
- ```
-
-
-
-{/* ## Compare Calls
-
-:::info[Comming Soon]
-::: */}
-
-## Call FAQ
-
-{/* TODO:
-Common Questions / Variations:
-* Images
-* Ops
-* Cost?
-* General data model
-*/}
-
-#### Call Schema
-
-Please see the [schema](../../reference/python-sdk/weave/trace_server/weave.trace_server.trace_server_interface#class-callschema) for a complete list of fields.
-
-
-| Property | Type | Description |
-|----------|------|-------------|
-| id | string (uuid) | Unique identifier for the call |
-| project_id | string (optional) | Associated project identifier |
-| op_name | string | Name of the operation (can be a reference) |
-| display_name | string (optional) | User-friendly name for the call |
-| trace_id | string (uuid) | Identifier for the trace this call belongs to |
-| parent_id | string (uuid) | Identifier of the parent call |
-| started_at | datetime | Timestamp when the call started |
-| attributes | Dict[str, Any] | User-defined metadata about the call |
-| inputs | Dict[str, Any] | Input parameters for the call |
-| ended_at | datetime (optional) | Timestamp when the call ended |
-| exception | string (optional) | Error message if the call failed |
-| output | Any (optional) | Result of the call |
-| summary | Optional[SummaryMap] | Post-execution summary information |
-| wb_user_id | Optional[str] | Associated Weights & Biases user ID |
-| wb_run_id | Optional[str] | Associated Weights & Biases run ID |
-| deleted_at | datetime (optional) | Timestamp of call deletion, if applicable |
-
-The table above outlines the key properties of a Call in Weave. Each property plays a crucial role in tracking and managing function calls:
-
-- The `id`, `trace_id`, and `parent_id` fields help in organizing and relating calls within the system.
-- Timing information (`started_at`, `ended_at`) allows for performance analysis.
-- The `attributes` and `inputs` fields provide context for the call, while `output` and `summary` capture the results.
-- Integration with Weights & Biases is facilitated through `wb_user_id` and `wb_run_id`.
-
-This comprehensive set of properties enables detailed tracking and analysis of function calls throughout your project.
-
-
-Calculated Fields:
- * Cost
- * Duration
- * Status
-
-# Tracing
-
-Weave provides powerful tracing capabilities to track and version objects and function calls in your applications. This comprehensive system enables better monitoring, debugging, and iterative development of AI-powered applications, allowing you to "track insights between commits."
-
-## Key Tracing Features
-
-Weave's tracing functionality comprises three main components:
-
-### Calls
-
-[Calls](/guides/tracking/tracing) trace function calls, inputs, and outputs, enabling you to:
-- Analyze data flow through your application
-- Debug complex interactions between components
-- Optimize application performance based on call patterns
-
-### Ops
-
-[Ops](/guides/tracking/ops) are automatically versioned and tracked functions (which produce Calls) that allow you to:
-- Monitor function performance and behavior
-- Maintain a record of function modifications
-- Ensure experiment reproducibility
-
-### Objects
-
-[Objects](/guides/tracking/objects) form Weave's extensible serialization layer, automatically versioning runtime objects (often the inputs and outputs of Calls). This feature allows you to:
-- Track changes in data structures over time
-- Maintain a clear history of object modifications
-- Easily revert to previous versions when needed
-
-By leveraging these tracing capabilities, you can gain deeper insights into your application's behavior, streamline your development process, and build more robust AI-powered systems.
diff --git a/docs/static/llms.txt b/docs/static/llms.txt
deleted file mode 100644
index cdbe9e5d9e6..00000000000
--- a/docs/static/llms.txt
+++ /dev/null
@@ -1,111 +0,0 @@
-# Weave
-> Weave is a lightweight toolkit for tracking and evaluating LLM applications, built by Weights & Biases.
-
-This document contains links to all the documentation for Weave.
-
-## Docs
-
-- [introduction](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/introduction.md):
-- [quickstart](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/quickstart.md):
-- [tutorial-eval](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/tutorial-eval.md):
-- [tutorial-rag](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/tutorial-rag.md):
-- [tutorial-tracing_2](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/tutorial-tracing_2.md):
-- [tutorial-weave_models](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/tutorial-weave_models.md):
-- [costs](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/costs.md):
-- [datasets](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/datasets.md):
-- [deploy](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tools/deploy.md):
-- [evaluations](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/evaluations.md):
-- [feedback](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/feedback.md):
-- [media](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/media.md):
-- [models](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/models.md):
-- [objects](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/objects.md):
-- [ops](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/ops.md):
-- [platform](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/platform/index.md):
-- [prod_dashboard](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/cookbooks/prod_dashboard.md):
-- [prompts](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/prompts.md):
-- [scorers](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/evaluation/scorers.md):
-- [serve](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tools/serve.md):
-- [tools](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tools/index.md):
-- [tracing](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/tracing.mdx):
-- [tracking](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/index.md):
-
-## Optional
-
-- [01-intro_notebook](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/01-intro_notebook.md):
-- [CallSchema](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/interfaces/CallSchema.md):
-- [CallsFilter](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/interfaces/CallsFilter.md):
-- [Dataset](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/classes/Dataset.md):
-- [Evaluation](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/classes/Evaluation.md):
-- [Op](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/type-aliases/Op.md):
-- [README](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/README.md):
-- [WeaveClient](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/classes/WeaveClient.md):
-- [WeaveObject](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/classes/WeaveObject.md):
-- [anthropic](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/anthropic.md):
-- [audio_with_weave](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/audio_with_weave.md):
-- [call-end-call-end-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-end-call-end-post.api.mdx):
-- [call-read-call-read-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-read-call-read-post.api.mdx):
-- [call-start-batch-call-upsert-batch-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-start-batch-call-upsert-batch-post.api.mdx):
-- [call-start-call-start-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-start-call-start-post.api.mdx):
-- [call-update-call-update-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-update-call-update-post.api.mdx):
-- [calls-delete-calls-delete-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/calls-delete-calls-delete-post.api.mdx):
-- [calls-query-stats-calls-query-stats-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/calls-query-stats-calls-query-stats-post.api.mdx):
-- [calls-query-stream-calls-stream-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/calls-query-stream-calls-stream-query-post.api.mdx):
-- [cerebras](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/cerebras.md):
-- [chain_of_density](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/chain_of_density.md):
-- [codegen](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/codegen.md):
-- [cohere](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/cohere.md):
-- [cost-create-cost-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/cost-create-cost-create-post.api.mdx):
-- [cost-purge-cost-purge-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/cost-purge-cost-purge-post.api.mdx):
-- [cost-query-cost-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/cost-query-cost-query-post.api.mdx):
-- [custom_model_cost](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/custom_model_cost.md):
-- [dspy](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/dspy.md):
-- [dspy_prompt_optimization](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/dspy_prompt_optimization.md):
-- [fastapi.info](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/fastapi.info.mdx):
-- [feedback-create-feedback-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/feedback-create-feedback-create-post.api.mdx):
-- [feedback-purge-feedback-purge-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/feedback-purge-feedback-purge-post.api.mdx):
-- [feedback-query-feedback-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/feedback-query-feedback-query-post.api.mdx):
-- [feedback_prod](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/feedback_prod.md):
-- [file-content-file-content-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/file-content-file-content-post.api.mdx):
-- [file-create-file-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/file-create-file-create-post.api.mdx):
-- [google-gemini](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/google-gemini.md):
-- [groq](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/groq.md):
-- [import_from_csv](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/import_from_csv.md):
-- [init](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/init.md):
-- [instructor](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/instructor.md):
-- [langchain](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/langchain.md):
-- [leaderboard_quickstart](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/leaderboard_quickstart.md):
-- [litellm](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/litellm.md):
-- [llamaindex](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/llamaindex.md):
-- [local_models](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/local_models.md):
-- [login](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/login.md):
-- [mistral](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/mistral.md):
-- [multi-agent-structured-output](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/multi-agent-structured-output.md):
-- [notdiamond](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/notdiamond.md):
-- [obj-create-obj-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/obj-create-obj-create-post.api.mdx):
-- [obj-read-obj-read-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/obj-read-obj-read-post.api.mdx):
-- [objs-query-objs-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/objs-query-objs-query-post.api.mdx):
-- [online_monitoring](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/online_monitoring.md):
-- [op](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/op.md):
-- [openai](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/openai.md):
-- [openrouter](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/openrouter.md):
-- [pii](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/pii.md):
-- [read-root-health-get.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/read-root-health-get.api.mdx):
-- [refs-read-batch-refs-read-batch-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/refs-read-batch-refs-read-batch-post.api.mdx):
-- [requireCurrentCallStackEntry](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/requireCurrentCallStackEntry.md):
-- [requireCurrentChildSummary](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/requireCurrentChildSummary.md):
-- [server-info-server-info-get.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/server-info-server-info-get.api.mdx):
-- [table-create-table-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/table-create-table-create-post.api.mdx):
-- [table-query-stats-table-query-stats-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/table-query-stats-table-query-stats-post.api.mdx):
-- [table-query-table-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/table-query-table-query-post.api.mdx):
-- [table-update-table-update-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/table-update-table-update-post.api.mdx):
-- [together_ai](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/together_ai.md):
-- [weave.trace.feedback](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace/weave.trace.feedback.md):
-- [weave.trace.op](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace/weave.trace.op.md):
-- [weave.trace.util](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace/weave.trace.util.md):
-- [weave.trace.weave_client](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace/weave.trace.weave_client.md):
-- [weave.trace_server.interface.query](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace_server/interface/weave.trace_server.interface.query.md):
-- [weave.trace_server.trace_server_interface](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace_server/weave.trace_server.trace_server_interface.md):
-- [weave.trace_server_bindings.remote_http_trace_server](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace_server_bindings/weave.trace_server_bindings.remote_http_trace_server.md):
-- [weaveAudio](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/weaveAudio.md):
-- [weaveImage](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/weaveImage.md):
-- [wrapOpenAI](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/wrapOpenAI.md):
From ae0cc04625d590cc3a722d80adef64a3281bcc72 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:03:16 -0700
Subject: [PATCH 02/15] Missed adding the updated toml and the action
---
.github/workflows/generate-llms.yaml | 47 ++++++++++++++++++++++++++++
pyproject.toml | 4 +++
2 files changed, 51 insertions(+)
create mode 100644 .github/workflows/generate-llms.yaml
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
new file mode 100644
index 00000000000..f9ccb3ec1bd
--- /dev/null
+++ b/.github/workflows/generate-llms.yaml
@@ -0,0 +1,47 @@
+name: Generate llms.txt
+
+on:
+ push:
+ branches:
+ - pr-3045-gh-action # Adjust if your default branch has a different name
+
+jobs:
+ generate-llms:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3.x' # Specify the required Python version
+
+ # Step 3: Install Hatch
+ - name: Install Hatch
+ run: pip install hatch
+
+ # Step 4: Install dependencies for docs
+ - name: Install documentation dependencies
+ run: hatch env create llmspy
+
+ # Step 5: Run the script
+ - name: Generate llms.txt
+ run: hatch run llmspy python weave/docs/scripts/generate-llmstxt.py
+
+ # Step 6: Verify generated file (optional)
+ - name: Check if llms.txt exists
+ run: |
+ if [ ! -f weave/docs/static/llms.txt ]; then
+ echo "Error: llms.txt was not generated."
+ exit 1
+ fi
+ - name: Commit and push changes
+ run: |
+ git config user.name "llms.py GitHub Actions Bot"
+ git config user.email "actions@github.com"
+ git add weave/docs/static/llms.txt
+ git diff-index --quiet HEAD || git commit -m "Auto-generate llms.txt"
+ git push origin test-llmspy-action
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/pyproject.toml b/pyproject.toml
index 5d5db9f1bc4..6c6caae8e01 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,6 +52,7 @@ anthropic = ["anthropic>=0.18.0"]
cerebras = ["cerebras-cloud-sdk"]
cohere = ["cohere>=5.9.1,<5.9.3"]
dspy = ["dspy>=0.1.5", "litellm<=1.49.1"]
+llmspy = ["PyYAML>=6.0"]
google_ai_studio = ["google-generativeai>=0.8.3"]
groq = ["groq>=0.9.0"]
instructor = ["instructor>=1.4.3"]
@@ -134,6 +135,9 @@ exclude = [
"weave/clear_cache.py",
]
+[tool.hatch.envs.llmspy]
+dependencies = ["PyYAML>=6.0"]
+
[tool.pytest.ini_options]
filterwarnings = [
From cb177e38948246d38ee0ff604e0ccab799e1a812 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:10:10 -0700
Subject: [PATCH 03/15] Update action logic to verify hatch and run properly
---
.github/workflows/generate-llms.yaml | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index f9ccb3ec1bd..190c010e900 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -21,15 +21,20 @@ jobs:
- name: Install Hatch
run: pip install hatch
+ # Step 3.1: Verify Hatch
+ - name: List Hatch environments
+ run: hatch env show
+
# Step 4: Install dependencies for docs
- name: Install documentation dependencies
run: hatch env create llmspy
# Step 5: Run the script
- name: Generate llms.txt
- run: hatch run llmspy python weave/docs/scripts/generate-llmstxt.py
+ run: |
+ hatch shell llmspy -c "python weave/docs/scripts/generate-llmstxt.py"
- # Step 6: Verify generated file (optional)
+ # Step 6: Verify generated file
- name: Check if llms.txt exists
run: |
if [ ! -f weave/docs/static/llms.txt ]; then
From 683ec991067a05bede1e2a91d76bf26aa33ec677 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:11:51 -0700
Subject: [PATCH 04/15] Remove c option
---
.github/workflows/generate-llms.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index 190c010e900..6bdb7572d91 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -32,7 +32,7 @@ jobs:
# Step 5: Run the script
- name: Generate llms.txt
run: |
- hatch shell llmspy -c "python weave/docs/scripts/generate-llmstxt.py"
+ hatch shell llmspy "python weave/docs/scripts/generate-llmstxt.py"
# Step 6: Verify generated file
- name: Check if llms.txt exists
From 92e69ad01984c1aa6f33fbf7fbbdd05e45a42c8b Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:16:21 -0700
Subject: [PATCH 05/15] hatch shell is interactive, replace with run cmd
---
.github/workflows/generate-llms.yaml | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index 6bdb7572d91..3108f2cdae2 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -31,9 +31,8 @@ jobs:
# Step 5: Run the script
- name: Generate llms.txt
- run: |
- hatch shell llmspy "python weave/docs/scripts/generate-llmstxt.py"
-
+ run: hatch run llmspy python weave/docs/scripts/generate-llmstxt.py
+
# Step 6: Verify generated file
- name: Check if llms.txt exists
run: |
From 90ca85761f99338eceadc4692f1dff56aca772a0 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:22:57 -0700
Subject: [PATCH 06/15] Swap Hatch for venv bc I can't get it to work
---
.github/workflows/generate-llms.yaml | 24 ++++++++++--------------
pyproject.toml | 5 -----
2 files changed, 10 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index 3108f2cdae2..d5602be228d 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -17,22 +17,18 @@ jobs:
with:
python-version: '3.x' # Specify the required Python version
- # Step 3: Install Hatch
- - name: Install Hatch
- run: pip install hatch
-
- # Step 3.1: Verify Hatch
- - name: List Hatch environments
- run: hatch env show
-
- # Step 4: Install dependencies for docs
- - name: Install documentation dependencies
- run: hatch env create llmspy
+ - name: Create virtual environment
+ run: |
+ python -m venv .venv
+ source .venv/bin/activate
+ pip install --upgrade pip
+ pip install PyYAML # Add other dependencies here if needed
- # Step 5: Run the script
- name: Generate llms.txt
- run: hatch run llmspy python weave/docs/scripts/generate-llmstxt.py
-
+ run: |
+ source .venv/bin/activate
+ python weave/docs/scripts/generate-llmstxt.py
+
# Step 6: Verify generated file
- name: Check if llms.txt exists
run: |
diff --git a/pyproject.toml b/pyproject.toml
index 6c6caae8e01..9e9555c0ca5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,7 +52,6 @@ anthropic = ["anthropic>=0.18.0"]
cerebras = ["cerebras-cloud-sdk"]
cohere = ["cohere>=5.9.1,<5.9.3"]
dspy = ["dspy>=0.1.5", "litellm<=1.49.1"]
-llmspy = ["PyYAML>=6.0"]
google_ai_studio = ["google-generativeai>=0.8.3"]
groq = ["groq>=0.9.0"]
instructor = ["instructor>=1.4.3"]
@@ -135,10 +134,6 @@ exclude = [
"weave/clear_cache.py",
]
-[tool.hatch.envs.llmspy]
-dependencies = ["PyYAML>=6.0"]
-
-
[tool.pytest.ini_options]
filterwarnings = [
# treat warnings as errors
From a778b240f19be4dd2a5f17488d736fba907f43bf Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:28:39 -0700
Subject: [PATCH 07/15] adjust path to account for GH exec env
---
.github/workflows/generate-llms.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index d5602be228d..ffe5111e504 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -27,8 +27,8 @@ jobs:
- name: Generate llms.txt
run: |
source .venv/bin/activate
- python weave/docs/scripts/generate-llmstxt.py
-
+ python docs/scripts/generate-llmstxt.py
+
# Step 6: Verify generated file
- name: Check if llms.txt exists
run: |
From f6090687d0a008e7ea28ca66abf496fa49fd12f6 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:36:25 -0700
Subject: [PATCH 08/15] Use GH action path var
---
.github/workflows/generate-llms.yaml | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index ffe5111e504..d29934647d9 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -3,7 +3,7 @@ name: Generate llms.txt
on:
push:
branches:
- - pr-3045-gh-action # Adjust if your default branch has a different name
+ - pr-3045-gh-action
jobs:
generate-llms:
@@ -15,7 +15,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v4
with:
- python-version: '3.x' # Specify the required Python version
+ python-version: '3.x'
- name: Create virtual environment
run: |
@@ -27,9 +27,8 @@ jobs:
- name: Generate llms.txt
run: |
source .venv/bin/activate
- python docs/scripts/generate-llmstxt.py
+ python $GITHUB_WORKSPACE/docs/scripts/generate-llmstxt.py
- # Step 6: Verify generated file
- name: Check if llms.txt exists
run: |
if [ ! -f weave/docs/static/llms.txt ]; then
From a6803e5b0bcf4881679e1e78ca769d315a323e6e Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:43:15 -0700
Subject: [PATCH 09/15] Add debug step to figure out path issues
---
.github/workflows/generate-llms.yaml | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index d29934647d9..203e5481d36 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -11,6 +11,18 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v3
+
+ - name: Debug path issues
+ run: |
+ echo "Current directory: $(pwd)"
+ echo "GitHub workspace: $GITHUB_WORKSPACE"
+ if [ "$(pwd)" != "$GITHUB_WORKSPACE" ]; then
+ echo "Error: Current directory does not match \$GITHUB_WORKSPACE"
+ exit 1
+ fi
+
+ - name: List directory contents
+ run: ls -R
- name: Set up Python
uses: actions/setup-python@v4
From 0a6210da1ecaa5f86d66a4739841ecdc2c1474f8 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:53:48 -0700
Subject: [PATCH 10/15] Add more debugs
---
.github/workflows/generate-llms.yaml | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index 203e5481d36..3fa30e3cb3a 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -24,6 +24,23 @@ jobs:
- name: List directory contents
run: ls -R
+ - name: Debug script path
+ run: |
+ ls -l $GITHUB_WORKSPACE/docs/scripts/generate-llmstxt.py
+
+ - name: Debug Python version
+ run: |
+ source .venv/bin/activate
+ python --version
+
+ - name: Check active Python environment
+ run: |
+ source .venv/bin/activate
+ which python
+
+ - name: Fix script permissions
+ run: chmod +rx $GITHUB_WORKSPACE/docs/scripts/generate-llmstxt.py
+
- name: Set up Python
uses: actions/setup-python@v4
with:
From 222edf3fa6775633877a467ceff722e674cefe90 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:58:59 -0700
Subject: [PATCH 11/15] it was an underscore, not a hyphen
---
.github/workflows/generate-llms.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index 3fa30e3cb3a..ee5b696e897 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -56,7 +56,7 @@ jobs:
- name: Generate llms.txt
run: |
source .venv/bin/activate
- python $GITHUB_WORKSPACE/docs/scripts/generate-llmstxt.py
+ python $GITHUB_WORKSPACE/docs/scripts/generate_llmstxt.py
- name: Check if llms.txt exists
run: |
From 67814d3d5d015324c82d0c856552ef3b840d2746 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 16:00:11 -0700
Subject: [PATCH 12/15] remove debugs
---
.github/workflows/generate-llms.yaml | 29 ----------------------------
1 file changed, 29 deletions(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index ee5b696e897..8714779753d 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -11,35 +11,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v3
-
- - name: Debug path issues
- run: |
- echo "Current directory: $(pwd)"
- echo "GitHub workspace: $GITHUB_WORKSPACE"
- if [ "$(pwd)" != "$GITHUB_WORKSPACE" ]; then
- echo "Error: Current directory does not match \$GITHUB_WORKSPACE"
- exit 1
- fi
-
- - name: List directory contents
- run: ls -R
-
- - name: Debug script path
- run: |
- ls -l $GITHUB_WORKSPACE/docs/scripts/generate-llmstxt.py
-
- - name: Debug Python version
- run: |
- source .venv/bin/activate
- python --version
-
- - name: Check active Python environment
- run: |
- source .venv/bin/activate
- which python
-
- - name: Fix script permissions
- run: chmod +rx $GITHUB_WORKSPACE/docs/scripts/generate-llmstxt.py
- name: Set up Python
uses: actions/setup-python@v4
From 59ea4cd0ea726557bee58c2fcdca7b36beb71a02 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 17:16:35 -0700
Subject: [PATCH 13/15] Remove check step, update path
---
.github/workflows/generate-llms.yaml | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index 8714779753d..19673663202 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -29,17 +29,11 @@ jobs:
source .venv/bin/activate
python $GITHUB_WORKSPACE/docs/scripts/generate_llmstxt.py
- - name: Check if llms.txt exists
- run: |
- if [ ! -f weave/docs/static/llms.txt ]; then
- echo "Error: llms.txt was not generated."
- exit 1
- fi
- name: Commit and push changes
run: |
git config user.name "llms.py GitHub Actions Bot"
git config user.email "actions@github.com"
- git add weave/docs/static/llms.txt
+ git add docs/static/llms.txt
git diff-index --quiet HEAD || git commit -m "Auto-generate llms.txt"
git push origin test-llmspy-action
env:
From a8f2926c77e57beee57426df4b0167d9ff40dcc8 Mon Sep 17 00:00:00 2001
From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com>
Date: Thu, 21 Nov 2024 17:18:34 -0700
Subject: [PATCH 14/15] Correct branch
---
.github/workflows/generate-llms.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/generate-llms.yaml b/.github/workflows/generate-llms.yaml
index 19673663202..afebb4d8298 100644
--- a/.github/workflows/generate-llms.yaml
+++ b/.github/workflows/generate-llms.yaml
@@ -35,6 +35,6 @@ jobs:
git config user.email "actions@github.com"
git add docs/static/llms.txt
git diff-index --quiet HEAD || git commit -m "Auto-generate llms.txt"
- git push origin test-llmspy-action
+ git push origin pr-3045-gh-action
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
From d39df3433edddd6499ee2f2c0cbbdc2bee1a1d83 Mon Sep 17 00:00:00 2001
From: "llms.py GitHub Actions Bot"
Date: Fri, 22 Nov 2024 00:19:00 +0000
Subject: [PATCH 15/15] Auto-generate llms.txt
---
docs/static/llms.txt | 111 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 111 insertions(+)
create mode 100644 docs/static/llms.txt
diff --git a/docs/static/llms.txt b/docs/static/llms.txt
new file mode 100644
index 00000000000..cdbe9e5d9e6
--- /dev/null
+++ b/docs/static/llms.txt
@@ -0,0 +1,111 @@
+# Weave
+> Weave is a lightweight toolkit for tracking and evaluating LLM applications, built by Weights & Biases.
+
+This document contains links to all the documentation for Weave.
+
+## Docs
+
+- [introduction](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/introduction.md):
+- [quickstart](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/quickstart.md):
+- [tutorial-eval](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/tutorial-eval.md):
+- [tutorial-rag](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/tutorial-rag.md):
+- [tutorial-tracing_2](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/tutorial-tracing_2.md):
+- [tutorial-weave_models](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/tutorial-weave_models.md):
+- [costs](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/costs.md):
+- [datasets](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/datasets.md):
+- [deploy](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tools/deploy.md):
+- [evaluations](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/evaluations.md):
+- [feedback](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/feedback.md):
+- [media](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/media.md):
+- [models](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/models.md):
+- [objects](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/objects.md):
+- [ops](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/ops.md):
+- [platform](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/platform/index.md):
+- [prod_dashboard](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/cookbooks/prod_dashboard.md):
+- [prompts](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/core-types/prompts.md):
+- [scorers](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/evaluation/scorers.md):
+- [serve](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tools/serve.md):
+- [tools](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tools/index.md):
+- [tracing](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/tracing.mdx):
+- [tracking](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/tracking/index.md):
+
+## Optional
+
+- [01-intro_notebook](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/01-intro_notebook.md):
+- [CallSchema](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/interfaces/CallSchema.md):
+- [CallsFilter](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/interfaces/CallsFilter.md):
+- [Dataset](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/classes/Dataset.md):
+- [Evaluation](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/classes/Evaluation.md):
+- [Op](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/type-aliases/Op.md):
+- [README](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/README.md):
+- [WeaveClient](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/classes/WeaveClient.md):
+- [WeaveObject](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/classes/WeaveObject.md):
+- [anthropic](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/anthropic.md):
+- [audio_with_weave](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/audio_with_weave.md):
+- [call-end-call-end-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-end-call-end-post.api.mdx):
+- [call-read-call-read-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-read-call-read-post.api.mdx):
+- [call-start-batch-call-upsert-batch-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-start-batch-call-upsert-batch-post.api.mdx):
+- [call-start-call-start-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-start-call-start-post.api.mdx):
+- [call-update-call-update-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/call-update-call-update-post.api.mdx):
+- [calls-delete-calls-delete-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/calls-delete-calls-delete-post.api.mdx):
+- [calls-query-stats-calls-query-stats-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/calls-query-stats-calls-query-stats-post.api.mdx):
+- [calls-query-stream-calls-stream-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/calls-query-stream-calls-stream-query-post.api.mdx):
+- [cerebras](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/cerebras.md):
+- [chain_of_density](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/chain_of_density.md):
+- [codegen](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/codegen.md):
+- [cohere](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/cohere.md):
+- [cost-create-cost-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/cost-create-cost-create-post.api.mdx):
+- [cost-purge-cost-purge-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/cost-purge-cost-purge-post.api.mdx):
+- [cost-query-cost-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/cost-query-cost-query-post.api.mdx):
+- [custom_model_cost](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/custom_model_cost.md):
+- [dspy](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/dspy.md):
+- [dspy_prompt_optimization](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/dspy_prompt_optimization.md):
+- [fastapi.info](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/fastapi.info.mdx):
+- [feedback-create-feedback-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/feedback-create-feedback-create-post.api.mdx):
+- [feedback-purge-feedback-purge-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/feedback-purge-feedback-purge-post.api.mdx):
+- [feedback-query-feedback-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/feedback-query-feedback-query-post.api.mdx):
+- [feedback_prod](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/feedback_prod.md):
+- [file-content-file-content-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/file-content-file-content-post.api.mdx):
+- [file-create-file-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/file-create-file-create-post.api.mdx):
+- [google-gemini](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/google-gemini.md):
+- [groq](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/groq.md):
+- [import_from_csv](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/import_from_csv.md):
+- [init](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/init.md):
+- [instructor](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/instructor.md):
+- [langchain](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/langchain.md):
+- [leaderboard_quickstart](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/leaderboard_quickstart.md):
+- [litellm](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/litellm.md):
+- [llamaindex](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/llamaindex.md):
+- [local_models](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/local_models.md):
+- [login](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/login.md):
+- [mistral](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/mistral.md):
+- [multi-agent-structured-output](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/multi-agent-structured-output.md):
+- [notdiamond](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/notdiamond.md):
+- [obj-create-obj-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/obj-create-obj-create-post.api.mdx):
+- [obj-read-obj-read-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/obj-read-obj-read-post.api.mdx):
+- [objs-query-objs-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/objs-query-objs-query-post.api.mdx):
+- [online_monitoring](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/online_monitoring.md):
+- [op](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/op.md):
+- [openai](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/openai.md):
+- [openrouter](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/openrouter.md):
+- [pii](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/gen_notebooks/pii.md):
+- [read-root-health-get.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/read-root-health-get.api.mdx):
+- [refs-read-batch-refs-read-batch-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/refs-read-batch-refs-read-batch-post.api.mdx):
+- [requireCurrentCallStackEntry](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/requireCurrentCallStackEntry.md):
+- [requireCurrentChildSummary](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/requireCurrentChildSummary.md):
+- [server-info-server-info-get.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/server-info-server-info-get.api.mdx):
+- [table-create-table-create-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/table-create-table-create-post.api.mdx):
+- [table-query-stats-table-query-stats-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/table-query-stats-table-query-stats-post.api.mdx):
+- [table-query-table-query-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/table-query-table-query-post.api.mdx):
+- [table-update-table-update-post.api](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/service-api/table-update-table-update-post.api.mdx):
+- [together_ai](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/guides/integrations/together_ai.md):
+- [weave.trace.feedback](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace/weave.trace.feedback.md):
+- [weave.trace.op](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace/weave.trace.op.md):
+- [weave.trace.util](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace/weave.trace.util.md):
+- [weave.trace.weave_client](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace/weave.trace.weave_client.md):
+- [weave.trace_server.interface.query](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace_server/interface/weave.trace_server.interface.query.md):
+- [weave.trace_server.trace_server_interface](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace_server/weave.trace_server.trace_server_interface.md):
+- [weave.trace_server_bindings.remote_http_trace_server](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/python-sdk/weave/trace_server_bindings/weave.trace_server_bindings.remote_http_trace_server.md):
+- [weaveAudio](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/weaveAudio.md):
+- [weaveImage](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/weaveImage.md):
+- [wrapOpenAI](https://raw.githubusercontent.com/wandb/weave/master/docs/docs/reference/typescript-sdk/weave/functions/wrapOpenAI.md):