Merge branch 'master' into feat/send_message_google

.github/workflows/weave-node-tests.yaml

@@ -0,0 +1,32 @@
+name: Node.js Tests
+
+on:
+  push:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 9
+          run_install: false
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'pnpm'
+          cache-dependency-path: sdks/node/pnpm-lock.yaml
+      - name: Install dependencies
+        run: pnpm install
+        working-directory: sdks/node
+      - name: Run tests
+        run: pnpm test
+        working-directory: sdks/node
+        env:
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}

sdks/node/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+coverage

sdks/node/.prettierrc

@@ -0,0 +1,16 @@
+{
+  "trailingComma": "es5",
+  "singleQuote": true,
+  "bracketSpacing": false,
+  "bracketSameLine": true,
+  "tabWidth": 2,
+  "arrowParens": "avoid",
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx"],
+      "options": {
+        "parser": "typescript"
+      }
+    }
+  ]
+}

sdks/node/Makefile

@@ -0,0 +1,7 @@
+install-pnpm:
+	curl -fsSL https://get.pnpm.io/install.sh | sh -
+
+bootstrap: install-pnpm
+	pnpm env use --global 20
+	pnpm install
+	pnpm run generate-api

sdks/node/README-DEV.md

@@ -0,0 +1,23 @@
+# Working notes
+
+1. Setup your env
+
+   ```sh
+   make boostrap
+
+   pnpm install
+   pnpm link --global
+   pnpm link --global weave
+   ```
+
+2. Run tests
+
+   ```sh
+   pnpm test
+   ```
+
+3. Format
+
+   ```sh
+   pnpm format
+   ```

sdks/node/README.md

@@ -0,0 +1,202 @@
+# Weave (Alpha)
+
+Weave is a library for tracing and monitoring AI applications.
+
+This is an Alpha release, APIs are extremely subject to change.
+
+## Installation
+
+You can install Weave via npm:
+
+```bash
+npm install weave
+```
+
+Ensure you have a wandb API key in ~/.netrc.
+
+Like
+
+```
+machine api.wandb.ai
+  login user
+  password <wandb-api-key>
+```
+
+Get your wandb API key from [here](https://wandb.ai/authorize).
+
+## Quickstart
+
+Put this in a file called `predict.mjs`:
+
+```javascript
+import { OpenAI } from "openai";
+import { init, op, wrapOpenAI } from "weave";
+
+const openai = wrapOpenAI(new OpenAI());
+
+async function extractDinos(input) {
+  const response = await openai.chat.completions.create({
+    model: "gpt-4o",
+    messages: [
+      {
+        role: "user",
+        content: `In JSON format extract a list of 'dinosaurs', with their 'name', their 'common_name', and whether its 'diet' is a herbivore or carnivore: ${input}`,
+      },
+    ],
+  });
+  return response.choices[0].message.content;
+}
+const extractDinosOp = op(extractDinos);
+
+async function main() {
+  await init("weave-quickstart");
+  const result = await extractDinosOp(
+    "I watched as a Tyrannosaurus rex (T. rex) chased after a Triceratops (Trike), both carnivore and herbivore locked in an ancient dance. Meanwhile, a gentle giant Brachiosaurus (Brachi) calmly munched on treetops, blissfully unaware of the chaos below."
+  );
+  console.log(result);
+}
+
+main();
+```
+
+and then run
+
+```
+node predict.mjs
+```
+
+## Usage
+
+### Initializing a Project
+
+Before you can start tracing operations, you need to initialize a project. This sets up the necessary environment for trace collection.
+
+```javascript
+import { init } from "weave";
+
+// Initialize your project with a unique project name
+init("my-awesome-ai-project");
+```
+
+### Tracing Operations
+
+You can trace specific operations using the `op` function. This function wraps your existing functions and tracks their execution.
+
+```javascript
+import { op } from "weave";
+
+// Define a function you want to trace
+async function myFunction(arg1, arg2) {
+  // Your function logic
+  return arg1 + arg2;
+}
+
+// Wrap the function with op to enable tracing
+const tracedFunction = op(myFunction, "myFunction");
+
+// Call the traced function
+tracedFunction(5, 10);
+```
+
+### OpenAI Integration
+
+Weave provides an integration with OpenAI, allowing you to trace API calls made to OpenAI's services seamlessly.
+
+```javascript
+import { wrapOpenAI } from "weave/integrations/openai";
+
+// Create a patched instance of OpenAI
+const openai = wrapOpenAI();
+
+// Use the OpenAI instance as usual
+openai.chat.completions.create({
+  model: "text-davinci-003",
+  prompt: 'Translate the following English text to French: "Hello, world!"',
+  max_tokens: 60,
+});
+
+// Weave tracks images too!
+openai.images.generate({
+  prompt: "A cute baby sea otter",
+  n: 3,
+  size: "256x256",
+  response_format: "b64_json",
+});
+```
+
+### Evaluations
+
+```typescript
+import { init, op, Dataset, Evaluation } from "weave";
+
+async function main() {
+  await init("weavejsdev-eval6");
+  const ds = new Dataset({
+    id: "My Dataset",
+    description: "This is a dataset",
+    rows: [
+      { name: "Alice", age: 25 },
+      { name: "Bob", age: 30 },
+      { name: "Charlie", age: 34 },
+    ],
+  });
+  const evaluation = new Evaluation({
+    dataset: ds,
+    scorers: [
+      op(
+        (modelOutput: any, datasetItem: any) => modelOutput == datasetItem.age,
+        { name: "isEqual" }
+      ),
+    ],
+  });
+
+  const model = op(async function myModel(input) {
+    return input.age;
+  });
+
+  const results = await evaluation.evaluate({ model });
+  console.log(JSON.stringify(results, null, 2));
+}
+
+main();
+```
+
+## Configuration
+
+Weave reads API keys from the `.netrc` file located in your home directory. Ensure you have the required API keys configured for seamless integration and tracking.
+
+```
+machine api.wandb.ai
+  login user
+  password <wandb-api-key>
+```
+
+Get your wandb API key from [here](https://wandb.ai/authorize).
+
+## License
+
+This project is licensed under the Apaache2 License - see the [LICENSE](../LICENSE) file for details.
+
+### Roadmap / TODO
+
+- [x] Return token counts
+- [x] Summary merging
+- [x] Image support
+- [x] Decide how to handle args in js, since they're not named
+- [x] Make sure LLM streaming is handled
+- [x] Op versioning / code capture
+- [ ] Retry logic
+- [ ] Handle other wandb backends (managed / self-hosted)
+- [ ] Include system information in call attributes including package version.
+- [x] Objects / Datasets / Models / Evaluations
+- [ ] Ref tracking
+- [ ] More integrations
+
+## Known Issues
+
+- [ ] openai choices > 1 and "tools" not handled (function works though)
+- [ ] we always inject params.stream_options.include_usage in openai request, need to not send back usage chunk if user didn't ask for it.
+- [ ] handle all openai.images.generate formats, and images in inputs.
+- [ ] openai.images.generate token counts.
+- [ ] if a streaming op with streamReducer returns its iterator instead of iterating within the op, summary rollups won't work.
+- [ ] Decide what's in public interface (WeaveClient)

sdks/node/examples/classesWithOps.ts

@@ -0,0 +1,28 @@
+import OpenAI from 'openai';
+import * as weave from 'weave';
+
+class ExampleModel {
+  private oaiClient: OpenAI;
+
+  constructor() {
+    this.oaiClient = weave.wrapOpenAI(new OpenAI());
+    this.invoke = weave.op(this, this.invoke);
+  }
+
+  async invoke(input: string) {
+    const response = await this.oaiClient.chat.completions.create({
+      model: 'gpt-4o',
+      messages: [{role: 'user', content: input}],
+    });
+    return response.choices[0].message.content;
+  }
+}
+
+async function main() {
+  await weave.init('examples');
+
+  const model = new ExampleModel();
+  await model.invoke('Tell me a joke');
+}
+
+main();

sdks/node/examples/createDataset.ts

@@ -0,0 +1,53 @@
+import fs from 'fs';
+import 'source-map-support/register';
+import * as weave from 'weave';
+
+const sentences = [
+  'There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.',
+  'Pounits are a bright green color and are more savory than sweet.',
+  'Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them.',
+];
+const labels = [
+  {fruit: 'neoskizzles', color: 'purple', flavor: 'candy'},
+  {fruit: 'pounits', color: 'bright green', flavor: 'savory'},
+  {fruit: 'glowls', color: 'pale orange', flavor: 'sour and bitter'},
+];
+const logsPng = fs.readFileSync('logs.png');
+const audioClip = fs.readFileSync('CantinaBand3.wav');
+const examples = [
+  {
+    id: '0',
+    sentence: sentences[0],
+    target: labels[0],
+    image: weave.weaveImage({data: logsPng, imageType: 'png'}),
+    audio: weave.weaveAudio({data: audioClip, audioType: 'wav'}),
+  },
+  {
+    id: '1',
+    sentence: sentences[1],
+    target: labels[1],
+    image: weave.weaveImage({data: logsPng, imageType: 'png'}),
+    audio: weave.weaveAudio({data: audioClip, audioType: 'wav'}),
+  },
+  {
+    id: '2',
+    sentence: sentences[2],
+    target: labels[2],
+    image: weave.weaveImage({data: logsPng, imageType: 'png'}),
+    audio: weave.weaveAudio({data: audioClip, audioType: 'wav'}),
+  },
+];
+
+async function main() {
+  await weave.init('examples');
+  const ds = new weave.Dataset({
+    id: 'Fruit Dataset',
+    rows: examples,
+  });
+
+  ds.save();
+  const ref = await ds.__savedRef;
+  console.log(ref);
+}
+
+main();