diff --git a/Claudia.sln b/Claudia.sln index 37145e6..446fb2e 100644 --- a/Claudia.sln +++ b/Claudia.sln @@ -19,7 +19,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BlazorApp1", "sandbox\Blazo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Claudia.FunctionGenerator", "src\Claudia.FunctionGenerator\Claudia.FunctionGenerator.csproj", "{8C464111-AD67-4D2B-9AE2-0B52AB077EBD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Claudia.FunctionGenerator.Tests", "tests\Claudia.FunctionGenerator.Tests\Claudia.FunctionGenerator.Tests.csproj", "{89A58A08-F553-4CEA-A2A8-783009501E05}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Claudia.FunctionGenerator.Tests", "tests\Claudia.FunctionGenerator.Tests\Claudia.FunctionGenerator.Tests.csproj", "{89A58A08-F553-4CEA-A2A8-783009501E05}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BedrockConsoleApp", "sandbox\BedrockConsoleApp\BedrockConsoleApp.csproj", "{79C84272-E0AB-4918-9454-B0AEA9CBE40A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Claudia.Bedrock", "src\Claudia.Bedrock\Claudia.Bedrock.csproj", "{9EC270A6-6E6F-44CF-8A4C-975A2A7344AA}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -51,6 +55,14 @@ Global {89A58A08-F553-4CEA-A2A8-783009501E05}.Debug|Any CPU.Build.0 = Debug|Any CPU {89A58A08-F553-4CEA-A2A8-783009501E05}.Release|Any CPU.ActiveCfg = Release|Any CPU {89A58A08-F553-4CEA-A2A8-783009501E05}.Release|Any CPU.Build.0 = Release|Any CPU + {79C84272-E0AB-4918-9454-B0AEA9CBE40A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {79C84272-E0AB-4918-9454-B0AEA9CBE40A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {79C84272-E0AB-4918-9454-B0AEA9CBE40A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {79C84272-E0AB-4918-9454-B0AEA9CBE40A}.Release|Any CPU.Build.0 = Release|Any CPU + {9EC270A6-6E6F-44CF-8A4C-975A2A7344AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9EC270A6-6E6F-44CF-8A4C-975A2A7344AA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9EC270A6-6E6F-44CF-8A4C-975A2A7344AA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9EC270A6-6E6F-44CF-8A4C-975A2A7344AA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -62,6 +74,8 @@ Global {8EEB0F69-132B-4887-959D-25531588FCD2} = {E61BFC87-2B96-4699-9B69-EE4B008AE0A0} {8C464111-AD67-4D2B-9AE2-0B52AB077EBD} = {B54A8855-F8F0-4015-80AA-86974E65AC2D} {89A58A08-F553-4CEA-A2A8-783009501E05} = {1B4BD6F6-8528-4409-BA55-085DA5486D36} + {79C84272-E0AB-4918-9454-B0AEA9CBE40A} = {E61BFC87-2B96-4699-9B69-EE4B008AE0A0} + {9EC270A6-6E6F-44CF-8A4C-975A2A7344AA} = {B54A8855-F8F0-4015-80AA-86974E65AC2D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {B7CEBA02-BB0C-4102-AE58-DFD114C3192A} diff --git a/README.md b/README.md index 171524c..7dc1bb3 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ This library is distributed via NuGet, supporting .NET Standard 2.1, .NET 6(.NET It can also be used with Unity Game Engine both Runtime and Editor. For instructions on how to use it, please refer to the [Unity section](#unity). +You can also use it with AWS Bedrock. Check the [AWS Bedrock section](#aws-bedrock) for more details. + Usage --- For details about the API, please check the [official API reference](https://docs.anthropic.com/claude/reference/getting-started-with-the-api). @@ -829,6 +831,67 @@ public partial class Home If you need to store the chat message history, you can serialize `List chatMessages` to JSON and save it to a file or database. +AWS Bedrock +--- +We provide support for the [Anthropic Bedrock API](https://aws.amazon.com/bedrock/claude/) through a separate package. + +> PM> Install-Package [Claudia.Bedrock](https://www.nuget.org/packages/Claudia.Bedrock) + +To create an `AmazonBedrockRuntimeClient` from the AWS SDK and specify the Bedrock Model ID using `UseAnthropic`, set the Model property of RequestMessage to `anthropic_version`. The rest is the same as a regular Anthropic Client. + +```csharp +// credentials is your own +AWSConfigs.AWSProfileName = ""; + +var bedrock = new AmazonBedrockRuntimeClient(RegionEndpoint.USEast1); +var anthropic = bedrock.UseAnthropic("anthropic.claude-3-haiku-20240307-v1:0"); // Model Id + +var response = await anthropic.Messages.CreateAsync(new() +{ + Model = "bedrock-2023-05-31", // anthropic_version + MaxTokens = 1024, + Messages = [new() { Role = "user", Content = "Hello, Claude" }] +}); + +Console.WriteLine(response); +``` + +Streaming Messages work in the same way. + +```csharp +var stream = anthropic.Messages.CreateStreamAsync(new() +{ + Model = "bedrock-2023-05-31", // anthropic_version + MaxTokens = 1024, + Messages = [new() { Role = "user", Content = "Hello, Claude" }] +}); + +await foreach (var item in stream) +{ + Console.WriteLine(item); +} +``` + +If you need the raw response, call `InvokeModelAsync` or `InvokeModelWithResponseStreamAsync` instead. This allows you to check the status code and headers before retrieving the result with `GetMessageResponse` or `GetMessageResponseAsync`. + +```csharp +var bedrock = new AmazonBedrockRuntimeClient(RegionEndpoint.USEast1); + +// (string modelId, MessageRequest request) +var response = await bedrock.InvokeModelAsync("anthropic.claude-3-haiku-20240307-v1:0", new() +{ + Model = "bedrock-2023-05-31", // anthropic_version + MaxTokens = 1024, + Messages = [new() { Role = "user", Content = "Hello, Claude" }] +}); + +Console.WriteLine(response.ResponseMetadata.RequestId); + +var responseMessage = response.GetMessageResponse(); + +Console.WriteLine(responseMessage); +``` + Unity --- Minimum supported Unity version is `2022.3.12f1`. You need to install from NuGet. We recommend using [NuGetForUnity](https://github.com/GlitchEnzo/NuGetForUnity). diff --git a/sandbox/BedrockConsoleApp/BedrockConsoleApp.csproj b/sandbox/BedrockConsoleApp/BedrockConsoleApp.csproj new file mode 100644 index 0000000..1f78f24 --- /dev/null +++ b/sandbox/BedrockConsoleApp/BedrockConsoleApp.csproj @@ -0,0 +1,20 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + + diff --git a/sandbox/BedrockConsoleApp/Program.cs b/sandbox/BedrockConsoleApp/Program.cs new file mode 100644 index 0000000..f6f9b29 --- /dev/null +++ b/sandbox/BedrockConsoleApp/Program.cs @@ -0,0 +1,63 @@ +using Amazon; +using Amazon.BedrockRuntime; +using Amazon.BedrockRuntime.Model; +using Amazon.BedrockRuntime.Model.Internal.MarshallTransformations; +using Amazon.Runtime.EventStreams.Internal; +using Claudia; +using System.Buffers; +using System.Collections.Generic; +using System.Formats.Asn1; +using System.IO.Pipelines; +using System.Reflection.PortableExecutable; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices.ObjectiveC; +using System.Text; +using System.Text.Json; +using System.Threading.Channels; +using ThirdParty.Json.LitJson; + + +// credentials is your own +AWSConfigs.AWSProfileName = ""; + +//var bedrock = new AmazonBedrockRuntimeClient(RegionEndpoint.USEast1); +//var anthropic = bedrock.UseAnthropic("anthropic.claude-3-haiku-20240307-v1:0"); + +//var response = await anthropic.Messages.CreateAsync(new() +//{ +// Model = "bedrock-2023-05-31", +// MaxTokens = 1024, +// Messages = [new() { Role = "user", Content = "Hello, Claude" }] +//}); + +//Console.WriteLine(response); + + +//var stream = anthropic.Messages.CreateStreamAsync(new() +//{ +// Model = "bedrock-2023-05-31", +// MaxTokens = 1024, +// Messages = [new() { Role = "user", Content = "Hello, Claude" }] +//}); + +//await foreach (var item in stream) +//{ +// Console.WriteLine(item); +//} + + +var bedrock = new AmazonBedrockRuntimeClient(RegionEndpoint.USEast1); + +// (string modelId, MessageRequest request) +var response = await bedrock.InvokeModelAsync("anthropic.claude-3-haiku-20240307-v1:0", new() +{ + Model = "bedrock-2023-05-31", // anthropic_version + MaxTokens = 1024, + Messages = [new() { Role = "user", Content = "Hello, Claude" }] +}); + +Console.WriteLine(response.ResponseMetadata.RequestId); + +var responseMessage = response.GetMessageResponse(); + +Console.WriteLine(responseMessage); \ No newline at end of file diff --git a/src/Claudia.Bedrock/BedrockAnthropicJsonSerialzierContext.cs b/src/Claudia.Bedrock/BedrockAnthropicJsonSerialzierContext.cs new file mode 100644 index 0000000..36fa35e --- /dev/null +++ b/src/Claudia.Bedrock/BedrockAnthropicJsonSerialzierContext.cs @@ -0,0 +1,110 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Claudia; + +internal static class BedrockAnthropicJsonSerialzierContext +{ + public static JsonSerializerOptions Options { get; } + + static BedrockAnthropicJsonSerialzierContext() + { + var options = new JsonSerializerOptions(InternalBedrockAnthropicJsonSerialzierContext.Default.Options); + options.TypeInfoResolverChain.Add(AnthropicJsonSerialzierContext.Default.Options.TypeInfoResolver!); + options.MakeReadOnly(); + + Options = options; + } +} + +[JsonSourceGenerationOptions( + GenerationMode = JsonSourceGenerationMode.Default, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false)] +[JsonSerializable(typeof(BedrockMessageRequest))] +internal partial class InternalBedrockAnthropicJsonSerialzierContext : JsonSerializerContext +{ +} + +// "model" -> "anthropic_version" +internal record class BedrockMessageRequest +{ + /// + /// The model that will complete your prompt. + /// + // [JsonPropertyName("model")] + [JsonPropertyName("anthropic_version")] + public required string Model { get; set; } + + /// + /// The maximum number of tokens to generate before stopping. + /// Note that our models may stop before reaching this maximum.This parameter only specifies the absolute maximum number of tokens to generate. + /// Different models have different maximum values for this parameter + /// + [JsonPropertyName("max_tokens")] + public required int MaxTokens { get; set; } + + /// + /// Input messages. + /// + [JsonPropertyName("messages")] + public required Message[] Messages { get; set; } + + // optional parameters + + /// + /// System prompt. + /// A system prompt is a way of providing context and instructions to Claude, such as specifying a particular goal or role. + /// + [JsonPropertyName("system")] + public string? System { get; set; } + + /// + /// An object describing metadata about the request. + /// + [JsonPropertyName("metadata")] + public Metadata? Metadata { get; set; } + + /// + /// Custom text sequences that will cause the model to stop generating. + /// Our models will normally stop when they have naturally completed their turn, which will result in a response stop_reason of "end_turn". + /// If you want the model to stop generating when it encounters custom strings of text, you can use the stop_sequences parameter.If the model encounters one of the custom sequences, the response stop_reason value will be "stop_sequence" and the response stop_sequence value will contain the matched stop sequence. + /// + [JsonPropertyName("stop_sequences")] + public string[]? StopSequences { get; set; } + + /// + /// Whether to incrementally stream the response using server-sent events. + /// + [JsonPropertyName("stream")] + [JsonInclude] // internal so requires Include. + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + internal bool? Stream { get; set; } + + /// + /// Amount of randomness injected into the response. + /// Defaults to 1.0. Ranges from 0.0 to 1.0. Use temperature closer to 0.0 for analytical / multiple choice, and closer to 1.0 for creative and generative tasks. + /// Note that even with temperature of 0.0, the results will not be fully deterministic. + /// + [JsonPropertyName("temperature")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public double? Temperature { get; set; } + + /// + /// Use nucleus sampling. + /// In nucleus sampling, we compute the cumulative distribution over all the options for each subsequent token in decreasing probability order and cut it off once it reaches a particular probability specified by top_p.You should either alter temperature or top_p, but not both. + /// Recommended for advanced use cases only. You usually only need to use temperature. + /// + [JsonPropertyName("top_p")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public double? TopP { get; set; } + + /// + /// Only sample from the top K options for each subsequent token. + /// Used to remove "long tail" low probability responses. + /// Recommended for advanced use cases only. You usually only need to use temperature. + /// + [JsonPropertyName("top_k")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public double? TopK { get; set; } +} \ No newline at end of file diff --git a/src/Claudia.Bedrock/BedrockExtensions.cs b/src/Claudia.Bedrock/BedrockExtensions.cs new file mode 100644 index 0000000..ca4a7df --- /dev/null +++ b/src/Claudia.Bedrock/BedrockExtensions.cs @@ -0,0 +1,107 @@ +using Amazon.BedrockRuntime; +using Amazon.BedrockRuntime.Model; +using System.Runtime.CompilerServices; +using System.Text.Json; + +namespace Claudia; + +public class BedrockAnthropicClient(AmazonBedrockRuntimeClient client, string modelId) : IMessages +{ + public IMessages Messages => this; + + // currently overrideOptions is not yet supported. + async Task IMessages.CreateAsync(MessageRequest request, RequestOptions? overrideOptions, CancellationToken cancellationToken) + { + var response = await client.InvokeModelAsync(modelId, request, cancellationToken); + return response.GetMessageResponse(); + } + + async IAsyncEnumerable IMessages.CreateStreamAsync(MessageRequest request, RequestOptions? overrideOptions, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var response = await client.InvokeModelWithResponseStreamAsync(modelId, request, cancellationToken); + await foreach (var item in response.GetMessageResponseAsync(cancellationToken)) + { + yield return item; + } + } +} + +public static class BedrockExtensions +{ + public static BedrockAnthropicClient UseAnthropic(this AmazonBedrockRuntimeClient client, string modelId) + { + return new BedrockAnthropicClient(client, modelId); + } + + public static Task InvokeModelAsync(this AmazonBedrockRuntimeClient client, string modelId, MessageRequest request, CancellationToken cancellationToken = default) + { + return client.InvokeModelAsync(new Amazon.BedrockRuntime.Model.InvokeModelRequest + { + ModelId = modelId, + Accept = "application/json", + ContentType = "application/json", + Body = Serialize(request), + }, cancellationToken); + } + + public static Task InvokeModelWithResponseStreamAsync(this AmazonBedrockRuntimeClient client, string modelId, MessageRequest request, CancellationToken cancellationToken = default) + { + return client.InvokeModelWithResponseStreamAsync(new Amazon.BedrockRuntime.Model.InvokeModelWithResponseStreamRequest + { + ModelId = modelId, + Accept = "application/json", + ContentType = "application/json", + Body = Serialize(request), + }, cancellationToken); + } + + public static MessageResponse GetMessageResponse(this InvokeModelResponse response) + { + if ((int)response.HttpStatusCode == 200) + { + return JsonSerializer.Deserialize(response.Body, AnthropicJsonSerialzierContext.Default.Options)!; + } + else + { + var shape = JsonSerializer.Deserialize(response.Body, AnthropicJsonSerialzierContext.Default.Options)!; + + var error = shape!.ErrorResponse; + var errorMsg = error.Message; + var code = (ErrorCode)response.HttpStatusCode; + throw new ClaudiaException(code, error.Type, errorMsg); + } + } + + public static IAsyncEnumerable GetMessageResponseAsync(this InvokeModelWithResponseStreamResponse response, CancellationToken cancellationToken = default) + { + return ResponseStreamReader.ToAsyncEnumerable(response.Body, cancellationToken); + } + + static MemoryStream Serialize(MessageRequest request) + { + var ms = new MemoryStream(); + var model = ConvertToBedrockModel(request); + + JsonSerializer.Serialize(ms, model, BedrockAnthropicJsonSerialzierContext.Options); + + ms.Flush(); + ms.Position = 0; + return ms; + } + + static BedrockMessageRequest ConvertToBedrockModel(MessageRequest request) + { + return new BedrockMessageRequest + { + Model = request.Model, + MaxTokens = request.MaxTokens, + Messages = request.Messages, + Metadata = request.Metadata, + StopSequences = request.StopSequences, + System = request.System, + Temperature = request.Temperature, + TopK = request.TopK, + TopP = request.TopP, + }; + } +} diff --git a/src/Claudia.Bedrock/Claudia.Bedrock.csproj b/src/Claudia.Bedrock/Claudia.Bedrock.csproj new file mode 100644 index 0000000..c396dea --- /dev/null +++ b/src/Claudia.Bedrock/Claudia.Bedrock.csproj @@ -0,0 +1,43 @@ + + + + netstandard2.1;net6.0;net8.0 + enable + enable + 12 + Claudia + true + 1701;1702;1591;1573 + + + ai; + AWS Bedrock support for Claudia. + true + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + diff --git a/src/Claudia.Bedrock/ResponseStreamReader.cs b/src/Claudia.Bedrock/ResponseStreamReader.cs new file mode 100644 index 0000000..9f15548 --- /dev/null +++ b/src/Claudia.Bedrock/ResponseStreamReader.cs @@ -0,0 +1,98 @@ +using Amazon.BedrockRuntime.Model; +using System.Text.Json; +using System.Threading.Channels; + +namespace Claudia; + +internal static class ResponseStreamReader +{ + public static IAsyncEnumerable ToAsyncEnumerable(ResponseStream stream, CancellationToken cancellationToken = default) + { + var channel = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true, AllowSynchronousContinuations = true }); + + stream.EventReceived += (_, e) => + { + if (e.EventStreamEvent is PayloadPart p) + { + var ms = p.Bytes; + IMessageStreamEvent? response = null; + + ms.Position = 9; + var c = (char)ms.ReadByte(); + ms.Position = 0; + + if (c == 'c') // content_block_start/delta/stop + { + ms.Position = 25; + switch (ms.ReadByte()) + { + case (byte)'a': // st[a]rt + ms.Position = 0; + response = JsonSerializer.Deserialize(ms, AnthropicJsonSerialzierContext.Default.Options)!; + break; + case (byte)'o': // st[o]p + ms.Position = 0; + response = JsonSerializer.Deserialize(ms, AnthropicJsonSerialzierContext.Default.Options)!; + break; + case (byte)'l': // de[l]ta + ms.Position = 0; + response = JsonSerializer.Deserialize(ms, AnthropicJsonSerialzierContext.Default.Options)!; + break; + default: + break; + } + ms.Position = 0; + } + else if (c == 'm') // message_start/delta/stop + { + ms.Position = 19; + switch (ms.ReadByte()) + { + case (byte)'a': // st[a]rt + ms.Position = 0; + response = JsonSerializer.Deserialize(ms, AnthropicJsonSerialzierContext.Default.Options)!; + break; + case (byte)'o': // st[o]p + ms.Position = 0; + response = JsonSerializer.Deserialize(ms, AnthropicJsonSerialzierContext.Default.Options)!; + break; + case (byte)'l': // de[l]ta + ms.Position = 0; + response = JsonSerializer.Deserialize(ms, AnthropicJsonSerialzierContext.Default.Options)!; + break; + default: + break; + } + } + else if (c == 'p') // ping + { + response = JsonSerializer.Deserialize(ms, AnthropicJsonSerialzierContext.Default.Options)!; + } + else if (c == 'e') // error + { + var error = JsonSerializer.Deserialize(ms, AnthropicJsonSerialzierContext.Default.Options); + var err = new ClaudiaException(error!.ErrorResponse.ToErrorCode(), error.ErrorResponse.Type, error.ErrorResponse.Message); + channel.Writer.Complete(err); + } + + if (response != null) + { + channel.Writer.TryWrite(response); + } + + if (response is MessageStop) + { + channel.Writer.TryComplete(); + } + } + }; + + stream.ExceptionReceived += (_, e) => + { + channel.Writer.Complete(e.EventStreamException); + }; + + stream.StartProcessing(); + return channel.Reader.ReadAllAsync(cancellationToken); + } +}