From 98b06742a504a11fb5921726256519c8ef242c4b Mon Sep 17 00:00:00 2001 From: Darren Reid Date: Fri, 12 Jul 2024 15:15:01 +1000 Subject: [PATCH] Test out AiServer replacement. --- BlazorDiffusion.Tests/CreativeServiceTests.cs | 4 +- BlazorDiffusion/AiServer/dtos.cs | 92 +++++++++++++++++-- BlazorDiffusion/AiServerClient.cs | 46 +++++----- BlazorDiffusion/Configure.AppHost.cs | 9 +- 4 files changed, 116 insertions(+), 35 deletions(-) diff --git a/BlazorDiffusion.Tests/CreativeServiceTests.cs b/BlazorDiffusion.Tests/CreativeServiceTests.cs index ffe724c..d4d8617 100644 --- a/BlazorDiffusion.Tests/CreativeServiceTests.cs +++ b/BlazorDiffusion.Tests/CreativeServiceTests.cs @@ -71,8 +71,10 @@ public override void Configure(Container container) }); container.Register( - new AiServerClient("https://openai.servicestack.net/",new MemoryVirtualFiles()) + new AiServerClient { + Client = new JsonApiClient("https://openai.servicestack.net/"), + VirtualFiles = new MemoryVirtualFiles(), OutputPathPrefix = Path.Join(ContentRootDirectory.RealPath.CombineWith("App_Files"),"artifacts") }); container.AddSingleton(); diff --git a/BlazorDiffusion/AiServer/dtos.cs b/BlazorDiffusion/AiServer/dtos.cs index 7fb890a..8f6027b 100644 --- a/BlazorDiffusion/AiServer/dtos.cs +++ b/BlazorDiffusion/AiServer/dtos.cs @@ -1,5 +1,5 @@ /* Options: -Date: 2024-07-10 07:46:36 +Date: 2024-07-12 04:09:19 Version: 8.31 Tip: To override a DTO option, remove "//" prefix before updating BaseUrl: https://openai.servicestack.net @@ -31,6 +31,7 @@ using System.Runtime.Serialization; using ServiceStack; using ServiceStack.DataAnnotations; +using System.IO; using AiServer.ServiceModel.Types; using AiServer.ServiceModel; @@ -1058,7 +1059,14 @@ public partial class ComfyImageToImage public partial class ComfyImageToImageResponse { - public virtual string FilePath { get; set; } + public ComfyImageToImageResponse() + { + Images = new List{}; + } + + public virtual string PromptId { get; set; } + public virtual ComfyWorkflowRequest Request { get; set; } + public virtual List Images { get; set; } } public partial class ComfyImageToImageUpscale @@ -1071,7 +1079,14 @@ public partial class ComfyImageToImageUpscale public partial class ComfyImageToImageUpscaleResponse { - public virtual string FilePath { get; set; } + public ComfyImageToImageUpscaleResponse() + { + Images = new List{}; + } + + public virtual string PromptId { get; set; } + public virtual ComfyWorkflowRequest Request { get; set; } + public virtual List Images { get; set; } } public partial class ComfyImageToImageWithMask @@ -1094,7 +1109,14 @@ public partial class ComfyImageToImageWithMask public partial class ComfyImageToImageWithMaskResponse { - public virtual string FilePath { get; set; } + public ComfyImageToImageWithMaskResponse() + { + Images = new List{}; + } + + public virtual string PromptId { get; set; } + public virtual ComfyWorkflowRequest Request { get; set; } + public virtual List Images { get; set; } } public partial class ComfyImageToText @@ -1105,7 +1127,9 @@ public partial class ComfyImageToText public partial class ComfyImageToTextResponse { - public virtual string Text { get; set; } + public virtual string PromptId { get; set; } + public virtual ComfyWorkflowRequest Request { get; set; } + public virtual ComfyTextOutput TextOutput { get; set; } } public enum ComfyMaskSource @@ -1163,7 +1187,9 @@ public partial class ComfySpeechToText public partial class ComfySpeechToTextResponse { - public virtual string Text { get; set; } + public virtual string PromptId { get; set; } + public virtual ComfyWorkflowRequest Request { get; set; } + public virtual ComfyTextOutput TextOutput { get; set; } } public enum ComfyTaskType @@ -1200,7 +1226,14 @@ public partial class ComfyTextToAudio public partial class ComfyTextToAudioResponse { - public virtual string FilePath { get; set; } + public ComfyTextToAudioResponse() + { + Sounds = new List{}; + } + + public virtual string PromptId { get; set; } + public virtual ComfyWorkflowRequest Request { get; set; } + public virtual List Sounds { get; set; } } public partial class ComfyTextToImage @@ -1226,6 +1259,8 @@ public ComfyTextToImageResponse() Images = new List{}; } + public virtual string PromptId { get; set; } + public virtual ComfyWorkflowRequest Request { get; set; } public virtual List Images { get; set; } } @@ -1238,7 +1273,47 @@ public partial class ComfyTextToSpeech public partial class ComfyTextToSpeechResponse { - public virtual string FilePath { get; set; } + public ComfyTextToSpeechResponse() + { + Speech = new List{}; + } + + public virtual string PromptId { get; set; } + public virtual ComfyWorkflowRequest Request { get; set; } + public virtual List Speech { get; set; } + } + + public partial class ComfyWorkflowRequest + { + public virtual long Id { get; set; } + public virtual string Model { get; set; } + public virtual int? Steps { get; set; } + public virtual int BatchSize { get; set; } + public virtual int? Seed { get; set; } + public virtual string PositivePrompt { get; set; } + public virtual string NegativePrompt { get; set; } + public virtual ComfyFileInput Image { get; set; } + public virtual ComfyFileInput Speech { get; set; } + public virtual ComfyFileInput Mask { get; set; } + public virtual Stream ImageInput { get; set; } + public virtual Stream SpeechInput { get; set; } + public virtual Stream MaskInput { get; set; } + public virtual ComfySampler? Sampler { get; set; } + public virtual ArtStyle? ArtStyle { get; set; } + public virtual string Scheduler { get; set; } + public virtual double? CfgScale { get; set; } + public virtual double? Denoise { get; set; } + public virtual string UpscaleModel { get; set; } + public virtual int? Width { get; set; } + public virtual int? Height { get; set; } + public virtual ComfyTaskType TaskType { get; set; } + public virtual string RefId { get; set; } + public virtual string Provider { get; set; } + public virtual string ReplyTo { get; set; } + public virtual string Tag { get; set; } + public virtual string Clip { get; set; } + public virtual double? SampleLength { get; set; } + public virtual ComfyMaskSource MaskChannel { get; set; } } public partial class ComfyWorkflowResponse @@ -1340,6 +1415,7 @@ public QueueComfyWorkflowResponse() TextOutputs = new List{}; } + public virtual ComfyWorkflowRequest Request { get; set; } public virtual ComfyWorkflowStatus Status { get; set; } public virtual ComfyWorkflowResponse WorkflowResponse { get; set; } public virtual string PromptId { get; set; } diff --git a/BlazorDiffusion/AiServerClient.cs b/BlazorDiffusion/AiServerClient.cs index 41d63f3..c92abc3 100644 --- a/BlazorDiffusion/AiServerClient.cs +++ b/BlazorDiffusion/AiServerClient.cs @@ -2,36 +2,31 @@ using BlazorDiffusion.ServiceInterface; using BlazorDiffusion.ServiceModel; using ServiceStack.IO; +using ServiceStack.Text; using JsonApiClient = ServiceStack.JsonApiClient; namespace BlazorDiffusion; public class AiServerClient: IStableDiffusionClient { - public JsonApiClient Client { get; } - public IVirtualFiles VirtualFiles { get; } + public JsonApiClient Client { get; set; } + public IVirtualFiles VirtualFiles { get; set; } public string? OutputPathPrefix { get; set; } - - public AiServerClient(string aiServerBaseUrl,IVirtualFiles virtualFiles) - { - Client = new JsonApiClient(aiServerBaseUrl); - VirtualFiles = virtualFiles; - } + public async Task GenerateImageAsync(ImageGeneration request) { var req = request.ToComfy(); var res = await Client.PostAsync(req); var now = DateTime.UtcNow; var key = $"{now:yyyy/MM/dd}/{(long)now.TimeOfDay.TotalMilliseconds}"; - - var promptId = Guid.NewGuid().ToString(); + var results = new List(); + var seed = (res.Request.Seed ?? 0).ConvertTo(); foreach (var item in res.Images) { var artifactUrl = $"{Client.BaseUri.TrimEnd('/')}/uploads{item.Url}"; - var seed = (uint)Random.Shared.Next(); - var output = Path.Join(OutputPathPrefix, key, $"output_{Guid.NewGuid()}.png"); + var output = Path.Join(OutputPathPrefix, key, $"output_{seed}.png"); var bytes = await artifactUrl.GetBytesFromUrlAsync(); await VirtualFiles.WriteFileAsync(output, bytes); var imageDetails = ImageDetails.Calculate(bytes); @@ -40,7 +35,7 @@ public async Task GenerateImageAsync(ImageGeneration re { Prompt = request.Prompt, Seed = seed, - AnswerId = promptId, + AnswerId = res.PromptId, FilePath = $"/artifacts/{key}/output_{seed}.png", FileName = $"output_{seed}.png", ContentLength = bytes.Length, @@ -48,30 +43,36 @@ public async Task GenerateImageAsync(ImageGeneration re Height = request.Height, ImageDetails = imageDetails, }); + // Assume incremental seeds for multiple images as comfyui does not provide the specific image seed back + seed++; } return new ImageGenerationResponse { - RequestId = Guid.NewGuid().ToString(), + RequestId = res.PromptId, EngineId = "comfy", Key = key, Results = results, }; } - public IVirtualFile? GetMetadataFile(Creative creative) - { - throw new NotImplementedException(); - } + public string GetMetadataPath(Creative creative) => OutputPathPrefix.CombineWith(creative.Key, "metadata.json"); + public IVirtualFile GetMetadataFile(Creative creative) => VirtualFiles.GetFile(GetMetadataPath(creative)); - public Task SaveMetadataAsync(Creative entry) + public async Task SaveMetadataAsync(Creative creative) { - throw new NotImplementedException(); + var vfsPathSuffix = creative.Key; + var outputDir = Path.Join(OutputPathPrefix, vfsPathSuffix); + await VirtualFiles.WriteFileAsync(Path.Join(outputDir, "metadata.json"), creative.ToJson().IndentJson()); } - public Task DeleteFolderAsync(Creative entry) + public Task DeleteFolderAsync(Creative creative) { - throw new NotImplementedException(); + var vfsPathSuffix = creative.Key; + var directory = VirtualFiles.GetDirectory(Path.Join(OutputPathPrefix, vfsPathSuffix)); + var allFiles = directory.GetAllFiles(); + VirtualFiles.DeleteFiles(allFiles); + return Task.CompletedTask; } } @@ -84,7 +85,6 @@ public static ComfyTextToImage ToComfy(this ImageGeneration request) Height = request.Height, Width = request.Width, Seed = request.Seed ?? Random.Shared.Next(), - Steps = request.Steps, BatchSize = request.Images, PositivePrompt = request.Prompt, }; diff --git a/BlazorDiffusion/Configure.AppHost.cs b/BlazorDiffusion/Configure.AppHost.cs index 32cc793..c0db694 100644 --- a/BlazorDiffusion/Configure.AppHost.cs +++ b/BlazorDiffusion/Configure.AppHost.cs @@ -98,13 +98,16 @@ public override void Configure(Container container) transformFile:ImageUtils.TransformAvatarAsync) )); + var aiServerClient = new JsonApiClient("https://openai.servicestack.net/"); + if(!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AI_SERVER_APIKEY"))) + aiServerClient.BearerToken = Environment.GetEnvironmentVariable("AI_SERVER_APIKEY"); // Don't use public prefix if working locally - Register(new DreamStudioClient + Register(new AiServerClient { - ApiKey = Environment.GetEnvironmentVariable("DREAMAI_APIKEY") ?? "", + Client = aiServerClient, OutputPathPrefix = "artifacts", - PublicPrefix = appConfig.AssetsBasePath, VirtualFiles = appFs + //PublicPrefix = appConfig.AssetsBasePath, }); ScriptContext.Args[nameof(AppData)] = AppData.Instance;