From 54def3f5a60436b15e6cf68d187d8c769c810382 Mon Sep 17 00:00:00 2001 From: pk5ls20 Date: Wed, 30 Oct 2024 10:56:27 +0800 Subject: [PATCH] [All] ImageOcr (#662) --- .../Common/Entity/ImageOcrResultEntry.cs | 49 +++++++++++++ .../Common/Interface/Api/OperationExt.cs | 9 +++ .../Logic/Implementation/OperationLogic.cs | 24 +++++++ .../Internal/Event/Action/ImageOcrEvent.cs | 25 +++++++ .../Oidb/Request/OidbSvcTrpcTcp.0xE07_0.cs | 41 +++++++++++ .../Response/OidbSvcTrpcTcp0xE07_0Response.cs | 72 +++++++++++++++++++ .../Service/Action/ImageOcrService.cs | 57 +++++++++++++++ .../Core/Entity/Action/OneBotOcrImage.cs | 9 +++ .../Operation/Ability/UploadImageOperation.cs | 13 +--- .../Operation/Generic/OcrImageOperation.cs | 27 +++++++ 10 files changed, 316 insertions(+), 10 deletions(-) create mode 100644 Lagrange.Core/Common/Entity/ImageOcrResultEntry.cs create mode 100644 Lagrange.Core/Internal/Event/Action/ImageOcrEvent.cs create mode 100644 Lagrange.Core/Internal/Packets/Service/Oidb/Request/OidbSvcTrpcTcp.0xE07_0.cs create mode 100644 Lagrange.Core/Internal/Packets/Service/Oidb/Response/OidbSvcTrpcTcp0xE07_0Response.cs create mode 100644 Lagrange.Core/Internal/Service/Action/ImageOcrService.cs create mode 100644 Lagrange.OneBot/Core/Entity/Action/OneBotOcrImage.cs create mode 100644 Lagrange.OneBot/Core/Operation/Generic/OcrImageOperation.cs diff --git a/Lagrange.Core/Common/Entity/ImageOcrResultEntry.cs b/Lagrange.Core/Common/Entity/ImageOcrResultEntry.cs new file mode 100644 index 000000000..aa9f9aa5c --- /dev/null +++ b/Lagrange.Core/Common/Entity/ImageOcrResultEntry.cs @@ -0,0 +1,49 @@ +using System.Text.Json.Serialization; + +namespace Lagrange.Core.Common.Entity +{ + [Serializable] + public class ImageOcrResult + { + [JsonPropertyName("texts")] public List Texts { get; set; } + + [JsonPropertyName("language")] public string Language { get; set; } + + public ImageOcrResult(List texts, string language) + { + Texts = texts; + Language = language; + } + } + + [Serializable] + public class TextDetection + { + [JsonPropertyName("text")] public string Text { get; set; } + + [JsonPropertyName("confidence")] public int Confidence { get; set; } + + [JsonPropertyName("coordinates")] public List Coordinates { get; set; } + + public TextDetection(string text, int confidence, List coordinates) + { + Text = text; + Confidence = confidence; + Coordinates = coordinates; + } + } + + [Serializable] + public class Coordinate + { + [JsonPropertyName("x")] public int X { get; set; } + + [JsonPropertyName("y")] public int Y { get; set; } + + public Coordinate(int x, int y) + { + X = x; + Y = y; + } + } +} \ No newline at end of file diff --git a/Lagrange.Core/Common/Interface/Api/OperationExt.cs b/Lagrange.Core/Common/Interface/Api/OperationExt.cs index fee6b08a2..d12d43acc 100644 --- a/Lagrange.Core/Common/Interface/Api/OperationExt.cs +++ b/Lagrange.Core/Common/Interface/Api/OperationExt.cs @@ -261,4 +261,13 @@ public static Task GroupJoinEmojiChain(this BotContext bot, uint groupUin, public static Task FriendJoinEmojiChain(this BotContext bot, uint friendUin, uint emojiId, uint targetMessageSeq) => bot.ContextCollection.Business.OperationLogic.FriendJoinEmojiChain(friendUin, emojiId, targetMessageSeq); + + public static Task UploadImage(this BotContext bot, ImageEntity entity) + => bot.ContextCollection.Business.OperationLogic.UploadImage(entity); + + public static Task OcrImage(this BotContext bot, string url) + => bot.ContextCollection.Business.OperationLogic.ImageOcr(url); + + public static Task OcrImage(this BotContext bot, ImageEntity entity) + => bot.ContextCollection.Business.OperationLogic.ImageOcr(entity); } diff --git a/Lagrange.Core/Internal/Context/Logic/Implementation/OperationLogic.cs b/Lagrange.Core/Internal/Context/Logic/Implementation/OperationLogic.cs index 1d1ce7796..32359d0ca 100644 --- a/Lagrange.Core/Internal/Context/Logic/Implementation/OperationLogic.cs +++ b/Lagrange.Core/Internal/Context/Logic/Implementation/OperationLogic.cs @@ -717,4 +717,28 @@ public async Task FriendJoinEmojiChain(uint friendUin, uint emojiId, uint var results = await Collection.Business.SendEvent(friendJoinEmojiChainEvent); return results.Count != 0 && results[0].ResultCode == 0; } + + public async Task UploadImage(ImageEntity image) + { + await Collection.Highway.ManualUploadEntity(image); + var msgInfo = image.MsgInfo; + if (msgInfo is null) throw new Exception(); + var downloadEvent = ImageDownloadEvent.Create(Collection.Keystore.Uid ?? "", msgInfo); + var result = await Collection.Business.SendEvent(downloadEvent); + var ret = (ImageDownloadEvent)result[0]; + return ret.ImageUrl; + } + + public async Task ImageOcr(string imageUrl) + { + var imageOcrEvent = ImageOcrEvent.Create(imageUrl); + var results = await Collection.Business.SendEvent(imageOcrEvent); + return results.Count != 0 ? ((ImageOcrEvent)results[0]).ImageOcrResult : null; + } + + public async Task ImageOcr(ImageEntity image) + { + var imageUrl = await UploadImage(image); + return await ImageOcr(imageUrl); + } } \ No newline at end of file diff --git a/Lagrange.Core/Internal/Event/Action/ImageOcrEvent.cs b/Lagrange.Core/Internal/Event/Action/ImageOcrEvent.cs new file mode 100644 index 000000000..c41c046bd --- /dev/null +++ b/Lagrange.Core/Internal/Event/Action/ImageOcrEvent.cs @@ -0,0 +1,25 @@ +using Lagrange.Core.Common.Entity; + +namespace Lagrange.Core.Internal.Event.Action; + +internal class ImageOcrEvent : ProtocolEvent +{ + public string Url { get; } = string.Empty; + + public ImageOcrResult ImageOcrResult { get; } + + private ImageOcrEvent(string url) : base(true) + { + Url = url; + ImageOcrResult = new ImageOcrResult(new List(), ""); + } + + private ImageOcrEvent(int resultCode, ImageOcrResult result) : base(resultCode) + { + ImageOcrResult = result; + } + + public static ImageOcrEvent Create(string url) => new(url); + + public static ImageOcrEvent Result(int resultCode, ImageOcrResult result) => new(resultCode, result); +} \ No newline at end of file diff --git a/Lagrange.Core/Internal/Packets/Service/Oidb/Request/OidbSvcTrpcTcp.0xE07_0.cs b/Lagrange.Core/Internal/Packets/Service/Oidb/Request/OidbSvcTrpcTcp.0xE07_0.cs new file mode 100644 index 000000000..4fc299c90 --- /dev/null +++ b/Lagrange.Core/Internal/Packets/Service/Oidb/Request/OidbSvcTrpcTcp.0xE07_0.cs @@ -0,0 +1,41 @@ +using ProtoBuf; + +#pragma warning disable CS8618 +// ReSharper disable InconsistentNaming + +namespace Lagrange.Core.Internal.Packets.Service.Oidb.Request; + +[ProtoContract] +[OidbSvcTrpcTcp(0xE07, 0)] +internal class OidbSvcTrpcTcp0xE07_0 +{ + [ProtoMember(1)] public uint Version { get; set; } + + [ProtoMember(2)] public uint Client { get; set; } + + [ProtoMember(3)] public uint Entrance { get; set; } + + [ProtoMember(10)] public OcrReqBody OcrReqBody { get; set; } +} + +[ProtoContract] +internal class OcrReqBody +{ + [ProtoMember(1)] public string ImageUrl { get; set; } + + [ProtoMember(2)] public uint LanguageType { get; set; } + + [ProtoMember(3)] public uint Scene { get; set; } + + [ProtoMember(10)] public string OriginMd5 { get; set; } + + [ProtoMember(11)] public string AfterCompressMd5 { get; set; } + + [ProtoMember(12)] public string AfterCompressFileSize { get; set; } + + [ProtoMember(13)] public string AfterCompressWeight { get; set; } + + [ProtoMember(14)] public string AfterCompressHeight { get; set; } + + [ProtoMember(15)] public bool IsCut { get; set; } +} \ No newline at end of file diff --git a/Lagrange.Core/Internal/Packets/Service/Oidb/Response/OidbSvcTrpcTcp0xE07_0Response.cs b/Lagrange.Core/Internal/Packets/Service/Oidb/Response/OidbSvcTrpcTcp0xE07_0Response.cs new file mode 100644 index 000000000..3c7b9fdad --- /dev/null +++ b/Lagrange.Core/Internal/Packets/Service/Oidb/Response/OidbSvcTrpcTcp0xE07_0Response.cs @@ -0,0 +1,72 @@ +using ProtoBuf; + +#pragma warning disable CS8618 +// ReSharper disable InconsistentNaming + +namespace Lagrange.Core.Internal.Packets.Service.Oidb.Response; + +[ProtoContract] +internal class OidbSvcTrpcTcp0xE07_0_Response +{ + [ProtoMember(1)] public int RetCode { get; set; } + + [ProtoMember(2)] public string ErrMsg { get; set; } + + [ProtoMember(3)] public string Wording { get; set; } + + [ProtoMember(10)] public OcrRspBody OcrRspBody { get; set; } +} + +[ProtoContract] +internal class OcrRspBody +{ + [ProtoMember(1)] public List TextDetections { get; set; } + + [ProtoMember(2)] public string Language { get; set; } + + [ProtoMember(3)] public string RequestId { get; set; } + + [ProtoMember(101)] public List OcrLanguageList { get; set; } + + [ProtoMember(102)] public List DstTranslateLanguageList { get; set; } + + [ProtoMember(103)] public List LanguageList { get; set; } + + [ProtoMember(111)] public uint AfterCompressWeight { get; set; } + + [ProtoMember(112)] public uint AfterCompressHeight { get; set; } +} + +[ProtoContract] +internal class TextDetection +{ + [ProtoMember(1)] public string DetectedText { get; set; } + + [ProtoMember(2)] public uint Confidence { get; set; } + + [ProtoMember(3)] public Polygon Polygon { get; set; } + + [ProtoMember(4)] public string AdvancedInfo { get; set; } +} + +[ProtoContract] +internal class Polygon +{ + [ProtoMember(1)] public List Coordinates { get; set; } +} + +[ProtoContract] +internal class Coordinate +{ + [ProtoMember(1)] public int X { get; set; } + + [ProtoMember(2)] public int Y { get; set; } +} + +[ProtoContract] +internal class Language +{ + [ProtoMember(1)] public string LanguageCode { get; set; } + + [ProtoMember(2)] public string LanguageDesc { get; set; } +} diff --git a/Lagrange.Core/Internal/Service/Action/ImageOcrService.cs b/Lagrange.Core/Internal/Service/Action/ImageOcrService.cs new file mode 100644 index 000000000..fadf49ab6 --- /dev/null +++ b/Lagrange.Core/Internal/Service/Action/ImageOcrService.cs @@ -0,0 +1,57 @@ +using Lagrange.Core.Common; +using Lagrange.Core.Common.Entity; +using Lagrange.Core.Internal.Event; +using Lagrange.Core.Internal.Event.Action; +using Lagrange.Core.Internal.Packets.Service.Oidb; +using Lagrange.Core.Internal.Packets.Service.Oidb.Request; +using Lagrange.Core.Internal.Packets.Service.Oidb.Response; +using Lagrange.Core.Utility.Extension; +using ProtoBuf; + +namespace Lagrange.Core.Internal.Service.Action; + +[EventSubscribe(typeof(ImageOcrEvent))] +[Service("OidbSvcTrpcTcp.0xe07_0")] +internal class ImageOcrService : BaseService +{ + protected override bool Build(ImageOcrEvent input, BotKeystore keystore, BotAppInfo appInfo, + BotDeviceInfo device, out Span output, out List>? extraPackets) + { + var packet = new OidbSvcTrpcTcpBase(new OidbSvcTrpcTcp0xE07_0 + { + Version = 1, + Client = 0, + Entrance = 1, + OcrReqBody = new OcrReqBody + { + ImageUrl = input.Url, + OriginMd5 = "", + AfterCompressMd5 = "", + AfterCompressFileSize = "", + AfterCompressWeight = "", + AfterCompressHeight = "", + IsCut = false + } + }); + output = packet.Serialize(); + extraPackets = null; + return true; + } + + protected override bool Parse(Span input, BotKeystore keystore, BotAppInfo appInfo, BotDeviceInfo device, + out ImageOcrEvent output, out List? extraEvents) + { + var packet = Serializer.Deserialize>(input); + var response = new ImageOcrResult( + packet?.Body.OcrRspBody.TextDetections.Select(d => new Common.Entity.TextDetection( + d.DetectedText, + (int)d.Confidence, + d.Polygon.Coordinates.Select(c => new Common.Entity.Coordinate(c.X, c.Y)).ToList() + )).ToList() ?? new List(), + packet?.Body.OcrRspBody.Language ?? string.Empty + ); + output = ImageOcrEvent.Result(packet?.Body.RetCode ?? -1, response); + extraEvents = null; + return true; + } +} \ No newline at end of file diff --git a/Lagrange.OneBot/Core/Entity/Action/OneBotOcrImage.cs b/Lagrange.OneBot/Core/Entity/Action/OneBotOcrImage.cs new file mode 100644 index 000000000..0c4cca5a7 --- /dev/null +++ b/Lagrange.OneBot/Core/Entity/Action/OneBotOcrImage.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace Lagrange.OneBot.Core.Entity.Action; + +[Serializable] +public class OneBotOcrImage +{ + [JsonPropertyName("image")] public string Image { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/Lagrange.OneBot/Core/Operation/Ability/UploadImageOperation.cs b/Lagrange.OneBot/Core/Operation/Ability/UploadImageOperation.cs index 78e39a444..45b13ece5 100644 --- a/Lagrange.OneBot/Core/Operation/Ability/UploadImageOperation.cs +++ b/Lagrange.OneBot/Core/Operation/Ability/UploadImageOperation.cs @@ -1,6 +1,6 @@ using System.Text.Json.Nodes; using Lagrange.Core; -using Lagrange.Core.Internal.Event.Message; +using Lagrange.Core.Common.Interface.Api; using Lagrange.Core.Message.Entity; using Lagrange.OneBot.Core.Entity.Action; using Lagrange.OneBot.Message.Entity; @@ -15,15 +15,8 @@ public async Task HandleOperation(BotContext context, JsonNode? pa if (payload?["file"]?.ToString() is { } file && CommonResolver.ResolveStream(file) is { } stream) { var entity = new ImageEntity(stream); - await context.ContextCollection.Highway.ManualUploadEntity(entity); - var msgInfo = entity.MsgInfo; - if (msgInfo is null) throw new Exception(); - - var downloadEvent = ImageDownloadEvent.Create(context.ContextCollection.Keystore.Uid ?? "", msgInfo); - var result = await context.ContextCollection.Business.SendEvent(downloadEvent); - var ret = (ImageDownloadEvent)result[0]; - - return new OneBotResult(ret.ImageUrl, 0, "ok"); + var url = await context.UploadImage(entity); + return new OneBotResult(url, 0, "ok"); } throw new Exception(); diff --git a/Lagrange.OneBot/Core/Operation/Generic/OcrImageOperation.cs b/Lagrange.OneBot/Core/Operation/Generic/OcrImageOperation.cs new file mode 100644 index 000000000..b203c4688 --- /dev/null +++ b/Lagrange.OneBot/Core/Operation/Generic/OcrImageOperation.cs @@ -0,0 +1,27 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Lagrange.Core; +using Lagrange.Core.Message.Entity; +using Lagrange.Core.Common.Interface.Api; +using Lagrange.OneBot.Core.Entity.Action; +using Lagrange.OneBot.Core.Operation.Converters; +using Lagrange.OneBot.Message.Entity; + +namespace Lagrange.OneBot.Core.Operation.Generic; + + +[Operation(".ocr_image")] +[Operation("ocr_image")] +public class OcrImageOperation() : IOperation +{ + public async Task HandleOperation(BotContext context, JsonNode? payload) + { + if (payload.Deserialize(SerializerOptions.DefaultOptions) is { } data && CommonResolver.ResolveStream(data.Image) is { } stream) + { + var entity = new ImageEntity(stream); + var res = await context.OcrImage(entity); + return new OneBotResult(res, 0, "ok"); + } + throw new Exception(); + } +} \ No newline at end of file