From 9ed9b65cc16a1dbea9f79db85167c08380596e36 Mon Sep 17 00:00:00 2001 From: Xiaowei Guan Date: Mon, 15 Apr 2024 20:29:50 +0800 Subject: [PATCH] Add embedder texture initial code for impeller Add registerTextureWithType interface to proc table --- shell/platform/embedder/embedder.cc | 22 +++ shell/platform/embedder/embedder.h | 50 +++++- shell/platform/embedder/embedder_engine.cc | 9 + shell/platform/embedder/embedder_engine.h | 2 + .../embedder/embedder_external_texture_gl.cc | 156 ++++++++++++++++-- .../embedder/embedder_external_texture_gl.h | 69 +++++++- .../embedder_external_texture_resolver.cc | 30 +++- .../embedder_external_texture_resolver.h | 3 + 8 files changed, 314 insertions(+), 27 deletions(-) diff --git a/shell/platform/embedder/embedder.cc b/shell/platform/embedder/embedder.cc index 52e49646c009a..0921b71a04cd0 100644 --- a/shell/platform/embedder/embedder.cc +++ b/shell/platform/embedder/embedder.cc @@ -2712,6 +2712,26 @@ FlutterEngineResult FlutterEngineRegisterExternalTexture( return kSuccess; } +FlutterEngineResult FlutterEngineRegisterExternalTextureWithType( + FLUTTER_API_SYMBOL(FlutterEngine) engine, + int64_t texture_identifier, + FlutterTextureType type) { + if (engine == nullptr) { + return LOG_EMBEDDER_ERROR(kInvalidArguments, "Engine handle was invalid."); + } + + if (texture_identifier == 0) { + return LOG_EMBEDDER_ERROR(kInvalidArguments, + "Texture identifier was invalid."); + } + if (!reinterpret_cast(engine)->RegisterTexture( + texture_identifier, type)) { + return LOG_EMBEDDER_ERROR(kInternalInconsistency, + "Could not register the specified texture."); + } + return kSuccess; +} + FlutterEngineResult FlutterEngineUnregisterExternalTexture( FLUTTER_API_SYMBOL(FlutterEngine) engine, int64_t texture_identifier) { @@ -3287,6 +3307,8 @@ FlutterEngineResult FlutterEngineGetProcAddresses( SET_PROC(SendPlatformMessageResponse, FlutterEngineSendPlatformMessageResponse); SET_PROC(RegisterExternalTexture, FlutterEngineRegisterExternalTexture); + SET_PROC(RegisterExternalTextureWithType, + FlutterEngineRegisterExternalTextureWithType); SET_PROC(UnregisterExternalTexture, FlutterEngineUnregisterExternalTexture); SET_PROC(MarkExternalTextureFrameAvailable, FlutterEngineMarkExternalTextureFrameAvailable); diff --git a/shell/platform/embedder/embedder.h b/shell/platform/embedder/embedder.h index d26e2215adf3a..6a879452cd139 100644 --- a/shell/platform/embedder/embedder.h +++ b/shell/platform/embedder/embedder.h @@ -294,6 +294,7 @@ typedef struct { } FlutterTransformation; typedef void (*VoidCallback)(void* /* user data */); +typedef bool (*BoolCallback)(void* /* user data */); typedef enum { /// Specifies an OpenGL texture target type. Textures are specified using @@ -371,6 +372,8 @@ typedef struct { uint32_t format; /// User data to be returned on the invocation of the destruction callback. void* user_data; + /// Callback invoked that texture start binding. + BoolCallback bind_callback; /// Callback invoked (on an engine managed thread) that asks the embedder to /// collect the texture. VoidCallback destruction_callback; @@ -382,6 +385,10 @@ typedef struct { size_t width; /// Height of the texture. size_t height; + /// The pixel data buffer. + const uint8_t* buffer; + /// The size of buffer. + size_t buffer_size; } FlutterOpenGLTexture; typedef struct { @@ -401,7 +408,15 @@ typedef struct { VoidCallback destruction_callback; } FlutterOpenGLFramebuffer; -typedef bool (*BoolCallback)(void* /* user data */); +// Possible values for the type specified in FlutterDesktopTextureInfo. +// Additional types may be added in the future. +typedef enum { + // A Pixel buffer-based texture. + kFlutterPixelBufferTexture, + // A platform-specific GPU surface-backed texture. + kFlutterGpuSurfaceTexture +} FlutterTextureType; + typedef FlutterTransformation (*TransformationCallback)(void* /* user data */); typedef uint32_t (*UIntCallback)(void* /* user data */); typedef bool (*SoftwareSurfacePresentCallback)(void* /* user data */, @@ -2679,6 +2694,32 @@ FlutterEngineResult FlutterEngineRegisterExternalTexture( FLUTTER_API_SYMBOL(FlutterEngine) engine, int64_t texture_identifier); +//------------------------------------------------------------------------------ +/// @brief Register an external texture with a unique (per engine) +/// identifier. Only rendering backends that support external +/// textures accept external texture registrations. After the +/// external texture is registered, the application can mark that a +/// frame is available by calling +/// `FlutterEngineMarkExternalTextureFrameAvailable`. +/// +/// @see FlutterEngineUnregisterExternalTexture() +/// @see FlutterEngineMarkExternalTextureFrameAvailable() +/// +/// @param[in] engine A running engine instance. +/// @param[in] texture_identifier The identifier of the texture to register +/// with the engine. The embedder may supply new +/// frames to this texture using the same +/// identifier. +/// @param[in] type The type of the texture. +/// +/// @return The result of the call. +/// +FLUTTER_EXPORT +FlutterEngineResult FlutterEngineRegisterExternalTextureWithType( + FLUTTER_API_SYMBOL(FlutterEngine) engine, + int64_t texture_identifier, + FlutterTextureType type); + //------------------------------------------------------------------------------ /// @brief Unregister a previous texture registration. /// @@ -3131,6 +3172,11 @@ typedef FlutterEngineResult (*FlutterEngineSendPlatformMessageResponseFnPtr)( typedef FlutterEngineResult (*FlutterEngineRegisterExternalTextureFnPtr)( FLUTTER_API_SYMBOL(FlutterEngine) engine, int64_t texture_identifier); +typedef FlutterEngineResult ( + *FlutterEngineRegisterExternalTextureWithTypeFnPtr)( + FLUTTER_API_SYMBOL(FlutterEngine) engine, + int64_t texture_identifier, + FlutterTextureType type); typedef FlutterEngineResult (*FlutterEngineUnregisterExternalTextureFnPtr)( FLUTTER_API_SYMBOL(FlutterEngine) engine, int64_t texture_identifier); @@ -3217,6 +3263,8 @@ typedef struct { PlatformMessageReleaseResponseHandle; FlutterEngineSendPlatformMessageResponseFnPtr SendPlatformMessageResponse; FlutterEngineRegisterExternalTextureFnPtr RegisterExternalTexture; + FlutterEngineRegisterExternalTextureWithTypeFnPtr + RegisterExternalTextureWithType; FlutterEngineUnregisterExternalTextureFnPtr UnregisterExternalTexture; FlutterEngineMarkExternalTextureFrameAvailableFnPtr MarkExternalTextureFrameAvailable; diff --git a/shell/platform/embedder/embedder_engine.cc b/shell/platform/embedder/embedder_engine.cc index b34da8a287f4e..16a4e08f17a94 100644 --- a/shell/platform/embedder/embedder_engine.cc +++ b/shell/platform/embedder/embedder_engine.cc @@ -153,6 +153,15 @@ bool EmbedderEngine::RegisterTexture(int64_t texture) { return true; } +bool EmbedderEngine::RegisterTexture(int64_t texture, FlutterTextureType type) { + if (!IsValid()) { + return false; + } + shell_->GetPlatformView()->RegisterTexture( + external_texture_resolver_->ResolveExternalTexture(texture, type)); + return true; +} + bool EmbedderEngine::UnregisterTexture(int64_t texture) { if (!IsValid()) { return false; diff --git a/shell/platform/embedder/embedder_engine.h b/shell/platform/embedder/embedder_engine.h index a587eeb3eaa74..472d108ddf8c8 100644 --- a/shell/platform/embedder/embedder_engine.h +++ b/shell/platform/embedder/embedder_engine.h @@ -58,6 +58,8 @@ class EmbedderEngine { bool RegisterTexture(int64_t texture); + bool RegisterTexture(int64_t texture, FlutterTextureType type); + bool UnregisterTexture(int64_t texture); bool MarkTextureFrameAvailable(int64_t texture); diff --git a/shell/platform/embedder/embedder_external_texture_gl.cc b/shell/platform/embedder/embedder_external_texture_gl.cc index bbf30d143d133..978b8409921e0 100644 --- a/shell/platform/embedder/embedder_external_texture_gl.cc +++ b/shell/platform/embedder/embedder_external_texture_gl.cc @@ -5,6 +5,11 @@ #include "flutter/shell/platform/embedder/embedder_external_texture_gl.h" #include "flutter/fml/logging.h" +#include "flutter/impeller/display_list/dl_image_impeller.h" +#include "flutter/impeller/renderer/backend/gles/context_gles.h" +#include "flutter/impeller/renderer/backend/gles/texture_gles.h" +#include "impeller/aiks/aiks_context.h" +#include "impeller/renderer/backend/gles/gles.h" #include "include/core/SkCanvas.h" #include "include/core/SkPaint.h" #include "third_party/skia/include/core/SkAlphaType.h" @@ -37,7 +42,7 @@ void EmbedderExternalTextureGL::Paint(PaintContext& context, if (last_image_ == nullptr) { last_image_ = ResolveTexture(Id(), // - context.gr_context, // + context, // SkISize::Make(bounds.width(), bounds.height()) // ); } @@ -55,12 +60,34 @@ void EmbedderExternalTextureGL::Paint(PaintContext& context, } } -sk_sp EmbedderExternalTextureGL::ResolveTexture( +// |flutter::Texture| +void EmbedderExternalTextureGL::OnGrContextCreated() {} + +// |flutter::Texture| +void EmbedderExternalTextureGL::OnGrContextDestroyed() {} + +// |flutter::Texture| +void EmbedderExternalTextureGL::MarkNewFrameAvailable() { + last_image_ = nullptr; +} + +// |flutter::Texture| +void EmbedderExternalTextureGL::OnTextureUnregistered() {} + +EmbedderExternalTextureSkiaGL::EmbedderExternalTextureSkiaGL( + int64_t texture_identifier, + const ExternalTextureCallback& callback) + : EmbedderExternalTextureGL(texture_identifier, callback) {} + +EmbedderExternalTextureSkiaGL::~EmbedderExternalTextureSkiaGL() = default; + +sk_sp EmbedderExternalTextureSkiaGL::ResolveTexture( int64_t texture_id, - GrDirectContext* context, + PaintContext& context, const SkISize& size) { - context->flushAndSubmit(); - context->resetContext(kAll_GrBackendState); + GrDirectContext* gr_context = context.gr_context; + gr_context->flushAndSubmit(); + gr_context->resetContext(kAll_GrBackendState); std::unique_ptr texture = external_texture_callback_(texture_id, size.width(), size.height()); @@ -83,7 +110,7 @@ sk_sp EmbedderExternalTextureGL::ResolveTexture( width, height, skgpu::Mipmapped::kNo, gr_texture_info); SkImages::TextureReleaseProc release_proc = texture->destruction_callback; auto image = - SkImages::BorrowTextureFrom(context, // context + SkImages::BorrowTextureFrom(gr_context, // context gr_backend_texture, // texture handle kTopLeft_GrSurfaceOrigin, // origin kRGBA_8888_SkColorType, // color type @@ -99,7 +126,6 @@ sk_sp EmbedderExternalTextureGL::ResolveTexture( if (release_proc) { release_proc(texture->user_data); } - FML_LOG(ERROR) << "Could not create external texture->"; return nullptr; } @@ -107,18 +133,116 @@ sk_sp EmbedderExternalTextureGL::ResolveTexture( return DlImage::Make(std::move(image)); } -// |flutter::Texture| -void EmbedderExternalTextureGL::OnGrContextCreated() {} +EmbedderExternalTextureGLImpellerPixelBuffer:: + EmbedderExternalTextureGLImpellerPixelBuffer( + int64_t texture_identifier, + const ExternalTextureCallback& callback) + : EmbedderExternalTextureGL(texture_identifier, callback) {} -// |flutter::Texture| -void EmbedderExternalTextureGL::OnGrContextDestroyed() {} +sk_sp EmbedderExternalTextureGLImpellerPixelBuffer::ResolveTexture( + int64_t texture_id, + PaintContext& context, + const SkISize& size) { + std::unique_ptr texture = + external_texture_callback_(texture_id, size.width(), size.height()); -// |flutter::Texture| -void EmbedderExternalTextureGL::MarkNewFrameAvailable() { - last_image_ = nullptr; + if (!texture) { + return nullptr; + } + + size_t width = size.width(); + size_t height = size.height(); + + if (texture->width != 0 && texture->height != 0) { + width = texture->width; + height = texture->height; + } + + impeller::TextureDescriptor desc; + desc.type = impeller::TextureType::kTexture2D; + impeller::AiksContext* aiks_context = context.aiks_context; + const auto& gl_context = + impeller::ContextGLES::Cast(*aiks_context->GetContext()); + desc.storage_mode = impeller::StorageMode::kDevicePrivate; + desc.format = impeller::PixelFormat::kR8G8B8A8UNormInt; + desc.size = {static_cast(width), static_cast(height)}; + desc.mip_count = 1; + auto textureGLES = + std::make_shared(gl_context.GetReactor(), desc); + if (!textureGLES->SetContents(texture->buffer, texture->buffer_size)) { + if (texture->destruction_callback) { + texture->destruction_callback(texture->user_data); + } + return nullptr; + } + if (texture->destruction_callback) { + texture->destruction_callback(texture->user_data); + } + return impeller::DlImageImpeller::Make(textureGLES); } -// |flutter::Texture| -void EmbedderExternalTextureGL::OnTextureUnregistered() {} +EmbedderExternalTextureGLImpellerPixelBuffer:: + ~EmbedderExternalTextureGLImpellerPixelBuffer() = default; + +EmbedderExternalTextureGLImpellerSurface:: + EmbedderExternalTextureGLImpellerSurface( + int64_t texture_identifier, + const ExternalTextureCallback& callback) + : EmbedderExternalTextureGL(texture_identifier, callback) {} + +sk_sp EmbedderExternalTextureGLImpellerSurface::ResolveTexture( + int64_t texture_id, + PaintContext& context, + const SkISize& size) { + std::unique_ptr texture = + external_texture_callback_(texture_id, size.width(), size.height()); + + if (!texture) { + return nullptr; + } + size_t width = size.width(); + size_t height = size.height(); + + if (texture->width != 0 && texture->height != 0) { + width = texture->width; + height = texture->height; + } + + impeller::TextureDescriptor desc; + desc.type = impeller::TextureType::kTextureExternalOES; + impeller::AiksContext* aiks_context = context.aiks_context; + const auto& gl_context = + impeller::ContextGLES::Cast(*aiks_context->GetContext()); + desc.storage_mode = impeller::StorageMode::kDevicePrivate; + desc.format = impeller::PixelFormat::kR8G8B8A8UNormInt; + desc.size = {static_cast(width), static_cast(height)}; + desc.mip_count = 1; + auto textureGLES = std::make_shared( + gl_context.GetReactor(), desc, + impeller::TextureGLES::IsWrapped::kWrapped); + textureGLES->SetCoordinateSystem( + impeller::TextureCoordinateSystem::kUploadFromHost); + if (!textureGLES->Bind()) { + if (texture->destruction_callback) { + texture->destruction_callback(texture->user_data); + } + return nullptr; + } + + if (!texture->bind_callback(texture->user_data)) { + if (texture->destruction_callback) { + texture->destruction_callback(texture->user_data); + } + return nullptr; + } + + if (texture->destruction_callback) { + texture->destruction_callback(texture->user_data); + } + + return impeller::DlImageImpeller::Make(textureGLES); +} +EmbedderExternalTextureGLImpellerSurface:: + ~EmbedderExternalTextureGLImpellerSurface() = default; } // namespace flutter diff --git a/shell/platform/embedder/embedder_external_texture_gl.h b/shell/platform/embedder/embedder_external_texture_gl.h index b01991d0cf1df..15457c4c0b00b 100644 --- a/shell/platform/embedder/embedder_external_texture_gl.h +++ b/shell/platform/embedder/embedder_external_texture_gl.h @@ -22,14 +22,6 @@ class EmbedderExternalTextureGL : public flutter::Texture { ~EmbedderExternalTextureGL(); - private: - const ExternalTextureCallback& external_texture_callback_; - sk_sp last_image_; - - sk_sp ResolveTexture(int64_t texture_id, - GrDirectContext* context, - const SkISize& size); - // |flutter::Texture| void Paint(PaintContext& context, const SkRect& bounds, @@ -48,9 +40,70 @@ class EmbedderExternalTextureGL : public flutter::Texture { // |flutter::Texture| void OnTextureUnregistered() override; + protected: + virtual sk_sp ResolveTexture(int64_t texture_id, + PaintContext& context, + const SkISize& size) = 0; + const ExternalTextureCallback& external_texture_callback_; + sk_sp last_image_; + + private: FML_DISALLOW_COPY_AND_ASSIGN(EmbedderExternalTextureGL); }; +class EmbedderExternalTextureSkiaGL + : public flutter::EmbedderExternalTextureGL { + public: + EmbedderExternalTextureSkiaGL(int64_t texture_identifier, + const ExternalTextureCallback& callback); + + ~EmbedderExternalTextureSkiaGL(); + + protected: + sk_sp ResolveTexture(int64_t texture_id, + PaintContext& context, + const SkISize& size) override; + + private: + FML_DISALLOW_COPY_AND_ASSIGN(EmbedderExternalTextureSkiaGL); +}; + +class EmbedderExternalTextureGLImpellerPixelBuffer + : public flutter::EmbedderExternalTextureGL { + public: + EmbedderExternalTextureGLImpellerPixelBuffer( + int64_t texture_identifier, + const ExternalTextureCallback& callback); + + ~EmbedderExternalTextureGLImpellerPixelBuffer(); + + protected: + sk_sp ResolveTexture(int64_t texture_id, + PaintContext& context, + const SkISize& size) override; + + private: + FML_DISALLOW_COPY_AND_ASSIGN(EmbedderExternalTextureGLImpellerPixelBuffer); +}; + +class EmbedderExternalTextureGLImpellerSurface + : public flutter::EmbedderExternalTextureGL { + public: + EmbedderExternalTextureGLImpellerSurface( + int64_t texture_identifier, + const ExternalTextureCallback& callback); + + ~EmbedderExternalTextureGLImpellerSurface(); + + protected: + sk_sp ResolveTexture(int64_t texture_id, + PaintContext& context, + const SkISize& size) override; + + private: + FML_DISALLOW_COPY_AND_ASSIGN(EmbedderExternalTextureGLImpellerSurface); +}; + } // namespace flutter #endif // FLUTTER_SHELL_PLATFORM_EMBEDDER_EMBEDDER_EXTERNAL_TEXTURE_GL_H_ diff --git a/shell/platform/embedder/embedder_external_texture_resolver.cc b/shell/platform/embedder/embedder_external_texture_resolver.cc index 4e4b41b8d4ad2..d4fbff9eb7ca0 100644 --- a/shell/platform/embedder/embedder_external_texture_resolver.cc +++ b/shell/platform/embedder/embedder_external_texture_resolver.cc @@ -21,12 +21,38 @@ EmbedderExternalTextureResolver::EmbedderExternalTextureResolver( : metal_callback_(std::move(metal_callback)) {} #endif +std::unique_ptr +EmbedderExternalTextureResolver::ResolveExternalTexture( + int64_t texture_id, + FlutterTextureType type) { +#ifdef SHELL_ENABLE_GL + if (gl_callback_) { + if (type == FlutterTextureType::kFlutterGpuSurfaceTexture) { + return std::make_unique( + texture_id, gl_callback_); + } else { + return std::make_unique( + texture_id, gl_callback_); + } + } +#endif + +#ifdef SHELL_ENABLE_METAL + if (metal_callback_) { + return std::make_unique(texture_id, + metal_callback_); + } +#endif + + return nullptr; +} + std::unique_ptr EmbedderExternalTextureResolver::ResolveExternalTexture(int64_t texture_id) { #ifdef SHELL_ENABLE_GL if (gl_callback_) { - return std::make_unique(texture_id, - gl_callback_); + return std::make_unique(texture_id, + gl_callback_); } #endif diff --git a/shell/platform/embedder/embedder_external_texture_resolver.h b/shell/platform/embedder/embedder_external_texture_resolver.h index 412301736e566..bc77a6885b9fa 100644 --- a/shell/platform/embedder/embedder_external_texture_resolver.h +++ b/shell/platform/embedder/embedder_external_texture_resolver.h @@ -8,6 +8,7 @@ #include #include "flutter/common/graphics/texture.h" +#include "flutter/shell/platform/embedder/embedder.h" #ifdef SHELL_ENABLE_GL #include "flutter/shell/platform/embedder/embedder_external_texture_gl.h" @@ -35,6 +36,8 @@ class EmbedderExternalTextureResolver { #endif std::unique_ptr ResolveExternalTexture(int64_t texture_id); + std::unique_ptr ResolveExternalTexture(int64_t texture_id, + FlutterTextureType type); bool SupportsExternalTextures();