diff --git a/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_delegate.cc b/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_delegate.cc deleted file mode 100644 index 430b82f946..0000000000 --- a/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_delegate.cc +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2024 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_delegate.h" - -#include -#include -#include - -#include "absl/log/absl_check.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "mediapipe/calculators/tensor/inference_calculator_utils.h" -#include "mediapipe/calculators/tensor/inference_io_mapper.h" -#include "mediapipe/calculators/tensor/tensor_span.h" -#include "mediapipe/framework/api2/packet.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_macros.h" -#include "mediapipe/util/tflite/tflite_model_loader.h" -#include "tensorflow/lite/c/c_api_types.h" -#include "tensorflow/lite/core/interpreter_builder.h" -#include "tensorflow/lite/interpreter.h" -#include "tensorflow/lite/util.h" -#include "third_party/ml_drift/contrib/tflite_op_resolver.h" -#include "third_party/odml/infra/ml_drift_delegate/ml_drift_cl.h" -#include "third_party/odml/infra/ml_drift_delegate/ml_drift_delegate.h" - -namespace mediapipe::api2 { - -using ::tflite::ml_drift::MlDriftClDelegateDefaultOptionsPtr; -using ::tflite::ml_drift::TfLiteCreateMlDriftClDelegate; - -absl::Status InferenceRunnerMlDriftOpenClDelegate::Init( - const mediapipe::InferenceCalculatorOptions& options, - Packet model_packet, - Packet op_resolver_packet) { - RET_CHECK_EQ(options.delegate().gpu().api(), - InferenceCalculatorOptions::Delegate::Gpu::ML_DRIFT_OPENCL); - model_packet_ = model_packet; - tflite::InterpreterBuilder builder(*model_packet_.Get(), - op_resolver_packet.Get()); - builder(&interpreter_); - ABSL_CHECK(interpreter_ != nullptr); - MP_ASSIGN_OR_RETURN( - input_output_tensor_names_, - InferenceIoMapper::GetInputOutputTensorNamesFromInterpreter( - *interpreter_)); - // Initialize ML Drift CL. - auto delegate_options = MlDriftClDelegateDefaultOptionsPtr(); - delegate_options->enable_fast_tuning = true; - if (options.delegate().gpu().allow_precision_loss()) { - delegate_options->precision = MlDriftDelegatePrecision::kDefault; - } else { - delegate_options->precision = MlDriftDelegatePrecision::kFp32; - } - tflite::Interpreter::TfLiteDelegatePtr delegate = - TfLiteCreateMlDriftClDelegate(std::move(delegate_options)); - ABSL_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(std::move(delegate)), - kTfLiteOk); - interpreter_->AllocateTensors(); - return absl::OkStatus(); -} - -absl::StatusOr> InferenceRunnerMlDriftOpenClDelegate::Run( - CalculatorContext* cc, const TensorSpan& input_tensors) { - // If the input tensors have dynamic shape, then the tensors need to be - // resized and reallocated before we can copy the tensor values. - bool resized_tensor_shapes = false; - for (int i = 0; i < input_tensors.size(); ++i) { - const Tensor& input_tensor = input_tensors[i]; - if (input_tensor.shape().is_dynamic) { - const TfLiteTensor* interpreter_tensor = - interpreter_->tensor(interpreter_->inputs()[i]); - // TODO: Can avoid copying even these <= 4 values in the future. - std::vector interpreter_dims{ - interpreter_tensor->dims->data, - interpreter_tensor->dims->data + interpreter_tensor->dims->size}; - if (interpreter_dims != input_tensor.shape().dims) { - interpreter_->ResizeInputTensorStrict(i, input_tensor.shape().dims); - resized_tensor_shapes = true; - } - } - } - // Reallocation is needed for memory sanity. - if (resized_tensor_shapes) { - interpreter_->AllocateTensors(); - } - - for (int i = 0; i < input_tensors.size(); ++i) { - const Tensor& input_tensor = input_tensors[i]; - MP_RETURN_IF_ERROR( - CopyCpuInputIntoInterpreterTensor(input_tensor, *interpreter_, i)); - } - ABSL_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk); - MP_ASSIGN_OR_RETURN(auto output_tensors, - AllocateOutputTensors(*interpreter_)); - for (int i = 0; i < output_tensors.size(); ++i) { - const int output_tensor_index = interpreter_->outputs()[i]; - MP_RETURN_IF_ERROR(CopyInterpreterTensorIntoCpuOutput( - *interpreter_, output_tensor_index, output_tensors[i])); - } - return output_tensors; -} - -absl::StatusOr> -InferenceRunnerMlDriftOpenClDelegate::AllocateOutputTensors( - const tflite::Interpreter& interpreter) { - const int num_outputs = interpreter.outputs().size(); - std::vector output_tensors; - output_tensors.reserve(num_outputs); - for (int i = 0; i < num_outputs; ++i) { - const TfLiteTensor* reference_tensor = - interpreter.tensor(interpreter.outputs()[i]); - MP_ASSIGN_OR_RETURN(Tensor output_tensor, - CreateTensorWithTfLiteTensorSpecs( - *reference_tensor, /*memory_manager=*/nullptr, - tflite::kDefaultTensorAlignment)); - output_tensors.push_back(std::move(output_tensor)); - } - return output_tensors; -} - -const InputOutputTensorNames& -InferenceRunnerMlDriftOpenClDelegate::GetInputOutputTensorNames() const { - return input_output_tensor_names_; -} - -} // namespace mediapipe::api2 diff --git a/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_delegate.h b/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_delegate.h deleted file mode 100644 index 605c1b78cf..0000000000 --- a/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_delegate.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2024 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_RUNNER_ML_DRIFT_OPENCL_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_RUNNER_ML_DRIFT_OPENCL_H_ - -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "mediapipe/calculators/tensor/inference_io_mapper.h" -#include "mediapipe/calculators/tensor/inference_runner.h" -#include "mediapipe/calculators/tensor/tensor_span.h" -#include "mediapipe/framework/api2/packet.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/util/tflite/tflite_model_loader.h" -#include "tensorflow/lite/interpreter.h" -#include "third_party/ml_drift/contrib/tflite_op_resolver.h" - -namespace mediapipe::api2 { - -// Inference runner implementation that uses the ML Drift OpenCL Delegate. -class InferenceRunnerMlDriftOpenClDelegate : public InferenceRunner { - public: - ~InferenceRunnerMlDriftOpenClDelegate() override = default; - - absl::Status Init( - const mediapipe::InferenceCalculatorOptions& options, - Packet model_packet, - Packet op_resolver_packet); - - absl::StatusOr> Run( - CalculatorContext* cc, const TensorSpan& input_tensors) override; - - const InputOutputTensorNames& GetInputOutputTensorNames() const override; - - private: - static absl::StatusOr> AllocateOutputTensors( - const tflite::Interpreter& interpreter); - - // TfLite requires us to keep the model alive as long as the interpreter is. - Packet model_packet_; - InputOutputTensorNames input_output_tensor_names_; - std::unique_ptr interpreter_; -}; - -} // namespace mediapipe::api2 - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_RUNNER_ML_DRIFT_OPENCL_H_ diff --git a/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_runtime.cc b/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_runtime.cc deleted file mode 100644 index 9c85a14fcf..0000000000 --- a/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_runtime.cc +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2024 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_runtime.h" - -#include -#include -#include - -#include "absl/log/log.h" -#include "absl/status/status.h" -#include "mediapipe/calculators/tensor/inference_io_mapper.h" -#include "mediapipe/calculators/tensor/tensor_span.h" // IWYU pragma: keep -#include "mediapipe/framework/api2/packet.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port.h" // IWYU pragma: keep -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_macros.h" -#include "mediapipe/util/tflite/tflite_model_loader.h" -#include "tensorflow/lite/core/api/op_resolver.h" -#include "tensorflow/lite/model_builder.h" -#include "third_party/GL/gl/include/GLES2/gl2.h" -#include "third_party/ml_drift/cl/api.h" -#include "third_party/ml_drift/common/data_type.h" -#include "third_party/ml_drift/common/model.h" -#include "third_party/ml_drift/common/model_builder.h" -#include "third_party/ml_drift/common/shape.h" -#include "third_party/ml_drift/contrib/tflite_op_resolver.h" -#include "third_party/ml_drift/delegate/api_cl_gl_vk.h" -#include "third_party/ml_drift/delegate/api_common.h" - -namespace mediapipe::api2 { - -namespace { - -using ::ml_drift::BHWC; -using ::ml_drift::DataLayout; -using ::ml_drift::DataType; -using ::ml_drift::GraphFloat32; -using ::ml_drift::InferenceBuilder; -using ::ml_drift::ObjectDef; -using ::ml_drift::ObjectType; -using ::ml_drift::OpenGlBuffer; -using ::ml_drift::cl::InferenceEnvironmentOptions; -using ::ml_drift::cl::InferenceEnvironmentProperties; - -ObjectDef GetSSboObjectDef(int channels) { - ObjectDef gpu_object_def; - gpu_object_def.data_type = DataType::FLOAT32; - gpu_object_def.data_layout = DataLayout::BHWC; - if (channels == 4) { - gpu_object_def.data_layout = DataLayout::DHWC4; - } - gpu_object_def.object_type = ObjectType::OPENGL_SSBO; - gpu_object_def.user_provided = true; - return gpu_object_def; -} - -} // namespace - -absl::Status InferenceRunnerMlDriftOpenClRuntime::Init( - const InferenceCalculatorOptions& options, - Packet model_packet, - Packet op_resolver_packet) { - RET_CHECK_EQ(options.delegate().gpu().api(), - InferenceCalculatorOptions::Delegate::Gpu::ML_DRIFT_OPENCL); - - bool allow_precision_loss = options.delegate().gpu().allow_precision_loss(); - - ml_drift::cl::InferenceOptions mldrift_options; - mldrift_options.priority1 = allow_precision_loss - ? ml_drift::InferencePriority::MIN_LATENCY - : ml_drift::InferencePriority::MAX_PRECISION; - mldrift_options.priority2 = ml_drift::InferencePriority::AUTO; - mldrift_options.priority3 = ml_drift::InferencePriority::AUTO; - switch (options.delegate().gpu().usage()) { - case mediapipe::InferenceCalculatorOptions::Delegate::Gpu:: - FAST_SINGLE_ANSWER: { - mldrift_options.usage = ml_drift::InferenceUsage::FAST_SINGLE_ANSWER; - break; - } - case mediapipe::InferenceCalculatorOptions::Delegate::Gpu:: - SUSTAINED_SPEED: { - mldrift_options.usage = ml_drift::InferenceUsage::SUSTAINED_SPEED; - break; - } - case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::UNSPECIFIED: { - return absl::InternalError("inference usage need to be specified."); - } - } - - MP_ASSIGN_OR_RETURN(input_output_tensor_names_, - InferenceIoMapper::GetInputOutputTensorNamesFromModel( - *model_packet.Get(), op_resolver_packet.Get())); - - MP_ASSIGN_OR_RETURN( - GraphFloat32 graph_cl, - InitModelFromFlatBuffer(*model_packet.Get(), op_resolver_packet.Get(), - /*allow_quant_ops=*/true)); - - tensor_output_shapes_.reserve(output_shapes_.size()); - for (int i = 0; i < output_shapes_.size(); ++i) { - tensor_output_shapes_.push_back({output_shapes_[i].b, output_shapes_[i].h, - output_shapes_[i].w, output_shapes_[i].c}); - } - - return InitializeMlDriftRuntime(std::move(graph_cl), mldrift_options); -} - -// Tensor::GetOpenGlBufferReadView is only defined for OpenGl ES 3.1. -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 -absl::StatusOr> InferenceRunnerMlDriftOpenClRuntime::Run( - CalculatorContext* cc, const TensorSpan& input_tensors) { - std::vector output_tensors; - for (int i = 0; i < input_tensors.size(); ++i) { - MP_RETURN_IF_ERROR(BindSsboToInputTensor( - input_tensors[i].GetOpenGlBufferReadView().name(), i)); - } - output_tensors.reserve(tensor_output_shapes_.size()); - for (int i = 0; i < tensor_output_shapes_.size(); ++i) { - output_tensors.emplace_back(Tensor::ElementType::kFloat32, - tensor_output_shapes_[i]); - MP_RETURN_IF_ERROR(BindSsboToOutputTensor( - output_tensors.back().GetOpenGlBufferWriteView().name(), i)); - } - // Run inference. - MP_RETURN_IF_ERROR(runner_->Run()); - return output_tensors; -} -#endif - -const InputOutputTensorNames& -InferenceRunnerMlDriftOpenClRuntime::GetInputOutputTensorNames() const { - return input_output_tensor_names_; -} - -absl::StatusOr -InferenceRunnerMlDriftOpenClRuntime::InitModelFromFlatBuffer( - const tflite::FlatBufferModel& flatbuffer, - const tflite::OpResolver& op_resolver, bool allow_quant_ops) { - GraphFloat32 graph_cl; - MP_RETURN_IF_ERROR( - BuildFromFlatBuffer(flatbuffer, op_resolver, &graph_cl, allow_quant_ops)); - - for (const auto& input : graph_cl.inputs()) { - input_shapes_.push_back(input->tensor.shape); - } - for (const auto& output : graph_cl.outputs()) { - output_shapes_.push_back(output->tensor.shape); - } - return graph_cl; -} - -absl::Status InferenceRunnerMlDriftOpenClRuntime::InitializeMlDriftRuntime( - GraphFloat32&& graph_cl, const ml_drift::cl::InferenceOptions& options) { - // 1. Prepare inference builder. - std::unique_ptr builder; - - InferenceEnvironmentOptions env_options; - InferenceEnvironmentProperties properties; - MP_RETURN_IF_ERROR( - NewInferenceEnvironment(env_options, &cl_environment_, &properties)); - - // Initialize from scratch. - MP_RETURN_IF_ERROR(cl_environment_->NewInferenceBuilder( - options, std::move(graph_cl), &builder)); - - // 2. Describe output/input objects for created builder. - for (int flow_index = 0; flow_index < input_shapes_.size(); ++flow_index) { - MP_RETURN_IF_ERROR(builder->SetInputObjectDef( - flow_index, GetSSboObjectDef(input_shapes_[flow_index].c))); - } - for (int flow_index = 0; flow_index < output_shapes_.size(); ++flow_index) { - MP_RETURN_IF_ERROR(builder->SetOutputObjectDef( - flow_index, GetSSboObjectDef(output_shapes_[flow_index].c))); - } - // 3. Build inference runner with the created builder. - MP_ASSIGN_OR_RETURN(runner_, builder->Build()); - return absl::OkStatus(); -} - -absl::Status InferenceRunnerMlDriftOpenClRuntime::BindSsboToInputTensor( - GLuint ssbo_id, int input_id) { - OpenGlBuffer buffer; - buffer.id = ssbo_id; - return runner_->SetInputObject(input_id, std::move(buffer)); -} - -absl::Status InferenceRunnerMlDriftOpenClRuntime::BindSsboToOutputTensor( - GLuint ssbo_id, int output_id) { - OpenGlBuffer buffer; - buffer.id = ssbo_id; - return runner_->SetOutputObject(output_id, std::move(buffer)); -} - -} // namespace mediapipe::api2 diff --git a/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_runtime.h b/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_runtime.h deleted file mode 100644 index c9d76f2d6a..0000000000 --- a/mediapipe/calculators/tensor/inference_runner_ml_drift_opencl_runtime.h +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2024 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_RUNNER_ML_DRIFT_OPENCL_RUNTIME_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_RUNNER_ML_DRIFT_OPENCL_RUNTIME_H_ - -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "mediapipe/calculators/tensor/inference_io_mapper.h" -#include "mediapipe/calculators/tensor/inference_runner.h" -#include "mediapipe/calculators/tensor/tensor_span.h" -#include "mediapipe/framework/api2/packet.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/gpu/gl_base.h" -#include "mediapipe/util/tflite/tflite_model_loader.h" -#include "tensorflow/lite/core/api/op_resolver.h" -#include "tensorflow/lite/model_builder.h" -#include "third_party/ml_drift/cl/api.h" -#include "third_party/ml_drift/common/model.h" -#include "third_party/ml_drift/common/shape.h" -#include "third_party/ml_drift/contrib/tflite_op_resolver.h" -#include "third_party/ml_drift/delegate/api_cl_gl_vk.h" - -namespace mediapipe::api2 { - -// Inference runner implementation that uses the ML Drift OpenCL runtime with -// GPU bindings. -class InferenceRunnerMlDriftOpenClRuntime : public InferenceRunner { - public: - ~InferenceRunnerMlDriftOpenClRuntime() override = default; - - absl::Status Init( - const mediapipe::InferenceCalculatorOptions& options, - Packet model_packet, - Packet op_resolver_packet); - - // This method must be executed on current OpenGL context / thread. - absl::StatusOr> Run( - CalculatorContext* cc, const TensorSpan& input_tensors) override; - - const InputOutputTensorNames& GetInputOutputTensorNames() const override; - - private: - absl::Status InitTFLiteGpuRunner( - CalculatorContext* cc, - const mediapipe::InferenceCalculatorOptions::Delegate& delegate); - - absl::StatusOr InitModelFromFlatBuffer( - const tflite::FlatBufferModel& flatbuffer, - const tflite::OpResolver& op_resolver, bool allow_quant_ops = false); - - absl::Status BindSsboToInputTensor(GLuint ssbo_id, int input_id); - absl::Status BindSsboToOutputTensor(GLuint ssbo_id, int output_id); - - absl::Status InitializeMlDriftRuntime( - ml_drift::GraphFloat32&& graph_cl, - const ml_drift::cl::InferenceOptions& options); - - InputOutputTensorNames input_output_tensor_names_; - - std::unique_ptr cl_environment_; - - std::unique_ptr runner_; - - std::vector input_shapes_; - std::vector output_shapes_; - std::vector tensor_output_shapes_; -}; - -}; // namespace mediapipe::api2 - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_RUNNER_ML_DRIFT_OPENCL_RUNTIME_H_