diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index 9b178a3e1f..375e3806ea 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -35,6 +35,32 @@ cc_library( alwayslink = 1, ) +mediapipe_proto_library( + name = "align_hand_to_pose_in_world_calculator_proto", + srcs = ["align_hand_to_pose_in_world_calculator.proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], +) + +cc_library( + name = "align_hand_to_pose_in_world_calculator", + srcs = ["align_hand_to_pose_in_world_calculator.cc"], + hdrs = ["align_hand_to_pose_in_world_calculator.h"], + deps = [ + ":align_hand_to_pose_in_world_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/status", + ], + alwayslink = 1, +) + mediapipe_proto_library( name = "annotation_overlay_calculator_proto", srcs = ["annotation_overlay_calculator.proto"], diff --git a/mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.cc b/mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.cc new file mode 100644 index 0000000000..38cee59059 --- /dev/null +++ b/mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.cc @@ -0,0 +1,80 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h" + +#include + +#include "absl/status/status.h" +#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.pb.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/port/ret_check.h" + +namespace mediapipe::api2 { + +namespace {} // namespace + +class AlignHandToPoseInWorldCalculatorImpl + : public NodeImpl { + public: + absl::Status Open(CalculatorContext* cc) override { + const auto& options = + cc->Options(); + hand_wrist_idx_ = options.hand_wrist_idx(); + pose_wrist_idx_ = options.pose_wrist_idx(); + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext* cc) override { + // Check that landmarks are not empty. + if (kInHandLandmarks(cc).IsEmpty()) { + return absl::OkStatus(); + } + + // Check that pose landmarks are provided. + RET_CHECK(!kInPoseLandmarks(cc).IsEmpty()); + + const auto& in_hand_landmarks = kInHandLandmarks(cc).Get(); + const auto& in_pose_landmarks = kInPoseLandmarks(cc).Get(); + + // Get hand and pose wrists. + RET_CHECK(hand_wrist_idx_ <= in_hand_landmarks.landmark_size()); + const auto& hand_wrist = in_hand_landmarks.landmark(hand_wrist_idx_); + RET_CHECK(pose_wrist_idx_ <= in_pose_landmarks.landmark_size()); + const auto& pose_wrist = in_pose_landmarks.landmark(pose_wrist_idx_); + + LandmarkList out_hand_landmarks; + for (int i = 0; i < in_hand_landmarks.landmark_size(); ++i) { + const auto& in_landmark = in_hand_landmarks.landmark(i); + Landmark* out_landmark = out_hand_landmarks.add_landmark(); + *out_landmark = in_landmark; + out_landmark->set_x(in_landmark.x() - hand_wrist.x() + pose_wrist.x()); + out_landmark->set_y(in_landmark.y() - hand_wrist.y() + pose_wrist.y()); + out_landmark->set_z(in_landmark.z() - hand_wrist.z() + pose_wrist.z()); + } + + kOutHandLandmarks(cc).Send(std::move(out_hand_landmarks)); + + return absl::OkStatus(); + } + + private: + int hand_wrist_idx_; + int pose_wrist_idx_; +}; +MEDIAPIPE_NODE_IMPLEMENTATION(AlignHandToPoseInWorldCalculatorImpl); + +} // namespace mediapipe::api2 diff --git a/mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h b/mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h new file mode 100644 index 0000000000..b2393d1632 --- /dev/null +++ b/mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h @@ -0,0 +1,74 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_ +#define MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_ + +#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.pb.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe::api2 { + +// A calculator to align hand world landmarks with pose world landmarks. +// +// When `mediapipe.aimatter.LandmarksDetector` projects world landmarks from ROI +// local coordinates to original scene coordinates, it applies only rotation +// (derived from ROI) but neither scale nor translation. This calculator +// utilizes pose semantic to compensate this lack of information: +// - Translation is determined by matching wrist from hand landmarks with +// wrist from pose landmarks. +// - Scale can be determined (but is not at the moment) by calculating +// expected hand size from pose landmarks proportions. +// +// Input: +// HAND_LANDMARKS - LandmarkList +// Hand world landmarks. +// POSE_LANDMARKS - LandmarkList +// Pose world landmarks. +// +// Output: +// HAND_LANDMARKS - LandmarkList +// Aligned hand world landmarks. +// +// Example: +// node { +// calculator: "AlignHandToPoseInWorldCalculator" +// input_stream: "HAND_LANDMARKS:hand_world_landmarks" +// input_stream: "POSE_LANDMARKS:pose_world_landmarks" +// output_stream: "HAND_LANDMARKS:hand_world_landmarks" +// options: { +// [mediapipe.AlignHandToPoseInWorldCalculatorOptions.ext] { +// hand_wrist_idx: 0 +// pose_wrist_idx: 15 # 16 for right +// } +// } +// } +class AlignHandToPoseInWorldCalculator : public NodeIntf { + public: + static constexpr Input kInHandLandmarks{ + "HAND_LANDMARKS"}; + static constexpr Input kInPoseLandmarks{ + "POSE_LANDMARKS"}; + static constexpr Output kOutHandLandmarks{ + "HAND_LANDMARKS"}; + MEDIAPIPE_NODE_INTERFACE(::mediapipe::AlignHandToPoseInWorldCalculator, + kInHandLandmarks, kInPoseLandmarks, + kOutHandLandmarks); +}; + +} // namespace mediapipe::api2 + +#endif // MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.proto b/mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.proto new file mode 100644 index 0000000000..17b2a53bec --- /dev/null +++ b/mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.proto @@ -0,0 +1,31 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message AlignHandToPoseInWorldCalculatorOptions { + extend CalculatorOptions { + optional AlignHandToPoseInWorldCalculatorOptions ext = 403449802; + } + + // Hand wrist index. + optional int32 hand_wrist_idx = 1; + + // Pose wrist index. + optional int32 pose_wrist_idx = 2; +} diff --git a/mediapipe/framework/tool/mediapipe_proto_allowlist.bzl b/mediapipe/framework/tool/mediapipe_proto_allowlist.bzl index fbf2c8fe15..26c20ae8d0 100644 --- a/mediapipe/framework/tool/mediapipe_proto_allowlist.bzl +++ b/mediapipe/framework/tool/mediapipe_proto_allowlist.bzl @@ -5,7 +5,6 @@ rewrite_target_list = [ "adaptive_crop_calculator_proto", "affine_transform_data_proto", "affine_transform_from_rect_calculator_proto", - "align_hand_to_pose_in_world_calculator_proto", "anchor_proto", "annotation_overlay_calculator_proto", "annotation_proto", diff --git a/mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt b/mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt index 242a092c1e..10c567766b 100644 --- a/mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt +++ b/mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt @@ -165,12 +165,12 @@ node { } } node { - calculator: "mediapipe.aimatter.AlignHandToPoseInWorldCalculator" + calculator: "AlignHandToPoseInWorldCalculator" input_stream: "HAND_LANDMARKS:__stream_14" input_stream: "POSE_LANDMARKS:pose_world_landmarks_in" output_stream: "HAND_LANDMARKS:left_hand_world_landmarks_out" options { - [mediapipe.aimatter.AlignHandToPoseInWorldCalculatorOptions.ext] { + [mediapipe.AlignHandToPoseInWorldCalculatorOptions.ext] { hand_wrist_idx: 0 pose_wrist_idx: 15 }