Internal change

PiperOrigin-RevId: 572125477

mediapipe/tasks/cc/vision/utils/ghum/BUILD

@@ -0,0 +1,61 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+cc_library(
+    name = "ghum_hand_topology",
+    hdrs = ["ghum_hand_topology.h"],
+)
+
+cc_library(
+    name = "ghum_hand_utils",
+    srcs = ["ghum_hand_utils.cc"],
+    hdrs = ["ghum_hand_utils.h"],
+    deps = [
+        ":ghum_hand_topology",
+        "//mediapipe/calculators/util:set_joints_visibility_calculator",
+        "//mediapipe/calculators/util:set_joints_visibility_calculator_cc_proto",
+        "//mediapipe/framework/api2:builder",
+        "//mediapipe/framework/formats:body_rig_cc_proto",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/tasks/cc/vision/hand_landmarker:hand_topology",
+        "@com_google_absl//absl/log:absl_check",
+    ],
+)
+
+cc_library(
+    name = "ghum_topology",
+    hdrs = ["ghum_topology.h"],
+)
+
+cc_library(
+    name = "ghum_utils",
+    srcs = ["ghum_utils.cc"],
+    hdrs = ["ghum_utils.h"],
+    deps = [
+        ":ghum_topology",
+        "//mediapipe/calculators/util:combine_joints_calculator",
+        "//mediapipe/calculators/util:combine_joints_calculator_cc_proto",
+        "//mediapipe/calculators/util:set_joints_visibility_calculator",
+        "//mediapipe/calculators/util:set_joints_visibility_calculator_cc_proto",
+        "//mediapipe/framework/api2:builder",
+        "//mediapipe/framework/formats:body_rig_cc_proto",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/tasks/cc/vision/pose_landmarker:pose_topology",
+        "@com_google_absl//absl/types:span",
+    ],
+)

mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_topology.h

@@ -0,0 +1,46 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Defines topology of the GHUM Hand model. This is related only to the 3D model of
+the hand, not the way NN model predicts joint rotations for it.
+==============================================================================*/
+
+#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_TOPOLOGY_H_
+#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_TOPOLOGY_H_
+
+namespace mediapipe::tasks::vision::utils::ghum {
+
+// GHUM hand 16 joint names in order they are produced by the HandRig pipeline.
+enum class GhumHandJointName {
+  kHand = 0,
+  kIndex01,
+  kIndex02,
+  kIndex03,
+  kMiddle01,
+  kMiddle02,
+  kMiddle03,
+  kRing01,
+  kRing02,
+  kRing03,
+  kPinky01,
+  kPinky02,
+  kPinky03,
+  kThumb01,
+  kThumb02,
+  kThumb03,
+};
+
+}  // namespace mediapipe::tasks::vision::utils::ghum
+
+#endif  // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_TOPOLOGY_H_

mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.cc

@@ -0,0 +1,80 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.h"
+
+#include <array>
+#include <utility>
+
+#include "absl/log/absl_check.h"
+#include "mediapipe/calculators/util/set_joints_visibility_calculator.h"
+#include "mediapipe/calculators/util/set_joints_visibility_calculator.pb.h"
+#include "mediapipe/framework/api2/builder.h"
+#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_topology.h"
+#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_topology.h"
+
+namespace mediapipe::tasks::vision::utils::ghum {
+
+namespace {
+
+using ::mediapipe::api2::SetJointsVisibilityCalculator;
+using ::mediapipe::api2::builder::Stream;
+using ::mediapipe::tasks::vision::hand_landmarker::HandLandmarkName;
+
+// Includes mapping for all 16 GHUM Hand joints.
+constexpr std::array<std::pair<GhumHandJointName, HandLandmarkName>, 16>
+    kGhumHandJointsToHandLandmarksMapping = {{
+        {GhumHandJointName::kHand, HandLandmarkName::kWrist},
+        {GhumHandJointName::kIndex01, HandLandmarkName::kIndex1},
+        {GhumHandJointName::kIndex02, HandLandmarkName::kIndex2},
+        {GhumHandJointName::kIndex03, HandLandmarkName::kIndex3},
+        {GhumHandJointName::kMiddle01, HandLandmarkName::kMiddle1},
+        {GhumHandJointName::kMiddle02, HandLandmarkName::kMiddle2},
+        {GhumHandJointName::kMiddle03, HandLandmarkName::kMiddle3},
+        {GhumHandJointName::kRing01, HandLandmarkName::kRing1},
+        {GhumHandJointName::kRing02, HandLandmarkName::kRing2},
+        {GhumHandJointName::kRing03, HandLandmarkName::kRing3},
+        {GhumHandJointName::kPinky01, HandLandmarkName::kPinky1},
+        {GhumHandJointName::kPinky02, HandLandmarkName::kPinky2},
+        {GhumHandJointName::kPinky03, HandLandmarkName::kPinky3},
+        {GhumHandJointName::kThumb01, HandLandmarkName::kThumb1},
+        {GhumHandJointName::kThumb02, HandLandmarkName::kThumb2},
+        {GhumHandJointName::kThumb03, HandLandmarkName::kThumb3},
+    }};
+
+}  // namespace
+
+Stream<JointList> SetGhumHandJointsVisibilityFromWorldLandmarks(
+    Stream<JointList> ghum_hand_joints,
+    Stream<LandmarkList> hand_world_landmarks,
+    mediapipe::api2::builder::Graph& graph) {
+  auto& node = graph.AddNode("SetJointsVisibilityCalculator");
+  auto& opts = node.GetOptions<SetJointsVisibilityCalculatorOptions>();
+  for (const auto& pair : kGhumHandJointsToHandLandmarksMapping) {
+    // Sanity check to verify that all hand joints are set and are set in the
+    // right order.
+    ABSL_CHECK_EQ(static_cast<int>(pair.first), opts.mapping_size());
+    auto* mapping = opts.add_mapping();
+    mapping->mutable_copy()->set_idx(static_cast<int>(pair.second));
+  }
+
+  ghum_hand_joints.ConnectTo(node[SetJointsVisibilityCalculator::kInJoints]);
+  hand_world_landmarks.ConnectTo(
+      node[SetJointsVisibilityCalculator::kInLandmarks]);
+
+  return node[SetJointsVisibilityCalculator::kOutJoints];
+}
+
+}  // namespace mediapipe::tasks::vision::utils::ghum

mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.h

@@ -0,0 +1,38 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Utility methods for populating Ghum Hand joints from joints produced by Hund
+Hand model and hand landmarks.
+==============================================================================*/
+
+#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_UTILS_H_
+#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_UTILS_H_
+
+#include "mediapipe/framework/api2/builder.h"
+#include "mediapipe/framework/formats/body_rig.pb.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe::tasks::vision::utils::ghum {
+
+// Sets visibility of 16 GHUM hand joints from 21 hand world landmarks.
+mediapipe::api2::builder::Stream<mediapipe::JointList>
+SetGhumHandJointsVisibilityFromWorldLandmarks(
+    mediapipe::api2::builder::Stream<mediapipe::JointList> ghum_hand_joints,
+    mediapipe::api2::builder::Stream<mediapipe::LandmarkList>
+        hand_world_landmarks,
+    mediapipe::api2::builder::Graph& graph);
+
+}  // namespace mediapipe::tasks::vision::utils::ghum
+
+#endif  // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_UTILS_H_