diff --git a/mediapipe/tasks/cc/vision/utils/ghum/BUILD b/mediapipe/tasks/cc/vision/utils/ghum/BUILD new file mode 100644 index 0000000000..1c74e1a130 --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/ghum/BUILD @@ -0,0 +1,61 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "ghum_hand_topology", + hdrs = ["ghum_hand_topology.h"], +) + +cc_library( + name = "ghum_hand_utils", + srcs = ["ghum_hand_utils.cc"], + hdrs = ["ghum_hand_utils.h"], + deps = [ + ":ghum_hand_topology", + "//mediapipe/calculators/util:set_joints_visibility_calculator", + "//mediapipe/calculators/util:set_joints_visibility_calculator_cc_proto", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/formats:body_rig_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/tasks/cc/vision/hand_landmarker:hand_topology", + "@com_google_absl//absl/log:absl_check", + ], +) + +cc_library( + name = "ghum_topology", + hdrs = ["ghum_topology.h"], +) + +cc_library( + name = "ghum_utils", + srcs = ["ghum_utils.cc"], + hdrs = ["ghum_utils.h"], + deps = [ + ":ghum_topology", + "//mediapipe/calculators/util:combine_joints_calculator", + "//mediapipe/calculators/util:combine_joints_calculator_cc_proto", + "//mediapipe/calculators/util:set_joints_visibility_calculator", + "//mediapipe/calculators/util:set_joints_visibility_calculator_cc_proto", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/formats:body_rig_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/tasks/cc/vision/pose_landmarker:pose_topology", + "@com_google_absl//absl/types:span", + ], +) diff --git a/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_topology.h b/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_topology.h new file mode 100644 index 0000000000..3b26d41534 --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_topology.h @@ -0,0 +1,46 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Defines topology of the GHUM Hand model. This is related only to the 3D model of +the hand, not the way NN model predicts joint rotations for it. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_TOPOLOGY_H_ +#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_TOPOLOGY_H_ + +namespace mediapipe::tasks::vision::utils::ghum { + +// GHUM hand 16 joint names in order they are produced by the HandRig pipeline. +enum class GhumHandJointName { + kHand = 0, + kIndex01, + kIndex02, + kIndex03, + kMiddle01, + kMiddle02, + kMiddle03, + kRing01, + kRing02, + kRing03, + kPinky01, + kPinky02, + kPinky03, + kThumb01, + kThumb02, + kThumb03, +}; + +} // namespace mediapipe::tasks::vision::utils::ghum + +#endif // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_TOPOLOGY_H_ diff --git a/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.cc b/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.cc new file mode 100644 index 0000000000..fbdeb4c4f6 --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.cc @@ -0,0 +1,80 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.h" + +#include +#include + +#include "absl/log/absl_check.h" +#include "mediapipe/calculators/util/set_joints_visibility_calculator.h" +#include "mediapipe/calculators/util/set_joints_visibility_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_topology.h" +#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_topology.h" + +namespace mediapipe::tasks::vision::utils::ghum { + +namespace { + +using ::mediapipe::api2::SetJointsVisibilityCalculator; +using ::mediapipe::api2::builder::Stream; +using ::mediapipe::tasks::vision::hand_landmarker::HandLandmarkName; + +// Includes mapping for all 16 GHUM Hand joints. +constexpr std::array, 16> + kGhumHandJointsToHandLandmarksMapping = {{ + {GhumHandJointName::kHand, HandLandmarkName::kWrist}, + {GhumHandJointName::kIndex01, HandLandmarkName::kIndex1}, + {GhumHandJointName::kIndex02, HandLandmarkName::kIndex2}, + {GhumHandJointName::kIndex03, HandLandmarkName::kIndex3}, + {GhumHandJointName::kMiddle01, HandLandmarkName::kMiddle1}, + {GhumHandJointName::kMiddle02, HandLandmarkName::kMiddle2}, + {GhumHandJointName::kMiddle03, HandLandmarkName::kMiddle3}, + {GhumHandJointName::kRing01, HandLandmarkName::kRing1}, + {GhumHandJointName::kRing02, HandLandmarkName::kRing2}, + {GhumHandJointName::kRing03, HandLandmarkName::kRing3}, + {GhumHandJointName::kPinky01, HandLandmarkName::kPinky1}, + {GhumHandJointName::kPinky02, HandLandmarkName::kPinky2}, + {GhumHandJointName::kPinky03, HandLandmarkName::kPinky3}, + {GhumHandJointName::kThumb01, HandLandmarkName::kThumb1}, + {GhumHandJointName::kThumb02, HandLandmarkName::kThumb2}, + {GhumHandJointName::kThumb03, HandLandmarkName::kThumb3}, + }}; + +} // namespace + +Stream SetGhumHandJointsVisibilityFromWorldLandmarks( + Stream ghum_hand_joints, + Stream hand_world_landmarks, + mediapipe::api2::builder::Graph& graph) { + auto& node = graph.AddNode("SetJointsVisibilityCalculator"); + auto& opts = node.GetOptions(); + for (const auto& pair : kGhumHandJointsToHandLandmarksMapping) { + // Sanity check to verify that all hand joints are set and are set in the + // right order. + ABSL_CHECK_EQ(static_cast(pair.first), opts.mapping_size()); + auto* mapping = opts.add_mapping(); + mapping->mutable_copy()->set_idx(static_cast(pair.second)); + } + + ghum_hand_joints.ConnectTo(node[SetJointsVisibilityCalculator::kInJoints]); + hand_world_landmarks.ConnectTo( + node[SetJointsVisibilityCalculator::kInLandmarks]); + + return node[SetJointsVisibilityCalculator::kOutJoints]; +} + +} // namespace mediapipe::tasks::vision::utils::ghum diff --git a/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.h b/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.h new file mode 100644 index 0000000000..1aba4b6291 --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.h @@ -0,0 +1,38 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Utility methods for populating Ghum Hand joints from joints produced by Hund +Hand model and hand landmarks. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_UTILS_H_ +#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_UTILS_H_ + +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/body_rig.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe::tasks::vision::utils::ghum { + +// Sets visibility of 16 GHUM hand joints from 21 hand world landmarks. +mediapipe::api2::builder::Stream +SetGhumHandJointsVisibilityFromWorldLandmarks( + mediapipe::api2::builder::Stream ghum_hand_joints, + mediapipe::api2::builder::Stream + hand_world_landmarks, + mediapipe::api2::builder::Graph& graph); + +} // namespace mediapipe::tasks::vision::utils::ghum + +#endif // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_UTILS_H_ diff --git a/mediapipe/tasks/cc/vision/utils/ghum/ghum_topology.h b/mediapipe/tasks/cc/vision/utils/ghum/ghum_topology.h new file mode 100644 index 0000000000..fde17a0a02 --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_topology.h @@ -0,0 +1,230 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Defines topology of the GHUM model. This is related only to the 3D model of the +body, not the way NN models predict joint rotations for it. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_TOPOLOGY_H_ +#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_TOPOLOGY_H_ + +#include + +namespace mediapipe::tasks::vision::utils::ghum { + +// GHUM 63 joint names in order they are produced by the PoseRig and +// XenomorphRig pipelines. +// +// Note that some joints will always remain in default rotation (i.e. +// kGhumDefaultJointRotation) depending on what joints are supported by the +// pipeline. +enum class GhumJointName { + kPelvis = 0, + kSpine01, + kSpine02, + kSpine03, + kNeck01, + kHead01, + kJaw01, + kEye01L, + kEyelidup01L, + kEye01R, + kEyelidup01R, + kEyeliddown01L, + kEyeliddown01R, + kClavicleL, + kUpperarmL, + kLowerarmL, + kHandL, + kIndex01L, + kIndex02L, + kIndex03L, + kMiddle01L, + kMiddle02L, + kMiddle03L, + kRing01L, + kRing02L, + kRing03L, + kPinky01L, + kPinky02L, + kPinky03L, + kThumb01L, + kThumb02L, + kThumb03L, + kClavicleR, + kUpperarmR, + kLowerarmR, + kHandR, + kIndex01R, + kIndex02R, + kIndex03R, + kMiddle01R, + kMiddle02R, + kMiddle03R, + kRing01R, + kRing02R, + kRing03R, + kPinky01R, + kPinky02R, + kPinky03R, + kThumb01R, + kThumb02R, + kThumb03R, + kThighL, + kCalfL, + kAnkleL, + kFootL, + kBallL, + kToes01L, + kThighR, + kCalfR, + kAnkleR, + kFootR, + kBallR, + kToes01R, +}; + +// GHUM default joint rotation. When applied - won't change initial pose of 3D +// model. +constexpr std::array kGhumDefaultJointRotation = {{1, 0, 0, 1, 0, 0}}; + +// GHUM resting joint positions defined in 6D rotation format. Joint rotations +// produced by Hund models should be applied ON TOP OF THEM (i.e. multiplied). +// +constexpr std::array, 63> kGhumRestingJointRotations = {{ + {3.3306690738754696e-16, -1.1102230246251565e-16, 0.9907665252685547, + 0.1355789452791214, -0.1355789452791214, 0.9907665252685547}, + {-2.220446049250313e-16, -2.220446049250313e-16, 0.9965463876724243, + -0.08303803205490112, 0.08303803205490112, 0.9965463876724243}, + {2.220446049250313e-16, -1.1102230246251565e-16, 0.9830689430236816, + 0.1832359880208969, -0.1832359880208969, 0.9830689430236816}, + {0.0, -1.1102230246251565e-16, 0.9998827576637268, -0.015312896110117435, + 0.015312896110117435, 0.9998827576637268}, + {1.1102230246251565e-16, -1.1102230246251565e-16, 0.9737479090690613, + -0.22762899100780487, 0.22762899100780487, 0.9737479090690613}, + {0.0, -2.220446049250313e-16, 0.9975937008857727, -0.06933078169822693, + 0.06933078169822693, 0.9975937008857727}, + {0.0, 0.0, -0.7646467089653015, 0.6444497108459473, 0.6444497108459473, + 0.7646467089653015}, + {1.0, 0.0, 0.0, 1.0, 0.0, 0.0}, + {1.0, 0.0, 0.0, 1.0, 0.0, 0.0}, + {1.0, 0.0, 0.0, -1.0, 0.0, -1.2246468525851679e-16}, + {1.0, 0.0, 0.0, -1.0, 0.0, -1.2246468525851679e-16}, + {1.0, 0.0, 0.0, 1.0, 0.0, 0.0}, + {1.0, 0.0, 0.0, -1.0, 0.0, -1.2246468525851679e-16}, + {0.8921475410461426, 0.2971198558807373, -0.17762203514575958, + 0.9233117699623108, -0.41535913944244385, 0.24334177374839783}, + {0.4769144654273987, 0.87869793176651, -0.878876805305481, + 0.4764299690723419, -0.011322125792503357, 0.03007427789270878}, + {0.5015625953674316, 0.865040123462677, -0.8650957345962524, + 0.5016061067581177, 0.00665692426264286, 0.009843245148658752}, + {0.5015625953674316, 0.865040123462677, -0.8650957345962524, + 0.5016061067581177, 0.00665692426264286, 0.009843245148658752}, + {0.37746769189834595, 0.8884291648864746, -0.9007542133331299, + 0.4176981747150421, 0.21484869718551636, 0.1903204619884491}, + {0.37291213870048523, 0.9067361354827881, -0.9177148342132568, + 0.39173614978790283, 0.13687947392463684, 0.1561165302991867}, + {0.37291213870048523, 0.9067361354827881, -0.9177148342132568, + 0.39173614978790283, 0.13687947392463684, 0.1561165302991867}, + {0.46410179138183594, 0.8850933313369751, -0.8857383131980896, + 0.4641091823577881, -0.008784775622189045, -0.03482021763920784}, + {0.4025099277496338, 0.9154127836227417, -0.9146613478660583, + 0.4022779166698456, 0.03715258091688156, -0.013853170908987522}, + {0.4025099277496338, 0.9154127836227417, -0.9146613478660583, + 0.4022779166698456, 0.03715258091688156, -0.013853170908987522}, + {0.34835490584373474, 0.8863212466239929, -0.9299189448356628, + 0.36770808696746826, -0.11789664626121521, -0.28147009015083313}, + {0.35047438740730286, 0.8873153328895569, -0.9331605434417725, + 0.35812410712242126, -0.07986952364444733, -0.2905488610267639}, + {0.35047438740730286, 0.8873153328895569, -0.9331605434417725, + 0.35812410712242126, -0.07986952364444733, -0.2905488610267639}, + {0.29830706119537354, 0.7789736986160278, -0.9084595441818237, + 0.4089738428592682, -0.2927698493003845, -0.4753318727016449}, + {0.2655753493309021, 0.7524029612541199, -0.9429818987846375, + 0.33284398913383484, -0.2006361186504364, -0.5684230327606201}, + {0.2655753493309021, 0.7524029612541199, -0.9429818987846375, + 0.33284398913383484, -0.2006361186504364, -0.5684230327606201}, + {-0.1977168470621109, -0.2275119423866272, -0.7548012137413025, + 0.6559536457061768, 0.6254464387893677, 0.719696581363678}, + {-0.07406296581029892, -0.2195579558610916, -0.947542130947113, + 0.3196309804916382, 0.31093189120292664, 0.9217539429664612}, + {-0.07406296581029892, -0.2195579558610916, -0.947542130947113, + 0.3196309804916382, 0.31093189120292664, 0.9217539429664612}, + {0.8921475410461426, 0.2971198558807373, 0.17762203514575958, + -0.9233117699623108, 0.41535913944244385, -0.24334177374839783}, + {0.4769144654273987, 0.87869793176651, 0.878876805305481, + -0.4764299690723419, 0.011322125792503357, -0.03007427789270878}, + {0.5015625953674316, 0.865040123462677, 0.8650957345962524, + -0.5016061067581177, -0.00665692426264286, -0.009843245148658752}, + {0.5015625953674316, 0.865040123462677, 0.8650957345962524, + -0.5016061067581177, -0.00665692426264286, -0.009843245148658752}, + {0.37746769189834595, 0.8884291648864746, 0.9007542133331299, + -0.4176981747150421, -0.21484869718551636, -0.1903204619884491}, + {0.37291213870048523, 0.9067361354827881, 0.9177148342132568, + -0.39173614978790283, -0.13687947392463684, -0.1561165302991867}, + {0.37291213870048523, 0.9067361354827881, 0.9177148342132568, + -0.39173614978790283, -0.13687947392463684, -0.1561165302991867}, + {0.46410179138183594, 0.8850933313369751, 0.8857383131980896, + -0.4641091823577881, 0.008784775622189045, 0.03482021763920784}, + {0.4025099277496338, 0.9154127836227417, 0.9146613478660583, + -0.4022779166698456, -0.03715258091688156, 0.013853170908987522}, + {0.4025099277496338, 0.9154127836227417, 0.9146613478660583, + -0.4022779166698456, -0.03715258091688156, 0.013853170908987522}, + {0.34835490584373474, 0.8863212466239929, 0.9299189448356628, + -0.36770808696746826, 0.11789664626121521, 0.28147009015083313}, + {0.35047438740730286, 0.8873153328895569, 0.9331605434417725, + -0.35812410712242126, 0.07986952364444733, 0.2905488610267639}, + {0.35047438740730286, 0.8873153328895569, 0.9331605434417725, + -0.35812410712242126, 0.07986952364444733, 0.2905488610267639}, + {0.29830706119537354, 0.7789736986160278, 0.9084595441818237, + -0.4089738428592682, 0.2927698493003845, 0.4753318727016449}, + {0.2655753493309021, 0.7524029612541199, 0.9429818987846375, + -0.33284398913383484, 0.2006361186504364, 0.5684230327606201}, + {0.2655753493309021, 0.7524029612541199, 0.9429818987846375, + -0.33284398913383484, 0.2006361186504364, 0.5684230327606201}, + {-0.1977168470621109, -0.2275119423866272, 0.7548012137413025, + -0.6559536457061768, -0.6254464387893677, -0.719696581363678}, + {-0.07406296581029892, -0.2195579558610916, 0.947542130947113, + -0.3196309804916382, -0.31093189120292664, -0.9217539429664612}, + {-0.07406295835971832, -0.2195579558610916, 0.947542130947113, + -0.3196309804916382, -0.31093189120292664, -0.9217539429664612}, + {-0.01636510342359543, 0.01171059813350439, -0.9988228678703308, + 0.045474592596292496, 0.04566308110952377, 0.9988968372344971}, + {0.05019572749733925, 0.024085810407996178, -0.9970070123672485, + -0.05764812231063843, -0.05879946053028107, 0.9980463981628418}, + {-0.012483594939112663, -0.038166593760252, -0.9944827556610107, + 0.104560986161232, 0.10415486991405487, 0.9937858581542969}, + {3.3306690738754696e-16, 2.7755575615628914e-17, -0.30688488483428955, + 0.9517466425895691, 0.9517466425895691, 0.30688488483428955}, + {-0.06284070760011673, -0.009795981459319592, -0.15403378009796143, + 0.9880656003952026, 0.9860652089118958, 0.15372197329998016}, + {2.220446049250313e-16, 6.938893903907228e-18, -0.0882575735449791, + 0.9960976839065552, 0.9960976839065552, 0.0882575735449791}, + {-0.01636510342359543, 0.01171059813350439, 0.9988228678703308, + -0.045474592596292496, -0.04566308110952377, -0.9988968372344971}, + {0.05019572749733925, 0.024085810407996178, 0.9970070123672485, + 0.05764812231063843, 0.05879946053028107, -0.9980463981628418}, + {-0.012483594939112663, -0.038166593760252, 0.9944827556610107, + -0.104560986161232, -0.10415486991405487, -0.9937858581542969}, + {3.3306690738754696e-16, -8.326672684688674e-17, 0.30688488483428955, + -0.9517466425895691, -0.9517466425895691, -0.30688488483428955}, + {-0.06284070760011673, -0.009795981459319592, 0.15403378009796143, + -0.9880656003952026, -0.9860652089118958, -0.15372197329998016}, + {2.220446049250313e-16, -1.1796119636642288e-16, 0.0882575735449791, + -0.9960976839065552, -0.9960976839065552, -0.0882575735449791}, +}}; + +} // namespace mediapipe::tasks::vision::utils::ghum + +#endif // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_TOPOLOGY_H_ diff --git a/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.cc b/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.cc new file mode 100644 index 0000000000..15d57f26c7 --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.cc @@ -0,0 +1,184 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.h" + +#include +#include +#include + +#include "absl/types/span.h" +#include "mediapipe/calculators/util/combine_joints_calculator.h" +#include "mediapipe/calculators/util/combine_joints_calculator.pb.h" +#include "mediapipe/calculators/util/set_joints_visibility_calculator.h" +#include "mediapipe/calculators/util/set_joints_visibility_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/pose_topology.h" +#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_topology.h" + +namespace mediapipe::tasks::vision::utils::ghum { + +namespace { + +using ::mediapipe::api2::CombineJointsCalculator; +using ::mediapipe::api2::SetJointsVisibilityCalculator; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::SidePacket; +using ::mediapipe::api2::builder::Stream; +using ::mediapipe::tasks::vision::pose_landmarker::PoseLandmarkName; + +} // namespace + +Stream SetGhumJointsFromHundJoints( + std::vector& mappings, Graph& graph) { + auto& to_ghum = graph.AddNode("CombineJointsCalculator"); + auto& to_ghum_options = to_ghum.GetOptions(); + to_ghum_options.set_num_joints(63); + + // Default joint values for not mapped joints. + for (float v : kGhumDefaultJointRotation) { + to_ghum_options.mutable_default_joint()->add_rotation_6d(v); + } + to_ghum_options.mutable_default_joint()->set_visibility(1.0); + + for (int i = 0; i < mappings.size(); ++i) { + auto* subset = to_ghum_options.add_joints_mapping(); + for (const auto& joint_name : mappings[i].ghum_joints_order) { + subset->add_idx(static_cast(joint_name)); + } + mappings[i].hund_joints.ConnectTo( + to_ghum[CombineJointsCalculator::kInJoints][i]); + } + + return to_ghum[CombineJointsCalculator::kOutJoints]; +} + +Stream SetGhumJointsVisibilityFromWorldLandmarks( + Stream ghum_joints, Stream pose_world_landmarks, + Graph& graph) { + auto& set_visibility = graph.AddNode("SetJointsVisibilityCalculator"); + auto& set_visibility_options = + set_visibility.GetOptions(); + std::vector>> mapping = + { + {GhumJointName::kPelvis, + {PoseLandmarkName::kLeftHip, PoseLandmarkName::kRightHip}}, + {GhumJointName::kSpine01, + {PoseLandmarkName::kLeftHip, PoseLandmarkName::kRightHip}}, + {GhumJointName::kSpine02, + {PoseLandmarkName::kLeftHip, PoseLandmarkName::kRightHip}}, + {GhumJointName::kSpine03, + {PoseLandmarkName::kLeftShoulder, PoseLandmarkName::kRightShoulder}}, + {GhumJointName::kNeck01, + {PoseLandmarkName::kLeftShoulder, PoseLandmarkName::kRightShoulder}}, + {GhumJointName::kHead01, + {PoseLandmarkName::kLeftShoulder, PoseLandmarkName::kRightShoulder}}, + {GhumJointName::kJaw01, + {PoseLandmarkName::kMouthLeft, PoseLandmarkName::kMouthRight}}, + {GhumJointName::kEye01L, {PoseLandmarkName::kLeftEye}}, + {GhumJointName::kEyelidup01L, {PoseLandmarkName::kLeftEye}}, + {GhumJointName::kEye01R, {PoseLandmarkName::kRightEye}}, + {GhumJointName::kEyelidup01R, {PoseLandmarkName::kRightEye}}, + {GhumJointName::kEyeliddown01L, {PoseLandmarkName::kLeftEye}}, + {GhumJointName::kEyeliddown01R, {PoseLandmarkName::kRightEye}}, + {GhumJointName::kClavicleL, {PoseLandmarkName::kLeftShoulder}}, + {GhumJointName::kUpperarmL, {PoseLandmarkName::kLeftShoulder}}, + {GhumJointName::kLowerarmL, {PoseLandmarkName::kLeftElbow}}, + {GhumJointName::kHandL, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kIndex01L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kIndex02L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kIndex03L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kMiddle01L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kMiddle02L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kMiddle03L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kRing01L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kRing02L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kRing03L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kPinky01L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kPinky02L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kPinky03L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kThumb01L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kThumb02L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kThumb03L, {PoseLandmarkName::kLeftWrist}}, + {GhumJointName::kClavicleR, {PoseLandmarkName::kRightShoulder}}, + {GhumJointName::kUpperarmR, {PoseLandmarkName::kRightShoulder}}, + {GhumJointName::kLowerarmR, {PoseLandmarkName::kRightElbow}}, + {GhumJointName::kHandR, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kIndex01R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kIndex02R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kIndex03R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kMiddle01R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kMiddle02R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kMiddle03R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kRing01R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kRing02R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kRing03R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kPinky01R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kPinky02R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kPinky03R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kThumb01R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kThumb02R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kThumb03R, {PoseLandmarkName::kRightWrist}}, + {GhumJointName::kThighL, {PoseLandmarkName::kLeftHip}}, + {GhumJointName::kCalfL, {PoseLandmarkName::kLeftKnee}}, + {GhumJointName::kAnkleL, {PoseLandmarkName::kLeftAnkle}}, + {GhumJointName::kFootL, {PoseLandmarkName::kLeftAnkle}}, + {GhumJointName::kBallL, {PoseLandmarkName::kLeftHeel}}, + {GhumJointName::kToes01L, {PoseLandmarkName::kLeftFootIndex}}, + {GhumJointName::kThighR, {PoseLandmarkName::kRightHip}}, + {GhumJointName::kCalfR, {PoseLandmarkName::kRightKnee}}, + {GhumJointName::kAnkleR, {PoseLandmarkName::kRightAnkle}}, + {GhumJointName::kFootR, {PoseLandmarkName::kRightAnkle}}, + {GhumJointName::kBallR, {PoseLandmarkName::kRightHeel}}, + {GhumJointName::kToes01R, {PoseLandmarkName::kRightFootIndex}}, + }; + for (const auto& pair : mapping) { + const auto& joint_name = pair.first; + const auto& landmark_names = pair.second; + + CHECK_EQ(static_cast(joint_name), + set_visibility_options.mapping_size()); + auto* mapping = set_visibility_options.add_mapping(); + + if (landmark_names.size() == 1) { + mapping->mutable_copy()->set_idx(static_cast(landmark_names[0])); + } else { + for (const auto& landmark_name : landmark_names) { + mapping->mutable_highest()->add_idx(static_cast(landmark_name)); + } + } + } + + ghum_joints.ConnectTo( + set_visibility[SetJointsVisibilityCalculator::kInJoints]); + pose_world_landmarks.ConnectTo( + set_visibility[SetJointsVisibilityCalculator::kInLandmarks]); + + return set_visibility[SetJointsVisibilityCalculator::kOutJoints]; +} + +std::vector> GetGhumRestingJointRotationsSubset( + absl::Span ghum_joint_names) { + std::vector> res; + res.reserve(ghum_joint_names.size()); + for (GhumJointName ghum_joint_name : ghum_joint_names) { + res.push_back( + kGhumRestingJointRotations[static_cast(ghum_joint_name)]); + } + return res; +} + +} // namespace mediapipe::tasks::vision::utils::ghum diff --git a/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.h b/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.h new file mode 100644 index 0000000000..aa8f6d8b46 --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.h @@ -0,0 +1,67 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Utility methods for populating Ghum joints from joints produced by Hund Hand +Pose and Hund Hand models and pose landmarks. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_UTILS_H_ +#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_UTILS_H_ + +#include +#include + +#include "absl/types/span.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/body_rig.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_topology.h" + +namespace mediapipe::tasks::vision::utils::ghum { + +struct HundToGhumJointsMapping { + // Joints produced by the HUND models. + mediapipe::api2::builder::Stream hund_joints; + // Order of joints in GHUM topology. + absl::Span ghum_joints_order; +}; + +// Sets GHUM joints from given HUND joints according to the mappings and in +// specified order. +// +// All joints that are not defined will remain in kGhumDefaultJointRotation +// and with 1.0 visibility. +// +// All joints specified later in order will override those that were specified +// earlier. +mediapipe::api2::builder::Stream +SetGhumJointsFromHundJoints( + std::vector& hund_to_ghum_joints_mappings, + mediapipe::api2::builder::Graph& graph); + +// Sets visibility of 63 GHUM joints from 33 pose world landmarks. +mediapipe::api2::builder::Stream +SetGhumJointsVisibilityFromWorldLandmarks( + mediapipe::api2::builder::Stream ghum_joints, + mediapipe::api2::builder::Stream + pose_world_landmarks, + mediapipe::api2::builder::Graph& graph); + +// Gets a subset of kGhumRestingJointRotations in 6D format. +std::vector> GetGhumRestingJointRotationsSubset( + absl::Span ghum_joint_names); + +} // namespace mediapipe::tasks::vision::utils::ghum + +#endif // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_UTILS_H_