build: mediapipe 0.8.6

- fix: SEGV when building TFLiteGPURunner - fix: pose tracking sample flickers
longgood · Jul 11, 2021 · e31b79d · e31b79d
1 parent 4ceaa57
commit e31b79d
Show file tree

Hide file tree

Showing 21 changed files with 97 additions and 63 deletions.
diff --git a/Assets/Mediapipe/Samples/Graphs/FaceDetection/Objects/FaceDetection.prefab b/Assets/Mediapipe/Samples/Graphs/FaceDetection/Objects/FaceDetection.prefab
@@ -12,7 +12,7 @@ GameObject:
   - component: {fileID: 5680418270291803700}
   - component: {fileID: 5680418270291803702}
   m_Layer: 0
-  m_Name: FaceDetectionDesktopGPU
+  m_Name: FaceDetection
   m_TagString: Untagged
   m_Icon: {fileID: 0}
   m_NavMeshLayer: 0

diff --git a/Assets/Mediapipe/Samples/Graphs/FaceDetection/Resources/face_detection_android.txt b/Assets/Mediapipe/Samples/Graphs/FaceDetection/Resources/face_detection_android.txt
@@ -51,9 +51,9 @@ node: {
   }
 }
 
-# Subgraph that detects faces.
+# Detects faces.
 node {
-  calculator: "FaceDetectionFrontGpu"
+  calculator: "FaceDetectionShortRangeGpu"
   input_stream: "IMAGE:transformed_input_video"
   output_stream: "DETECTIONS:face_detections"
 }

diff --git a/Assets/Mediapipe/Samples/Graphs/FaceDetection/Resources/face_detection_desktop_cpu.txt b/Assets/Mediapipe/Samples/Graphs/FaceDetection/Resources/face_detection_desktop_cpu.txt
@@ -58,9 +58,9 @@ node: {
   }
 }
 
-# Subgraph that detects faces.
+# Detects faces.
 node {
-  calculator: "FaceDetectionFrontCpu"
+  calculator: "FaceDetectionShortRangeCpu"
   input_stream: "IMAGE:transformed_input_video"
   output_stream: "DETECTIONS:face_detections"
 }

diff --git a/Assets/Mediapipe/Samples/Graphs/FaceDetection/Resources/face_detection_desktop_gpu.txt b/Assets/Mediapipe/Samples/Graphs/FaceDetection/Resources/face_detection_desktop_gpu.txt
@@ -65,9 +65,9 @@ node: {
   }
 }
 
-# Subgraph that detects faces.
+# Detects faces.
 node {
-  calculator: "FaceDetectionFrontGpu"
+  calculator: "FaceDetectionShortRangeGpu"
   input_stream: "IMAGE:transformed_input_video"
   output_stream: "DETECTIONS:face_detections"
 }

diff --git a/Assets/Mediapipe/Samples/Graphs/FaceDetection/Scripts/FaceDetectionGraph.cs b/Assets/Mediapipe/Samples/Graphs/FaceDetection/Scripts/FaceDetectionGraph.cs
@@ -41,6 +41,6 @@ private void RenderAnnotation(WebCamScreenController screenController, List<Dete
   }
 
   protected override void PrepareDependentAssets() {
-    PrepareDependentAsset("face_detection_front.bytes");
+    PrepareDependentAsset("face_detection_short_range.bytes");
   }
 }
diff --git a/Assets/Mediapipe/Samples/Graphs/FaceMesh/Scripts/FaceMeshGraph.cs b/Assets/Mediapipe/Samples/Graphs/FaceMesh/Scripts/FaceMeshGraph.cs
@@ -101,7 +101,7 @@ private void RenderAnnotation(WebCamScreenController screenController, FaceMeshV
   }
 
   protected override void PrepareDependentAssets() {
-    PrepareDependentAsset("face_detection_front.bytes");
+    PrepareDependentAsset("face_detection_short_range.bytes");
     PrepareDependentAsset("face_landmark.bytes");
   }
 }
diff --git a/Assets/Mediapipe/Samples/Graphs/Holistic/Scripts/HolisticGraph.cs b/Assets/Mediapipe/Samples/Graphs/Holistic/Scripts/HolisticGraph.cs
@@ -188,7 +188,7 @@ private void RenderAnnotation(WebCamScreenController screenController, HolisticV
   }
 
   protected override void PrepareDependentAssets() {
-    PrepareDependentAsset("face_detection_front.bytes");
+    PrepareDependentAsset("face_detection_short_range.bytes");
     PrepareDependentAsset("face_landmark.bytes");
     PrepareDependentAsset("iris_landmark.bytes");
     PrepareDependentAsset("hand_landmark.bytes");

diff --git a/Assets/Mediapipe/Samples/Graphs/IrisTracking/Scripts/IrisTrackingGraph.cs b/Assets/Mediapipe/Samples/Graphs/IrisTracking/Scripts/IrisTrackingGraph.cs
@@ -94,7 +94,7 @@ private void RenderAnnotation(WebCamScreenController screenController, IrisTrack
   }
 
   protected override void PrepareDependentAssets() {
-    PrepareDependentAsset("face_detection_front.bytes");
+    PrepareDependentAsset("face_detection_short_range.bytes");
     PrepareDependentAsset("face_landmark.bytes");
     PrepareDependentAsset("iris_landmark.bytes");
   }

diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_android.txt b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_android.txt
@@ -41,7 +41,7 @@ output_stream: "pose_detection_presence"
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"
-  input_stream: "FINISHED:pose_landmarks_presence"
+  input_stream: "FINISHED:pose_detection_presence"
   input_stream_info: {
     tag_index: "FINISHED"
     back_edge: true

diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_cpu.txt b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_cpu.txt
@@ -41,7 +41,7 @@ output_stream: "pose_detection_presence"
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"
-  input_stream: "FINISHED:pose_landmarks_presence"
+  input_stream: "FINISHED:pose_detection_presence"
   input_stream_info: {
     tag_index: "FINISHED"
     back_edge: true

diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_gpu.txt b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_gpu.txt
@@ -42,7 +42,7 @@ output_stream: "pose_detection_presence"
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"
-  input_stream: "FINISHED:pose_landmarks_presence"
+  input_stream: "FINISHED:pose_detection_presence"
   input_stream_info: {
     tag_index: "FINISHED"
     back_edge: true

diff --git a/...Resources/face_detection_front.bytes.meta → ...rces/face_detection_full_range.bytes.meta b/...Resources/face_detection_front.bytes.meta → ...rces/face_detection_full_range.bytes.meta
diff --git a/...om.github.homuler.mediapipe/Runtime/Resources/face_detection_full_range_sparse.bytes.meta b/...om.github.homuler.mediapipe/Runtime/Resources/face_detection_full_range_sparse.bytes.meta
diff --git a/...ages/com.github.homuler.mediapipe/Runtime/Resources/face_detection_short_range.bytes.meta b/...ages/com.github.homuler.mediapipe/Runtime/Resources/face_detection_short_range.bytes.meta
diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 # MediaPipe Unity Plugin
-This is a Unity (2020.3.8f1) Plugin to use MediaPipe (0.8.5).
+This is a Unity (2020.3.8f1) Plugin to use MediaPipe (0.8.6).
 
 ## Prerequisites
 To use this plugin, you need to build native libraries for the target platforms (Desktop/UnityEditor, Android, iOS).

diff --git a/WORKSPACE b/WORKSPACE
@@ -4,9 +4,10 @@ workspace(name = "mediapipe_api")
 #
 # - com_google_mediapipe dependency
 # - @//third_party -> @com_google_mediapipe//third_party
-#    - exception: opencv_linux, opencv_windows
+#    - exception: opencv_linux, opencv_windows, ffmpeg_linux, ffmpeg_macos
 # - android_opencv sha256 is added
 # - unity dependency
+# - rules_foreign_cc's version
 # - rules_pkg
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
@@ -42,20 +43,19 @@ new_local_repository(
 # mediapipe
 http_archive(
     name = "com_google_mediapipe",
-    strip_prefix = "mediapipe-0.8.5",
-    sha256 = "2a233540837b6e590a54e0a3c65d60d4d7d2113ec76b96504799a28003aab029",
+    strip_prefix = "mediapipe-0.8.6",
+    sha256 = "3fa1548474d8ef943361804a9f86a3edaa00b0976b9d58fd3b41092d339053c6",
     patches = [
         "@//third_party:mediapipe_opencv.diff",
-        "@//third_party:mediapipe_android.diff",
-        "@//third_party:mediapipe_ios.diff",
+        "@//third_party:mediapipe_workaround.diff",
         "@//third_party:mediapipe_visibility.diff",
         "@//third_party:mediapipe_model_path.diff",
         "@//third_party:mediapipe_extension.diff",
     ],
     patch_args = [
         "-p1",
     ],
-    urls = ["https://github.com/google/mediapipe/archive/v0.8.5.tar.gz"],
+    urls = ["https://github.com/google/mediapipe/archive/v0.8.6.tar.gz"],
 )
 
 # ABSL cpp library lts_2020_09_23
@@ -284,6 +284,20 @@ http_archive(
     url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
 )
 
+http_archive(
+    name = "stblib",
+    strip_prefix = "stb-b42009b3b9d4ca35bc703f5310eedc74f584be58",
+    sha256 = "13a99ad430e930907f5611325ec384168a958bf7610e63e60e2fd8e7b7379610",
+    urls = ["https://github.com/nothings/stb/archive/b42009b3b9d4ca35bc703f5310eedc74f584be58.tar.gz"],
+    build_file = "@com_google_mediapipe//third_party:stblib.BUILD",
+    patches = [
+        "@com_google_mediapipe//third_party:stb_image_impl.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
+)
+
 # You may run setup_android.sh to install Android SDK and NDK.
 android_ndk_repository(
     name = "androidndk",
@@ -379,8 +393,8 @@ maven_install(
         "androidx.test.espresso:espresso-core:3.1.1",
         "com.github.bumptech.glide:glide:4.11.0",
         "com.google.android.material:material:aar:1.0.0-rc01",
-        "com.google.auto.value:auto-value:1.6.4",
-        "com.google.auto.value:auto-value-annotations:1.6.4",
+        "com.google.auto.value:auto-value:1.8.1",
+        "com.google.auto.value:auto-value-annotations:1.8.1",
         "com.google.code.findbugs:jsr305:3.0.2",
         "com.google.flogger:flogger-system-backend:0.3.1",
         "com.google.flogger:flogger:0.3.1",
@@ -411,9 +425,9 @@ http_archive(
 )
 
 # Tensorflow repo should always go after the other external dependencies.
-# 2021-05-27
-_TENSORFLOW_GIT_COMMIT = "d6bfcdb0926173dbb7aa02ceba5aae6250b8aaa6"
-_TENSORFLOW_SHA256 = "ec40e1462239d8783d02f76a43412c8f80bac71ea20e41e1b7729b990aad6923"
+# 2021-06-07
+_TENSORFLOW_GIT_COMMIT = "700533808e6016dc458bb2eeecfca4babfc482ec"
+_TENSORFLOW_SHA256 = "b6edd7f4039bfc19f3e77594ecff558ba620091d0dc48181484b3d9085026126"
 http_archive(
     name = "org_tensorflow",
     urls = [

diff --git a/mediapipe_api/BUILD b/mediapipe_api/BUILD
@@ -144,7 +144,9 @@ pkg_asset(
         "@com_google_mediapipe//mediapipe/models:hair_segmentation.tflite",
         "@com_google_mediapipe//mediapipe/models:ssdlite_object_detection.tflite",
         "@com_google_mediapipe//mediapipe/models:ssdlite_object_detection_labelmap.txt",
-        "@com_google_mediapipe//mediapipe/modules/face_detection:face_detection_front.tflite",
+        # "@com_google_mediapipe//mediapipe/modules/face_detection:face_detection_full_range.tflite",
+        "@com_google_mediapipe//mediapipe/modules/face_detection:face_detection_full_range_sparse.tflite",
+        "@com_google_mediapipe//mediapipe/modules/face_detection:face_detection_short_range.tflite",
         "@com_google_mediapipe//mediapipe/modules/face_landmark:face_landmark.tflite",
         "@com_google_mediapipe//mediapipe/modules/hand_landmark:hand_landmark.tflite",
         "@com_google_mediapipe//mediapipe/modules/hand_landmark:handedness.txt",

diff --git a/mediapipe_api/modules/iris_landmark/BUILD b/mediapipe_api/modules/iris_landmark/BUILD
@@ -12,7 +12,7 @@ mediapipe_simple_subgraph(
     deps = [
         "@com_google_mediapipe//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_gpu",
         "@com_google_mediapipe//mediapipe/calculators/core:concatenate_normalized_landmark_list_calculator",
-        "@com_google_mediapipe//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "@com_google_mediapipe//mediapipe/calculators/core:split_landmarks_calculator",
         "@com_google_mediapipe//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator",
     ],
 )
@@ -24,7 +24,7 @@ mediapipe_simple_subgraph(
     deps = [
         "@com_google_mediapipe//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_cpu",
         "@com_google_mediapipe//mediapipe/calculators/core:concatenate_normalized_landmark_list_calculator",
-        "@com_google_mediapipe//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "@com_google_mediapipe//mediapipe/calculators/core:split_landmarks_calculator",
         "@com_google_mediapipe//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator",
     ],
 )
diff --git a/third_party/mediapipe_android.diff b/third_party/mediapipe_android.diff
diff --git a/third_party/mediapipe_model_path.diff b/third_party/mediapipe_model_path.diff
@@ -245,7 +245,7 @@ index 9bc43ba..088f8f8 100644
      LOG(ERROR) << "Failed to load animation asset.";
      return absl::UnknownError("Failed to load animation asset.");
 diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD
-index da12244..10b2034 100644
+index 11f08ad..4d2db85 100644
 --- a/mediapipe/util/BUILD
 +++ b/mediapipe/util/BUILD
 @@ -155,12 +155,8 @@ cc_library(
@@ -263,20 +263,12 @@ index da12244..10b2034 100644
      hdrs = [
          "resource_util.h",
      ],
-@@ -176,27 +172,14 @@ cc_library(
-     deps = [
-         ":resource_util_custom",
-         "@com_google_absl//absl/container:flat_hash_map",
-+        "//mediapipe/framework/deps:file_path",
-         "//mediapipe/framework/port:ret_check",
-         "//mediapipe/framework/port:singleton",
-         "//mediapipe/framework/port:status",
+@@ -183,19 +179,7 @@ cc_library(
          "//mediapipe/framework/port:statusor",
          "//mediapipe/framework/port:file_helpers",
          "@com_google_absl//absl/strings",
 -    ] + select({
 -        "//conditions:default": [
--            "//mediapipe/framework/deps:file_path",
 -            "@com_google_absl//absl/flags:flag",
 -        ],
 -        "//mediapipe:android": [
@@ -285,7 +277,6 @@ index da12244..10b2034 100644
 -        ],
 -        "//mediapipe:ios": [],
 -        "//mediapipe:macos": [
--            "//mediapipe/framework/deps:file_path",
 -            "@com_google_absl//absl/flags:flag",
 -        ],
 -    }),

diff --git a/third_party/mediapipe_ios.diff → third_party/mediapipe_workaround.diff b/third_party/mediapipe_ios.diff → third_party/mediapipe_workaround.diff
@@ -1,8 +1,26 @@
+diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD
+index 71be05f..5dffa2f 100644
+--- a/mediapipe/calculators/tensor/BUILD
++++ b/mediapipe/calculators/tensor/BUILD
+@@ -87,7 +87,12 @@ cc_library(
+         "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
+         "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
+         "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
+-    ],
++    ] + select({
++        "//mediapipe:android": [
++            "//mediapipe/util/android/file/base",
++        ],
++        "//conditions:default": [],
++    }),
+     alwayslink = 1,
+ )
+
 diff --git a/mediapipe/modules/objectron/calculators/BUILD b/mediapipe/modules/objectron/calculators/BUILD
-index 0a8b326..a1e0e09 100644
+index 66150c5..b3a14a9 100644
 --- a/mediapipe/modules/objectron/calculators/BUILD
 +++ b/mediapipe/modules/objectron/calculators/BUILD
-@@ -170,6 +170,15 @@ cc_library(
+@@ -181,6 +181,15 @@ cc_library(
      hdrs = [
          "tensor_util.h",
      ],
@@ -18,7 +36,7 @@ index 0a8b326..a1e0e09 100644
      deps = [
          "//mediapipe/framework/formats:tensor",
          "//mediapipe/framework/port:logging",
-@@ -285,6 +294,15 @@ cc_library(
+@@ -296,6 +305,15 @@ cc_library(
      name = "lift_2d_frame_annotation_to_3d_calculator",
      srcs = ["lift_2d_frame_annotation_to_3d_calculator.cc"],
      visibility = ["//visibility:public"],
@@ -34,3 +52,16 @@ index 0a8b326..a1e0e09 100644
      deps = [
          ":annotation_cc_proto",
          ":belief_decoder_config_cc_proto",
+diff --git a/mediapipe/util/tflite/tflite_gpu_runner.cc b/mediapipe/util/tflite/tflite_gpu_runner.cc
+index 5445f02..78af4f6 100644
+--- a/mediapipe/util/tflite/tflite_gpu_runner.cc
++++ b/mediapipe/util/tflite/tflite_gpu_runner.cc
+@@ -214,7 +214,7 @@ absl::Status TFLiteGPURunner::InitializeOpenCL(
+   MP_RETURN_IF_ERROR(cl_environment_->NewInferenceBuilder(
+       cl_options, std::move(*graph_cl_), builder));
+ #endif
+-  return absl::OkStatus();
++  return mediapipe::UnimplementedError("Currently only Android is supported");
+ }
+
+ }  // namespace gpu