Merge pull request #5397 from priankakariatyml:ios-audio-task-runner

PiperOrigin-RevId: 632546454

mediapipe/tasks/ios/audio/core/BUILD

@@ -78,3 +78,48 @@ objc_library(
         "//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
     ],
 )
+
+objc_library(
+    name = "MPPAudioPacketCreator",
+    srcs = ["sources/MPPAudioPacketCreator.mm"],
+    hdrs = ["sources/MPPAudioPacketCreator.h"],
+    copts = [
+        "-ObjC++",
+        "-std=c++17",
+    ],
+    deps = [
+        ":MPPAudioData",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/tasks/ios/common:MPPCommon",
+        "//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
+    ],
+)
+
+objc_library(
+    name = "MPPAudioRunningMode",
+    hdrs = ["sources/MPPAudioRunningMode.h"],
+)
+
+objc_library(
+    name = "MPPAudioTaskRunner",
+    srcs = ["sources/MPPAudioTaskRunner.mm"],
+    hdrs = ["sources/MPPAudioTaskRunner.h"],
+    copts = [
+        "-ObjC++",
+        "-std=c++17",
+    ],
+    deps = [
+        ":MPPAudioData",
+        ":MPPAudioPacketCreator",
+        ":MPPAudioRunningMode",
+        "//mediapipe/framework:packet",
+        "//mediapipe/tasks/ios/common:MPPCommon",
+        "//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
+        "//mediapipe/tasks/ios/common/utils:NSStringHelpers",
+        "//mediapipe/tasks/ios/core:MPPPacketCreator",
+        "//mediapipe/tasks/ios/core:MPPTaskInfo",
+        "//mediapipe/tasks/ios/core:MPPTaskRunner",
+    ],
+)

mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h

@@ -34,7 +34,7 @@ NS_SWIFT_NAME(AudioData)
 @interface MPPAudioData : NSObject
 
 /** Audio format specifying the number of channels and sample rate supported. */
-@property(nonatomic, readonly) MPPAudioDataFormat *audioFormat;
+@property(nonatomic, readonly) MPPAudioDataFormat *format;
 
 /**
  * A copy of all the internal buffer elements in order with the most recent elements appearing at

mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m

@@ -24,7 +24,7 @@ @implementation MPPAudioData {
 - (instancetype)initWithFormat:(MPPAudioDataFormat *)format sampleCount:(NSUInteger)sampleCount {
   self = [super init];
   if (self) {
-    _audioFormat = format;
+    _format = format;
 
     const NSInteger length = sampleCount * format.channelCount;
     _ringBuffer = [[MPPFloatRingBuffer alloc] initWithLength:length];
@@ -40,7 +40,7 @@ - (BOOL)loadBuffer:(MPPFloatBuffer *)buffer
 }
 
 - (BOOL)loadAudioRecord:(MPPAudioRecord *)audioRecord error:(NSError **)error {
-  if (![audioRecord.audioDataFormat isEqual:self.audioFormat]) {
+  if (![audioRecord.audioDataFormat isEqual:self.format]) {
     [MPPCommonUtils createCustomError:error
                              withCode:MPPTasksErrorCodeInvalidArgumentError
                           description:@"The provided audio record has incompatible audio format"];

mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.h

@@ -0,0 +1,54 @@
+// Copyright 2024 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+
+#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h"
+
+#include "mediapipe/framework/packet.h"
+
+/**
+ * This class helps create various kinds of packets for MediaPipe Audio Tasks.
+ */
+@interface MPPAudioPacketCreator : NSObject
+
+/**
+ * Creates a MediapPipe Packet wrapping the buffer of a `MPPAudioData` that can be send to a graph.
+ *
+ * @param audioData The audio data of type `MPPAudioData` to send to the MediaPipe graph.
+ * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no
+ *    error will be saved.
+ *
+ * @return The MediaPipe packet containing the buffer of the given audio data. An empty packet is
+ * returned if an error occurred during the conversion.
+ */
++ (mediapipe::Packet)createPacketWithAudioData:(MPPAudioData *)audioData error:(NSError **)error;
+
+/**
+ * Creates a MediapPipe Packet wrapping the buffer of a `MPPAudioData` that can be send to a graph
+ * at the specified timestamp.
+ *
+ * @param audioData The audio data of type `MPPAudioData` to send to the MediaPipe graph.
+ * @param timestampInMilliseconds The timestamp (in milliseconds) to assign to the packet.
+ * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no
+ *    error will be saved.
+ *
+ * @return The MediaPipe packet containing the buffer of the given audio data. An empty packet is
+ * returned if an error occurred during the conversion.
+ */
++ (mediapipe::Packet)createPacketWithAudioData:(MPPAudioData *)audioData
+                       timestampInMilliseconds:(NSInteger)timestampInMilliseconds
+                                         error:(NSError **)error;
+
+@end

mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.mm

@@ -0,0 +1,76 @@
+// Copyright 2024 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.h"
+
+#import "mediapipe/tasks/ios/common/sources/MPPCommon.h"
+#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h"
+
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/timestamp.h"
+
+static const NSUInteger kMicrosecondsPerMillisecond = 1000;
+
+namespace {
+using ::mediapipe::Adopt;
+using ::mediapipe::Matrix;
+using ::mediapipe::Packet;
+using ::mediapipe::Timestamp;
+}  // namespace
+
+@implementation MPPAudioPacketCreator
+
++ (Packet)createPacketWithAudioData:(MPPAudioData *)audioData error:(NSError **)error {
+  std::unique_ptr<Matrix> matrix = [MPPAudioPacketCreator createMatrixWithAudioData:audioData
+                                                                              error:error];
+  if (!matrix) {
+    return Packet();
+  }
+
+  return mediapipe::Adopt(matrix.release());
+}
+
++ (Packet)createPacketWithAudioData:(MPPAudioData *)audioData
+            timestampInMilliseconds:(NSInteger)timestampInMilliseconds
+                              error:(NSError **)error {
+  std::unique_ptr<Matrix> matrix = [MPPAudioPacketCreator createMatrixWithAudioData:audioData
+                                                                              error:error];
+  if (!matrix) {
+    return Packet();
+  }
+  return Adopt(matrix.release())
+      .At(Timestamp(int64_t(timestampInMilliseconds * kMicrosecondsPerMillisecond)));
+}
+
++ (std::unique_ptr<Matrix>)createMatrixWithAudioData:(MPPAudioData *)audioData
+                                               error:(NSError **)error {
+  MPPFloatBuffer *audioDataBuffer = audioData.buffer;
+  if (!audioDataBuffer.data) {
+    [MPPCommonUtils createCustomError:error
+                             withCode:MPPTasksErrorCodeInvalidArgumentError
+                          description:@"Audio data buffer cannot be nil."];
+    return nullptr;
+  }
+
+  NSUInteger rowCount = audioData.format.channelCount;
+  NSUInteger colCount = audioData.bufferLength;
+
+  std::unique_ptr<mediapipe::Matrix> matrix(new mediapipe::Matrix(rowCount, colCount));
+  // iOS is always little-endian. Hence, data can be copied directly.
+  memcpy(matrix->data(), audioDataBuffer.data, rowCount * colCount * sizeof(float));
+
+  return matrix;
+}
+
+@end

mediapipe/tasks/ios/audio/core/sources/MPPAudioRunningMode.h

@@ -0,0 +1,48 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+/**
+ * MediaPipe audio task running mode. A MediaPipe audio task can be run with three different
+ * modes: image, video and live stream.
+ */
+typedef NS_ENUM(NSUInteger, MPPAudioRunningMode) {
+
+  /** The mode for running a mediapipe audio task on independent audio clips. */
+  MPPAudioRunningModeAudioClips NS_SWIFT_NAME(audioClips),
+
+      /**
+       * The mode for running a mediapipe audio task on an audio stream, such as from a microphone.
+       */
+      MPPAudioRunningModeAudioStream NS_SWIFT_NAME(audioStream),
+
+  } NS_SWIFT_NAME(RunningMode);  // In Swift `RunningMode` can be resolved as
+                                 // `MediaPipeTasksAudio.RunningMode` when used alongside the other
+                                 // task libraries.
+
+NS_INLINE NSString *MPPAudioRunningModeDisplayName(MPPAudioRunningMode runningMode) {
+  switch (runningMode) {
+    case MPPAudioRunningModeAudioClips:
+      return @"Audio Clips";
+    case MPPAudioRunningModeAudioStream:
+      return @"Audio Stream";
+    default:
+      return nil;
+  }
+}
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/audio/core/sources/MPPAudioTaskRunner.h

@@ -0,0 +1,88 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+#import <UIKit/UIKit.h>
+
+#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h"
+#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioRunningMode.h"
+#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h"
+
+#include "mediapipe/framework/packet.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+/**
+ * This class is used to create and call appropriate methods on the C++ Task Runner to initialize,
+ * execute and terminate any MediaPipe audio task.
+ */
+@interface MPPAudioTaskRunner : MPPTaskRunner
+
+/**
+ * Initializes a new `MPPAudioTaskRunner` with the given task info, audio running mode, packets
+ * callback, audio input and sample rate stream names. Make sure that the packets callback is set
+ * properly based on the audio task's running mode. In case of audio stream running mode, a C++
+ * packets callback that is intended to deliver inference results must be provided. In audio clips
+ * mode, packets callback must be set to nil.
+ *
+ * @param taskInfo A `MPPTaskInfo` initialized by the task.
+ * @param runningMode MediaPipe audio task running mode.
+ * @param packetsCallback An optional C++ callback function that takes a list of output packets as
+ * the input argument. If provided, the callback must in turn call the block provided by the user in
+ * the appropriate task options. Make sure that the packets callback is set properly based on the
+ * audio task's running mode. In case of audio stream running mode, a C++ packets callback that is
+ * intended to deliver inference results must be provided. In audio clips running mode, packets
+ * callback must be set to nil.
+ * @param audioInputStreamName Name of the audio input stream of the task.
+ * @param sampleRatInputStreamName Name of the sample rate input stream of the task.
+ *
+ * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no
+ * error will be saved.
+ *
+ * @return An instance of `MPPAudioTaskRunner` initialized with the given task info, running mode,
+ * packets callback, audio input and sample rate stream names.
+ */
+
+- (nullable instancetype)initWithTaskInfo:(MPPTaskInfo *)taskInfo
+                              runningMode:(MPPAudioRunningMode)runningMode
+                          packetsCallback:(mediapipe::tasks::core::PacketsCallback)packetsCallback
+                     audioInputStreamName:(NSString *)imageInputStreamName
+                sampleRateInputStreamName:(nullable NSString *)normRectInputStreamName
+                                    error:(NSError **)error NS_DESIGNATED_INITIALIZER;
+
+/**
+ * A synchronous method to invoke the C++ task runner to process standalone audio clip inputs. The
+ * call blocks the current thread until a failure status or a successful result is returned.
+ *
+ *
+ * @param audioClip An audio clip input of type `MPPAudioData` to the task.
+ * @param error Pointer to the memory location where errors if any should be
+ * saved. If @c NULL, no error will be saved.
+ *
+ * @return An optional `PacketMap` containing pairs of output stream name and data packet.
+ */
+- (std::optional<mediapipe::tasks::core::PacketMap>)processAudioClip:(MPPAudioData *)audioClip
+                                                               error:(NSError **)error;
+
+- (instancetype)initWithTaskInfo:(MPPTaskInfo *)taskInfo
+                 packetsCallback:(mediapipe::tasks::core::PacketsCallback)packetsCallback
+                           error:(NSError **)error NS_UNAVAILABLE;
+
+- (instancetype)init NS_UNAVAILABLE;
+
++ (instancetype)new NS_UNAVAILABLE;
+
+@end
+
+NS_ASSUME_NONNULL_END