diff --git a/mediapipe/tasks/ios/audio/core/BUILD b/mediapipe/tasks/ios/audio/core/BUILD index 794c7275cc..b500d2d2be 100644 --- a/mediapipe/tasks/ios/audio/core/BUILD +++ b/mediapipe/tasks/ios/audio/core/BUILD @@ -78,3 +78,48 @@ objc_library( "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", ], ) + +objc_library( + name = "MPPAudioPacketCreator", + srcs = ["sources/MPPAudioPacketCreator.mm"], + hdrs = ["sources/MPPAudioPacketCreator.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + ":MPPAudioData", + "//mediapipe/framework:packet", + "//mediapipe/framework:timestamp", + "//mediapipe/framework/formats:matrix", + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + ], +) + +objc_library( + name = "MPPAudioRunningMode", + hdrs = ["sources/MPPAudioRunningMode.h"], +) + +objc_library( + name = "MPPAudioTaskRunner", + srcs = ["sources/MPPAudioTaskRunner.mm"], + hdrs = ["sources/MPPAudioTaskRunner.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + ":MPPAudioData", + ":MPPAudioPacketCreator", + ":MPPAudioRunningMode", + "//mediapipe/framework:packet", + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPPacketCreator", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/core:MPPTaskRunner", + ], +) diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h b/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h index bb9f7ac030..11975da75d 100644 --- a/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h @@ -34,7 +34,7 @@ NS_SWIFT_NAME(AudioData) @interface MPPAudioData : NSObject /** Audio format specifying the number of channels and sample rate supported. */ -@property(nonatomic, readonly) MPPAudioDataFormat *audioFormat; +@property(nonatomic, readonly) MPPAudioDataFormat *format; /** * A copy of all the internal buffer elements in order with the most recent elements appearing at diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m b/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m index 8cebe38af8..201345e824 100644 --- a/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m @@ -24,7 +24,7 @@ @implementation MPPAudioData { - (instancetype)initWithFormat:(MPPAudioDataFormat *)format sampleCount:(NSUInteger)sampleCount { self = [super init]; if (self) { - _audioFormat = format; + _format = format; const NSInteger length = sampleCount * format.channelCount; _ringBuffer = [[MPPFloatRingBuffer alloc] initWithLength:length]; @@ -40,7 +40,7 @@ - (BOOL)loadBuffer:(MPPFloatBuffer *)buffer } - (BOOL)loadAudioRecord:(MPPAudioRecord *)audioRecord error:(NSError **)error { - if (![audioRecord.audioDataFormat isEqual:self.audioFormat]) { + if (![audioRecord.audioDataFormat isEqual:self.format]) { [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError description:@"The provided audio record has incompatible audio format"]; diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.h b/mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.h new file mode 100644 index 0000000000..f382582fe4 --- /dev/null +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.h @@ -0,0 +1,54 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h" + +#include "mediapipe/framework/packet.h" + +/** + * This class helps create various kinds of packets for MediaPipe Audio Tasks. + */ +@interface MPPAudioPacketCreator : NSObject + +/** + * Creates a MediapPipe Packet wrapping the buffer of a `MPPAudioData` that can be send to a graph. + * + * @param audioData The audio data of type `MPPAudioData` to send to the MediaPipe graph. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return The MediaPipe packet containing the buffer of the given audio data. An empty packet is + * returned if an error occurred during the conversion. + */ ++ (mediapipe::Packet)createPacketWithAudioData:(MPPAudioData *)audioData error:(NSError **)error; + +/** + * Creates a MediapPipe Packet wrapping the buffer of a `MPPAudioData` that can be send to a graph + * at the specified timestamp. + * + * @param audioData The audio data of type `MPPAudioData` to send to the MediaPipe graph. + * @param timestampInMilliseconds The timestamp (in milliseconds) to assign to the packet. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return The MediaPipe packet containing the buffer of the given audio data. An empty packet is + * returned if an error occurred during the conversion. + */ ++ (mediapipe::Packet)createPacketWithAudioData:(MPPAudioData *)audioData + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error; + +@end diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.mm b/mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.mm new file mode 100644 index 0000000000..b5986435ce --- /dev/null +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.mm @@ -0,0 +1,76 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.h" + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/timestamp.h" + +static const NSUInteger kMicrosecondsPerMillisecond = 1000; + +namespace { +using ::mediapipe::Adopt; +using ::mediapipe::Matrix; +using ::mediapipe::Packet; +using ::mediapipe::Timestamp; +} // namespace + +@implementation MPPAudioPacketCreator + ++ (Packet)createPacketWithAudioData:(MPPAudioData *)audioData error:(NSError **)error { + std::unique_ptr matrix = [MPPAudioPacketCreator createMatrixWithAudioData:audioData + error:error]; + if (!matrix) { + return Packet(); + } + + return mediapipe::Adopt(matrix.release()); +} + ++ (Packet)createPacketWithAudioData:(MPPAudioData *)audioData + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::unique_ptr matrix = [MPPAudioPacketCreator createMatrixWithAudioData:audioData + error:error]; + if (!matrix) { + return Packet(); + } + return Adopt(matrix.release()) + .At(Timestamp(int64_t(timestampInMilliseconds * kMicrosecondsPerMillisecond))); +} + ++ (std::unique_ptr)createMatrixWithAudioData:(MPPAudioData *)audioData + error:(NSError **)error { + MPPFloatBuffer *audioDataBuffer = audioData.buffer; + if (!audioDataBuffer.data) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Audio data buffer cannot be nil."]; + return nullptr; + } + + NSUInteger rowCount = audioData.format.channelCount; + NSUInteger colCount = audioData.bufferLength; + + std::unique_ptr matrix(new mediapipe::Matrix(rowCount, colCount)); + // iOS is always little-endian. Hence, data can be copied directly. + memcpy(matrix->data(), audioDataBuffer.data, rowCount * colCount * sizeof(float)); + + return matrix; +} + +@end diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioRunningMode.h b/mediapipe/tasks/ios/audio/core/sources/MPPAudioRunningMode.h new file mode 100644 index 0000000000..3b50591b15 --- /dev/null +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioRunningMode.h @@ -0,0 +1,48 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * MediaPipe audio task running mode. A MediaPipe audio task can be run with three different + * modes: image, video and live stream. + */ +typedef NS_ENUM(NSUInteger, MPPAudioRunningMode) { + + /** The mode for running a mediapipe audio task on independent audio clips. */ + MPPAudioRunningModeAudioClips NS_SWIFT_NAME(audioClips), + + /** + * The mode for running a mediapipe audio task on an audio stream, such as from a microphone. + */ + MPPAudioRunningModeAudioStream NS_SWIFT_NAME(audioStream), + + } NS_SWIFT_NAME(RunningMode); // In Swift `RunningMode` can be resolved as + // `MediaPipeTasksAudio.RunningMode` when used alongside the other + // task libraries. + +NS_INLINE NSString *MPPAudioRunningModeDisplayName(MPPAudioRunningMode runningMode) { + switch (runningMode) { + case MPPAudioRunningModeAudioClips: + return @"Audio Clips"; + case MPPAudioRunningModeAudioStream: + return @"Audio Stream"; + default: + return nil; + } +} + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioTaskRunner.h b/mediapipe/tasks/ios/audio/core/sources/MPPAudioTaskRunner.h new file mode 100644 index 0000000000..662b1b4e6e --- /dev/null +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioTaskRunner.h @@ -0,0 +1,88 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import + +#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h" +#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioRunningMode.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" + +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, + * execute and terminate any MediaPipe audio task. + */ +@interface MPPAudioTaskRunner : MPPTaskRunner + +/** + * Initializes a new `MPPAudioTaskRunner` with the given task info, audio running mode, packets + * callback, audio input and sample rate stream names. Make sure that the packets callback is set + * properly based on the audio task's running mode. In case of audio stream running mode, a C++ + * packets callback that is intended to deliver inference results must be provided. In audio clips + * mode, packets callback must be set to nil. + * + * @param taskInfo A `MPPTaskInfo` initialized by the task. + * @param runningMode MediaPipe audio task running mode. + * @param packetsCallback An optional C++ callback function that takes a list of output packets as + * the input argument. If provided, the callback must in turn call the block provided by the user in + * the appropriate task options. Make sure that the packets callback is set properly based on the + * audio task's running mode. In case of audio stream running mode, a C++ packets callback that is + * intended to deliver inference results must be provided. In audio clips running mode, packets + * callback must be set to nil. + * @param audioInputStreamName Name of the audio input stream of the task. + * @param sampleRatInputStreamName Name of the sample rate input stream of the task. + * + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return An instance of `MPPAudioTaskRunner` initialized with the given task info, running mode, + * packets callback, audio input and sample rate stream names. + */ + +- (nullable instancetype)initWithTaskInfo:(MPPTaskInfo *)taskInfo + runningMode:(MPPAudioRunningMode)runningMode + packetsCallback:(mediapipe::tasks::core::PacketsCallback)packetsCallback + audioInputStreamName:(NSString *)imageInputStreamName + sampleRateInputStreamName:(nullable NSString *)normRectInputStreamName + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * A synchronous method to invoke the C++ task runner to process standalone audio clip inputs. The + * call blocks the current thread until a failure status or a successful result is returned. + * + * + * @param audioClip An audio clip input of type `MPPAudioData` to the task. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An optional `PacketMap` containing pairs of output stream name and data packet. + */ +- (std::optional)processAudioClip:(MPPAudioData *)audioClip + error:(NSError **)error; + +- (instancetype)initWithTaskInfo:(MPPTaskInfo *)taskInfo + packetsCallback:(mediapipe::tasks::core::PacketsCallback)packetsCallback + error:(NSError **)error NS_UNAVAILABLE; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioTaskRunner.mm b/mediapipe/tasks/ios/audio/core/sources/MPPAudioTaskRunner.mm new file mode 100644 index 0000000000..38c0bbc382 --- /dev/null +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioTaskRunner.mm @@ -0,0 +1,147 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioTaskRunner.h" + +#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPPacketCreator.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" + +#include + +namespace { +using ::mediapipe::Packet; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; +} // namespace + +static NSString *const kTaskPrefix = @"com.mediapipe.tasks.audio"; + +@interface MPPAudioTaskRunner () { + MPPAudioRunningMode _runningMode; + NSString *_audioInputStreamName; + NSString *_sampleRateInputStreamName; +} +@end + +@implementation MPPAudioTaskRunner + +- (nullable instancetype)initWithTaskInfo:(MPPTaskInfo *)taskInfo + runningMode:(MPPAudioRunningMode)runningMode + packetsCallback:(mediapipe::tasks::core::PacketsCallback)packetsCallback + audioInputStreamName:(NSString *)audioInputStreamName + sampleRateInputStreamName:(nullable NSString *)sampleRateInputStreamName + error:(NSError **)error { + if (!taskInfo) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"`taskInfo` cannot be `nil`."]; + return nil; + } + + if (!audioInputStreamName) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"`audioInputStreamName` cannot be `nil.`"]; + return nil; + } + + if (!sampleRateInputStreamName) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"`sampleRateInputStreamName` cannot be `nil.`"]; + return nil; + } + + _audioInputStreamName = audioInputStreamName; + _sampleRateInputStreamName = sampleRateInputStreamName; + + switch (runningMode) { + case MPPAudioRunningModeAudioClips: { + if (packetsCallback) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"The audio task is in audio clips mode. The " + @"delegate must not be set in the task's options."]; + return nil; + } + break; + } + case MPPAudioRunningModeAudioStream: { + if (!packetsCallback) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The audio task is in audio stream mode. An object must be set as the " + @"delegate of the task in its options to ensure asynchronous delivery of " + @"results."]; + return nil; + } + break; + } + default: { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Unrecognized running mode"]; + return nil; + } + } + + _runningMode = runningMode; + + self = [super initWithTaskInfo:taskInfo packetsCallback:packetsCallback error:error]; + return self; +} + +- (std::optional)processAudioClip:(MPPAudioData *)audioClip + error:(NSError **)error { + if (_runningMode != MPPAudioRunningModeAudioClips) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:[NSString stringWithFormat:@"The audio task is not initialized with " + @"audio clips. Current Running Mode: %@", + MPPAudioRunningModeDisplayName(_runningMode)]]; + return std::nullopt; + } + + std::optional inputPacketMap = [self inputPacketMapWithMPPAudioData:audioClip + error:error]; + + return inputPacketMap.has_value() ? [self processPacketMap:inputPacketMap.value() error:error] + : std::nullopt; +} + +- (std::optional)inputPacketMapWithMPPAudioData:(MPPAudioData *)audioData + error:(NSError **)error { + PacketMap inputPacketMap; + + Packet matrixPacket = [MPPAudioPacketCreator createPacketWithAudioData:audioData error:error]; + if (matrixPacket.IsEmpty()) { + return std::nullopt; + } + + inputPacketMap[_audioInputStreamName.cppString] = matrixPacket; + + inputPacketMap[_sampleRateInputStreamName.cppString] = + [MPPPacketCreator createWithDouble:(double)audioData.format.sampleRate]; + + return inputPacketMap; +} + +@end diff --git a/mediapipe/tasks/ios/core/BUILD b/mediapipe/tasks/ios/core/BUILD index ad8fa3f23d..6995ab2e60 100644 --- a/mediapipe/tasks/ios/core/BUILD +++ b/mediapipe/tasks/ios/core/BUILD @@ -95,3 +95,14 @@ objc_library( "@com_google_protobuf//:protobuf", ], ) + +objc_library( + name = "MPPPacketCreator", + srcs = ["sources/MPPPacketCreator.mm"], + hdrs = ["sources/MPPPacketCreator.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = ["//mediapipe/framework:packet"], +) diff --git a/mediapipe/tasks/ios/core/sources/MPPPacketCreator.h b/mediapipe/tasks/ios/core/sources/MPPPacketCreator.h new file mode 100644 index 0000000000..a9810f0235 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPPacketCreator.h @@ -0,0 +1,24 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include "mediapipe/framework/packet.h" + +/** This class helps create various kinds of commonly used packets for all MediaPipe Tasks. */ +@interface MPPPacketCreator : NSObject + ++ (mediapipe::Packet)createWithDouble:(double)doubleValue; + +@end diff --git a/mediapipe/tasks/ios/core/sources/MPPPacketCreator.mm b/mediapipe/tasks/ios/core/sources/MPPPacketCreator.mm new file mode 100644 index 0000000000..acd7d98de7 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPPacketCreator.mm @@ -0,0 +1,27 @@ +// Copyright 2024 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/sources/MPPPacketCreator.h" + +namespace { +using ::mediapipe::MakePacket; +} // namespace + +@implementation MPPPacketCreator + ++ (mediapipe::Packet)createWithDouble:(double)doubleValue{ + return MakePacket(doubleValue); +} + +@end