From c320cb854c16b6a029300de29ec7561b102770a5 Mon Sep 17 00:00:00 2001 From: Victor Gaydov Date: Thu, 11 Jul 2024 20:11:50 +0400 Subject: [PATCH] [spo] [api] gh-731: Add PLC to C API Built-in PLC support: - add roc_plc_backend - add roc_receiver_config.plc_backend Custom PLC support: - add plugin.h - add ROC_ENCODING_ID_MIN, ROC_ENCODING_ID_MAX - add ROC_PLUGIN_ID_MIN, ROC_PLUGIN_ID_MAX - add roc_plugin_plc - add roc_context_register_plc Extra: - add API usage example - add API integration test Sponsored-by: waspd --- docs/sphinx/api/reference.rst | 22 ++ .../roc_status/code_to_str.cpp | 2 + src/internal_modules/roc_status/status_code.h | 8 + src/public_api/examples/plugin_plc.c | 171 ++++++++++ src/public_api/include/roc/config.h | 48 ++- src/public_api/include/roc/context.h | 33 +- src/public_api/include/roc/plugin.h | 194 +++++++++++ src/public_api/src/adapters.cpp | 47 ++- src/public_api/src/adapters.h | 2 + src/public_api/src/context.cpp | 41 +++ src/public_api/src/plugin_plc.cpp | 153 +++++++++ src/public_api/src/plugin_plc.h | 68 ++++ src/tests/public_api/test_helpers/context.h | 4 + src/tests/public_api/test_plugin_plc.cpp | 318 ++++++++++++++++++ 14 files changed, 1090 insertions(+), 21 deletions(-) create mode 100644 src/public_api/examples/plugin_plc.c create mode 100644 src/public_api/include/roc/plugin.h create mode 100644 src/public_api/src/plugin_plc.cpp create mode 100644 src/public_api/src/plugin_plc.h create mode 100644 src/tests/public_api/test_plugin_plc.cpp diff --git a/docs/sphinx/api/reference.rst b/docs/sphinx/api/reference.rst index 1caf8476c..ddcea99e4 100644 --- a/docs/sphinx/api/reference.rst +++ b/docs/sphinx/api/reference.rst @@ -26,6 +26,8 @@ roc_context .. doxygenfunction:: roc_context_register_encoding +.. doxygenfunction:: roc_context_register_plc + .. doxygenfunction:: roc_context_close roc_sender @@ -209,6 +211,8 @@ roc_config .. doxygenenum:: roc_resampler_profile +.. doxygenenum:: roc_plc_backend + .. doxygenstruct:: roc_context_config :members: @@ -234,6 +238,24 @@ roc_metrics .. doxygenstruct:: roc_receiver_metrics :members: +roc_plugin +========== + +.. code-block:: c + + #include + +.. doxygenenumvalue:: ROC_ENCODING_ID_MIN + +.. doxygenenumvalue:: ROC_ENCODING_ID_MAX + +.. doxygenenumvalue:: ROC_PLUGIN_ID_MIN + +.. doxygenenumvalue:: ROC_PLUGIN_ID_MAX + +.. doxygenstruct:: roc_plugin_plc + :members: + roc_log ======= diff --git a/src/internal_modules/roc_status/code_to_str.cpp b/src/internal_modules/roc_status/code_to_str.cpp index d665018ea..8f61ce5b4 100644 --- a/src/internal_modules/roc_status/code_to_str.cpp +++ b/src/internal_modules/roc_status/code_to_str.cpp @@ -36,6 +36,8 @@ const char* code_to_str(StatusCode code) { return "NoRoute"; case StatusNoDriver: return "NoDriver"; + case StatusNoPlugin: + return "NoPlugin"; case StatusErrDevice: return "ErrDevice"; case StatusErrFile: diff --git a/src/internal_modules/roc_status/status_code.h b/src/internal_modules/roc_status/status_code.h index 8d8d67c0a..5ff8a15d2 100644 --- a/src/internal_modules/roc_status/status_code.h +++ b/src/internal_modules/roc_status/status_code.h @@ -98,6 +98,14 @@ enum StatusCode { //! supports only wav files. StatusNoDriver, + //! No plugin found. + //! @remarks + //! Indicates that plugin lookup or initialization failed. + //! @note + //! Example: we're trying to create PLC plugin, but use-provided callback + //! failed to allocate it. + StatusNoPlugin, + //! Failure with audio device. //! @remarks //! Indicates that error occurred when working with audio device. diff --git a/src/public_api/examples/plugin_plc.c b/src/public_api/examples/plugin_plc.c new file mode 100644 index 000000000..04a9895ef --- /dev/null +++ b/src/public_api/examples/plugin_plc.c @@ -0,0 +1,171 @@ +/* + * Register custom Packet loss concealment (PLC) plugin. + * + * PLC allows to reduce distortion caused by packet losses by replacing + * gaps with interpolated data. It is used only when FEC wasn't able to + * repair lost packets. + * + * Building: + * cc plugin_plc.c -lroc + * + * Running: + * ./a.out + * + * License: + * public domain + */ + +#include +#include +#include + +#include +#include +#include + +/* Any number in range [ROC_PLUGIN_ID_MIN; ROC_PLUGIN_ID_MAX] */ +#define MY_PLC_PLUGIN_ID ROC_PLUGIN_ID_MIN + 1 + +#define MY_SAMPLE_RATE 44100 +#define MY_CHANNEL_COUNT 2 +#define MY_LOOKAHEAD_SIZE 4410 /* 100 ms */ + +#define oops() \ + do { \ + fprintf(stderr, "oops: failure on %s:%d\n", __FILE__, __LINE__); \ + fprintf(stderr, "exiting!\n"); \ + exit(1); \ + } while (0) + +/* PLC plugin instance. + * roc_receiver will create an instance for every connection. */ +struct my_plc { + /* Here we could put state needed for interpolation. */ + unsigned int history_frame_counter; + unsigned int lost_frame_counter; +}; + +/* Create plugin instance. */ +static void* my_plc_new(roc_plugin_plc* plugin) { + return calloc(1, sizeof(struct my_plc)); +} + +/* Delete plugin instance. */ +static void my_plc_delete(void* plugin_instance) { + struct my_plc* plc = (struct my_plc*)plugin_instance; + + free(plc); +} + +/* Get look-ahead length - how many samples after the lost frame + * do we need for interpolation. + * Returned value is measured as the number of samples per channel, + * e.g. if sample rate is 44100Hz, length 4410 is 100ms */ +static unsigned int my_plc_lookahead_len(void* plugin_instance) { + return MY_LOOKAHEAD_SIZE; +} + +/* Called when next frame is good (no loss). */ +static void my_plc_process_history(void* plugin_instance, + const roc_frame* history_frame) { + struct my_plc* plc = (struct my_plc*)plugin_instance; + + /* Here we can copy samples from history_frame to ring buffer. + * In this example we just ignore frame. */ + plc->history_frame_counter++; +} + +/* Called when next frame is lost and we must fill it with interpolated data. + * + * lost_frame is the frame to be filled (we must fill its buffer with the + * interpolated samples) + * + * lookahead_frame contains samples going after the lost frame, which we can + * use to improve interpolation results. Its size may vary from 0 to MY_LOOKAHEAD_SIZE. + */ +static void my_plc_process_loss(void* plugin_instance, + roc_frame* lost_frame, + const roc_frame* lookahead_frame) { + struct my_plc* plc = (struct my_plc*)plugin_instance; + + /* Here we can implement interpolation. + * In this example we just fill frame with constants. + * Samples are float because we use ROC_FORMAT_PCM_FLOAT32. + * There are two channels because we use ROC_CHANNEL_LAYOUT_STEREO. */ + float* lost_samples = lost_frame->samples; + size_t lost_sample_count = + lost_frame->samples_size / sizeof(float) / MY_CHANNEL_COUNT; + + for (unsigned ns = 0; ns < lost_sample_count; ns++) { + for (unsigned c = 0; c < MY_CHANNEL_COUNT; c++) { + lost_samples[0] = 0.123f; /* left channel */ + lost_samples[1] = 0.456f; /* right channel */ + } + lost_samples += MY_CHANNEL_COUNT; + } + + plc->lost_frame_counter++; +} + +int main() { + roc_log_set_level(ROC_LOG_INFO); + + /* Create context. */ + roc_context_config context_config; + memset(&context_config, 0, sizeof(context_config)); + + roc_context* context = NULL; + if (roc_context_open(&context_config, &context) != 0) { + oops(); + } + + /* Register plugin. */ + roc_plugin_plc plc_plugin; + memset(&plc_plugin, 0, sizeof(plc_plugin)); + + plc_plugin.new_cb = &my_plc_new; + plc_plugin.delete_cb = &my_plc_delete; + plc_plugin.lookahead_len_cb = &my_plc_lookahead_len; + plc_plugin.process_history_cb = &my_plc_process_history; + plc_plugin.process_loss_cb = &my_plc_process_loss; + + if (roc_context_register_plc(context, MY_PLC_PLUGIN_ID, &plc_plugin) != 0) { + oops(); + } + + /* Prepare receiver config. */ + roc_receiver_config receiver_config; + memset(&receiver_config, 0, sizeof(receiver_config)); + + /* Setup frame format. + * This format applies to frames that we read from receiver, as well as to + * the frames passed to PLC plugin. */ + receiver_config.frame_encoding.rate = MY_SAMPLE_RATE; + receiver_config.frame_encoding.format = ROC_FORMAT_PCM_FLOAT32; + receiver_config.frame_encoding.channels = ROC_CHANNEL_LAYOUT_STEREO; + + /* Enable PLC plugin. */ + receiver_config.plc_backend = (roc_plc_backend)MY_PLC_PLUGIN_ID; + + /* Create receiver. */ + roc_receiver* receiver = NULL; + if (roc_receiver_open(context, &receiver_config, &receiver) != 0) { + oops(); + } + + /* + * Here we can run receiver loop. + */ + + /* Destroy receiver. */ + if (roc_receiver_close(receiver) != 0) { + oops(); + } + + /* Destroy context. */ + if (roc_context_close(context) != 0) { + oops(); + } + + return 0; +} diff --git a/src/public_api/include/roc/config.h b/src/public_api/include/roc/config.h index 4335e7bfe..e02c04ba7 100644 --- a/src/public_api/include/roc/config.h +++ b/src/public_api/include/roc/config.h @@ -569,8 +569,8 @@ typedef enum roc_resampler_backend { * stage is needed, and this becomes fastest possible backend working almost as fast * as memcpy(). * - * When frame and packet rates are different, usage of this backend compared to - * \c ROC_RESAMPLER_BACKEND_SPEEX allows to sacrify some quality, but somewhat + * When frame and packet rates are different, usage of this backend, compared to + * \c ROC_RESAMPLER_BACKEND_SPEEX, allows to sacrify some quality, but somewhat * improve scaling precision and CPU usage in return. * * This backend is available only when SpeexDSP was enabled at build time. @@ -600,6 +600,25 @@ typedef enum roc_resampler_profile { ROC_RESAMPLER_PROFILE_LOW = 3 } roc_resampler_profile; +/** PLC backend. + * + * Packet loss concealment (PLC), is used to reduce distortion caused by lost + * packets by filling gaps with interpolated or extrapolated data. + * + * PLC is used when a packet was lost and FEC was not able to recover it. + */ +typedef enum roc_plc_backend { + /** No PLC. + * Gaps are filled with zeros (silence). + */ + ROC_PLC_BACKEND_DISABLE = -1, + + /** Default backend. + * Current default is \c ROC_PLC_BACKEND_DISABLE. + */ + ROC_PLC_BACKEND_DEFAULT = 0, +} roc_plc_backend; + /** Context configuration. * * It is safe to memset() this struct with zeros to get a default config. It is also @@ -652,15 +671,14 @@ typedef struct roc_sender_config { * automatically. * * If zero, sender selects packet encoding automatically based on \c frame_encoding. - * This automatic selection matches only encodings that have exact same sample rate - * and channel layout, and hence don't require conversions. If you need conversions, - * you should set packet encoding explicitly. + * This automatic selection matches only encodings that have exact same sample rate, + * channel layout, and format, hence don't require conversions. If you need + * conversions, you should set packet encoding explicitly. * - * If you want to force specific packet encoding, and built-in set of encodings is - * not enough, you can use \ref roc_context_register_encoding() to register custom - * encoding, and set \c packet_encoding to registered identifier. If you use signaling - * protocol like RTSP, it's enough to register in just on sender; otherwise, you - * need to do the same on receiver as well. + * You can use \ref roc_context_register_encoding() to register custom encoding, and + * set \c packet_encoding to registered identifier. If you use signaling protocol like + * RTSP, it's enough to register in just on sender; otherwise, you need to do the same + * on receiver as well. */ roc_packet_encoding packet_encoding; @@ -854,6 +872,16 @@ typedef struct roc_receiver_config { */ roc_resampler_profile resampler_profile; + /** PLC backend. + * Allows to reduce distortion cased by packet loss. + * + * If zero, default backend is used (\ref ROC_PLC_BACKEND_DEFAULT). + * + * You can use \ref roc_context_register_plc() to register custom PLC implementation, + * and set \c plc_backend to registered identifier. + */ + roc_plc_backend plc_backend; + /** Target latency, in nanoseconds. * * How latency is calculated depends on \c latency_tuner_backend field. diff --git a/src/public_api/include/roc/context.h b/src/public_api/include/roc/context.h index 071044755..f7e013499 100644 --- a/src/public_api/include/roc/context.h +++ b/src/public_api/include/roc/context.h @@ -16,6 +16,7 @@ #include "roc/config.h" #include "roc/platform.h" +#include "roc/plugin.h" #ifdef __cplusplus extern "C" { @@ -57,6 +58,8 @@ typedef struct roc_context roc_context; * - returns a negative value if there are not enough resources * * **Ownership** + * - doesn't take or share the ownership of \p config; it may be safely deallocated + * after the function returns * - passes the ownership of \p result to the user; the user is responsible to call * roc_context_close() to free it */ @@ -64,20 +67,17 @@ ROC_API int roc_context_open(const roc_context_config* config, roc_context** res /** Register custom encoding. * - * Registers \p encoding with given \p encoding_id. Registered encodings complement + * Registers \p encoding with given \p encoding_id. Registered encodings extend * built-in encodings defined by \ref roc_packet_encoding enum. Whenever you need to * specify packet encoding, you can use both built-in and registered encodings. * * On sender, you should register custom encoding and set to \c packet_encoding field - * of \c roc_sender_config, if you need to force specific encoding of packets, but - * built-in set of encodings is not enough. + * of \c roc_sender_config, if you need to force specific encoding of packets. * * On receiver, you should register custom encoding with same id and specification, * if you did so on sender, and you're not using any signaling protocol (like RTSP) * that is capable of automatic exchange of encoding information. * - * In case of RTP, encoding id is mapped directly to payload type field (PT). - * * **Parameters** * - \p context should point to an opened context * - \p encoding_id should be in range [ROC_ENCODING_ID_MIN; ROC_ENCODING_ID_MAX] @@ -96,6 +96,29 @@ ROC_API int roc_context_register_encoding(roc_context* context, int encoding_id, const roc_media_encoding* encoding); +/** Register custom PLC backend. + * + * Registers plugin that implements custom PLC backend. Registered backends extend + * built-in PLC backends defined by \ref roc_plc_backend enum. Whenever you need to + * specify PLC backend, you can use both built-in and registered backends. + * + * **Parameters** + * - \p context should point to an opened context + * - \p plugin_id should be in range [ROC_PLUGIN_ID_MIN; ROC_PLUGIN_ID_MAX] + * - \p plugin should point to plugin callback table + * + * **Returns** + * - returns zero if plugin was successfully registered + * - returns a negative value if the arguments are invalid + * - returns a negative value if plugin with given identifier already exists + * + * **Ownership** + * - stores \p plugin pointer internally for later use; \p plugin should remain valid + * until \p context is closed + */ +ROC_API int +roc_context_register_plc(roc_context* context, int plugin_id, roc_plugin_plc* plugin); + /** Close the context. * * Stops any started background threads, deinitializes and deallocates the context. diff --git a/src/public_api/include/roc/plugin.h b/src/public_api/include/roc/plugin.h new file mode 100644 index 000000000..c077e4c43 --- /dev/null +++ b/src/public_api/include/roc/plugin.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2024 Roc Streaming authors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +/** + * \file roc/plugin.h + * \brief User plugins. + */ + +#ifndef ROC_PLUGIN_H_ +#define ROC_PLUGIN_H_ + +#include "roc/frame.h" +#include "roc/platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + /** Minumum allowed packet encoding id. + * + * \ref ROC_ENCODING_ID_MIN and \ref ROC_ENCODING_ID_MAX define allowed + * range for encoding identifiers registered by user. + * + * See \ref roc_context_register_encoding(). + */ + ROC_ENCODING_ID_MIN = 100, + + /** Maximum allowed packet encoding id. + * + * \ref ROC_ENCODING_ID_MIN and \ref ROC_ENCODING_ID_MAX define allowed + * range for encoding identifiers registered by user. + * + * See \ref roc_context_register_encoding(). + */ + ROC_ENCODING_ID_MAX = 127 +}; + +enum { + /** Minumum allowed plugin id. + * + * \ref ROC_PLUGIN_ID_MIN and \ref ROC_PLUGIN_ID_MAX define allowed + * range for plugin identifiers registered by user. + * + * See roc_context_register_plc(). + */ + ROC_PLUGIN_ID_MIN = 1000, + + /** Maximum allowed plugin id. + * + * \ref ROC_PLUGIN_ID_MIN and \ref ROC_PLUGIN_ID_MAX define allowed + * range for plugin identifiers registered by user. + * + * See roc_context_register_plc(). + */ + ROC_PLUGIN_ID_MAX = 9999 +}; + +/** PLC backend plugin. + * + * Packet loss concealment (PLC) is used to reduce distortion caused by lost packets + * by filling gaps with interpolated or extrapolated data. It is used only when FEC + * was not able to restore the packets. + * + * **Life cycle** + * + * PLC plugin is instantiated on receiver for every incoming connection from sender. + * + * This struct defines plugin callback table. For every connection, new_cb() is + * invoked to create a new plugin instance, and then other callbacks are invoked on the + * instance. When the connection is closed, delete_cb() is invoked to destroy instance. + * + * Multiple plugin instances may co-exist if there are multiple connections. + * + * **Workflow** + * + * When it's time to produce next frame (e.g. to be played on sound card), receiver calls + * one of the two callbacks of the plugin instance: + * + * - When the frame is successfully decoded from packet(s), receiver invokes + * process_history_cb(). Plugin may copy data from the frame and remember + * it for later use. + * + * - When the frame is a gap caused by lost packet(s), receiver invokes + * process_loss_cb(). Plugin must fill the provided frame with the + * interpolated data. + * + * If lookahead_len_cb() returns non-zero, process_loss_cb() will be provided + * with the frame following the lost one, if it is available. + * + * **Registration** + * + * PLC plugin should be registered using roc_context_register_plc() and then + * enabled using \c plc_backend field of \ref roc_receiver_config. + * + * Plugin callback table is not copied, but is stored by reference inside \ref + * roc_context. The callback table should remain valid and immutable until the + * context is closed. + * + * **Thread-safety** + * + * Plugin callback table may be accessed from multiple threads concurrently and its + * callbacks may be invoked concurrently. However, calls on the same plugin instance + * returned from new_cb() are always serialized. Besides new_cb(), only calls + * on different instances may happen concurrently. + */ +typedef struct roc_plugin_plc { + /** Callback to create plugin instance. + * + * Invoked on receiver to create a plugin instance for a new connection. + * Returned pointer is opaque. It is used as the argument to other callbacks. + * + * **Parameters** + * - \p plugin is a pointer to plugin callback table passed to + * roc_context_register_plc() + */ + void* (*new_cb)(struct roc_plugin_plc* plugin); + + /** Callback to delete plugin instance. + * + * Invoked on receiver to destroy a plugin instance created by new_cb(). + * + * **Parameters** + * - \p plugin_instance is a pointer to the instance returned by new_cb() + */ + void (*delete_cb)(void* plugin_instance); + + /** Obtain PLC look-ahead length, as number of samples per channel. + * + * Returned value defines how many samples following immediately after the lost frame + * PLC wants to use for interpolation. See process_loss_cb() for details. + * + * **Parameters** + * - \p plugin_instance is a pointer to the instance returned by new_cb() + */ + unsigned int (*lookahead_len_cb)(void* plugin_instance); + + /** Callback for frames without losses. + * + * Invoked on receiver when next frame was successfully decoded from packets. + * If plugin wants to store frame for later use, it should copy its samples. + * + * The size of \p history_frame is arbitrary and may vary each call. The format of + * the frame is defined by \c frame_encoding field of \ref roc_receiver_config. + * + * **Parameters** + * - \p plugin_instance is a pointer to the instance returned by new_cb() + * - \p history_frame is a pointer to a read-only frame with decoded data + * + * **Ownership** + * - frame and its data can't be used after the callback returns + */ + void (*process_history_cb)(void* plugin_instance, const roc_frame* history_frame); + + /** Callback for frames with losses. + * + * Invoked on receiver when next frame is a gap caused by packet loss. + * Plugin must fill \p lost_frame with the interpolated data. + * Plugin must not change buffer and size of \p lost_frame, it is expected to + * write samples into existing buffer. + * + * If lookahead_len_cb() returned non-zero length, \p lookahead_frame holds up + * to that many samples, decoded from packets that follow the loss. + * \p lookahead_frame may be shorter than look-ahead length and may be empty. + * It's present only if packets following the loss happened to arrive early enough. + * + * The size of both frames is arbitrary and may vary each call. The format of the + * frames is defined by \c frame_encoding field of \ref roc_receiver_config. + * + * **Parameters** + * - \p plugin_instance is a pointer to the instance returned by new_cb() + * - \p lost_frame is a pointer to a writable frame to be filled with the + * interpolated data + * - \p lookahead_frame is a pointer to a read-only frame following the + * lost one + * + * **Ownership** + * - frames and their data can't be used after the callback returns + */ + void (*process_loss_cb)(void* plugin_instance, + roc_frame* lost_frame, + const roc_frame* lookahead_frame); +} roc_plugin_plc; + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* ROC_PLUGIN_H_ */ diff --git a/src/public_api/src/adapters.cpp b/src/public_api/src/adapters.cpp index 805846c6d..c57bbf323 100644 --- a/src/public_api/src/adapters.cpp +++ b/src/public_api/src/adapters.cpp @@ -6,6 +6,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "roc/plugin.h" + #include "adapters.h" #include "roc_address/interface.h" @@ -65,7 +67,8 @@ bool sender_config_from_user(node::Context& context, if (!packet_encoding_from_user(out.payload_type, in.packet_encoding)) { roc_log(LogError, "bad configuration: invalid roc_sender_config.packet_encoding:" - " should be zero or valid encoding id"); + " should be either zero, or valid enum value," + " or belong to the range [ROC_ENCODING_ID_MIN; ROC_ENCODING_ID_MAX]"); return false; } const rtp::Encoding* encoding = @@ -230,6 +233,14 @@ bool receiver_config_from_user(node::Context&, return false; } + if (!plc_backend_from_user(out.session_defaults.plc.backend, in.plc_backend)) { + roc_log(LogError, + "bad configuration: invalid roc_receiver_config.plc_backend:" + " should be either valid enum value, " + " or belong to the range [ROC_PLUGIN_ID_MIN; ROC_PLUGIN_ID_MAX]"); + return false; + } + return true; } @@ -473,19 +484,43 @@ bool resampler_profile_from_user(audio::ResamplerProfile& out, roc_resampler_pro } ROC_ATTR_NO_SANITIZE_UB -bool packet_encoding_from_user(unsigned& out_pt, roc_packet_encoding in) { +bool plc_backend_from_user(int& out_id, roc_plc_backend in) { + switch (enum_from_user(in)) { + case ROC_PLC_BACKEND_DISABLE: + out_id = audio::PlcBackend_None; + return true; + + case ROC_PLC_BACKEND_DEFAULT: + out_id = audio::PlcBackend_Default; + return true; + } + + if ((int)in >= (int)ROC_PLUGIN_ID_MIN && (int)in <= (int)ROC_PLUGIN_ID_MAX) { + out_id = (int)in; + return true; + } + + return false; +} + +ROC_ATTR_NO_SANITIZE_UB +bool packet_encoding_from_user(unsigned& out_id, roc_packet_encoding in) { switch (enum_from_user(in)) { case ROC_PACKET_ENCODING_AVP_L16_MONO: - out_pt = rtp::PayloadType_L16_Mono; + out_id = rtp::PayloadType_L16_Mono; return true; case ROC_PACKET_ENCODING_AVP_L16_STEREO: - out_pt = rtp::PayloadType_L16_Stereo; + out_id = rtp::PayloadType_L16_Stereo; return true; } - out_pt = in; - return true; + if ((int)in >= (int)ROC_ENCODING_ID_MIN && (int)in <= (int)ROC_ENCODING_ID_MAX) { + out_id = (unsigned)in; + return true; + } + + return false; } ROC_ATTR_NO_SANITIZE_UB diff --git a/src/public_api/src/adapters.h b/src/public_api/src/adapters.h index 1cf27fab4..e4da26b12 100644 --- a/src/public_api/src/adapters.h +++ b/src/public_api/src/adapters.h @@ -53,6 +53,8 @@ bool latency_tuner_profile_from_user(audio::LatencyTunerProfile& out, bool resampler_backend_from_user(audio::ResamplerBackend& out, roc_resampler_backend in); bool resampler_profile_from_user(audio::ResamplerProfile& out, roc_resampler_profile in); +bool plc_backend_from_user(int& out, roc_plc_backend in); + bool packet_encoding_from_user(unsigned& out_pt, roc_packet_encoding in); bool fec_encoding_from_user(packet::FecScheme& out, roc_fec_encoding in); diff --git a/src/public_api/src/context.cpp b/src/public_api/src/context.cpp index 631c485f4..2c5b8b7de 100644 --- a/src/public_api/src/context.cpp +++ b/src/public_api/src/context.cpp @@ -10,6 +10,7 @@ #include "adapters.h" #include "arena.h" +#include "plugin_plc.h" #include "roc_audio/pcm_decoder.h" #include "roc_audio/pcm_encoder.h" @@ -108,6 +109,46 @@ int roc_context_register_encoding(roc_context* context, return 0; } +int roc_context_register_plc(roc_context* context, + int plugin_id, + roc_plugin_plc* plugin) { + if (!context) { + roc_log(LogError, + "roc_context_register_plc(): invalid arguments:" + " context is null"); + return -1; + } + + if (plugin_id < ROC_PLUGIN_ID_MIN || plugin_id > ROC_PLUGIN_ID_MAX) { + roc_log(LogError, + "roc_context_register_plc(): invalid arguments:" + " plugin_id out of range: value=%d range=[%d; %d]", + plugin_id, ROC_PLUGIN_ID_MIN, ROC_PLUGIN_ID_MAX); + return -1; + } + + if (!api::PluginPlc::validate(plugin)) { + roc_log(LogError, + "roc_context_register_plc(): invalid arguments:" + " invalid function table"); + return -1; + } + + node::Context* imp_context = (node::Context*)context; + + const status::StatusCode code = imp_context->processor_map().register_plc( + plugin_id, plugin, &api::PluginPlc::construct); + + if (code != status::StatusOK) { + roc_log(LogError, + "roc_context_register_plc(): failed to register encoding: status=%s", + status::code_to_str(code)); + return -1; + } + + return 0; +} + int roc_context_close(roc_context* context) { if (!context) { roc_log(LogError, "roc_context_close(): invalid arguments: context is null"); diff --git a/src/public_api/src/plugin_plc.cpp b/src/public_api/src/plugin_plc.cpp new file mode 100644 index 000000000..4376301f1 --- /dev/null +++ b/src/public_api/src/plugin_plc.cpp @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2024 Roc Streaming authors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include "plugin_plc.h" + +#include "roc_core/log.h" +#include "roc_core/panic.h" + +namespace roc { +namespace api { + +bool PluginPlc::validate(roc_plugin_plc* plugin) { + if (!plugin) { + roc_log(LogError, "roc_plugin_plc: callback table is null"); + return false; + } + + if (!plugin->new_cb) { + roc_log(LogError, "roc_plugin_plc: new_cb is null"); + return false; + } + + if (!plugin->delete_cb) { + roc_log(LogError, "roc_plugin_plc: delete_cb is null"); + return false; + } + + if (!plugin->lookahead_len_cb) { + roc_log(LogError, "roc_plugin_plc: lookahead_len_cb is null"); + return false; + } + + if (!plugin->process_history_cb) { + roc_log(LogError, "roc_plugin_plc: process_history_cb is null"); + return false; + } + + if (!plugin->process_loss_cb) { + roc_log(LogError, "roc_plugin_plc: process_loss_cb is null"); + return false; + } + + return true; +} + +audio::IPlc* PluginPlc::construct(void* plugin, + core::IArena& arena, + audio::FrameFactory& frame_factory, + const audio::PlcConfig& config, + const audio::SampleSpec& sample_spec) { + return new (arena) PluginPlc((roc_plugin_plc*)plugin, sample_spec); +} + +PluginPlc::PluginPlc(roc_plugin_plc* plugin, const audio::SampleSpec& sample_spec) + : plugin_(plugin) + , plugin_instance_(NULL) + , sample_spec_(sample_spec) { + roc_panic_if(!plugin_); + roc_panic_if(!validate(plugin_)); + + plugin_instance_ = plugin_->new_cb(plugin_); + if (!plugin_instance_) { + roc_log( + LogError, + "roc_plugin_plc: failed to create plugin instance: new_cb() returned null"); + } +} + +PluginPlc::~PluginPlc() { + if (plugin_instance_) { + plugin_->delete_cb(plugin_instance_); + } +} + +status::StatusCode PluginPlc::init_status() const { + if (!plugin_instance_) { + return status::StatusNoPlugin; + } + return status::StatusOK; +} + +audio::SampleSpec PluginPlc::sample_spec() const { + return sample_spec_; +} + +packet::stream_timestamp_t PluginPlc::lookbehind_len() { + roc_panic_if(!plugin_); + roc_panic_if(!plugin_instance_); + + // PluginPlc doesn't need prev_frame, because this feature is not exposed + // via C API to keep the interface simple. Users can implement ring buffer + // by themselves. + + return 0; +} + +packet::stream_timestamp_t PluginPlc::lookahead_len() { + roc_panic_if(!plugin_); + roc_panic_if(!plugin_instance_); + + return (packet::stream_timestamp_t)plugin_->lookahead_len_cb(plugin_instance_); +} + +void PluginPlc::process_history(audio::Frame& imp_hist_frame) { + roc_panic_if(!plugin_); + roc_panic_if(!plugin_instance_); + + if (!plugin_->process_history_cb) { + return; + } + + roc_frame hist_frame; + memset(&hist_frame, 0, sizeof(hist_frame)); + + sample_spec_.validate_frame(imp_hist_frame); + hist_frame.samples = imp_hist_frame.bytes(); + hist_frame.samples_size = imp_hist_frame.num_bytes(); + + plugin_->process_history_cb(plugin_instance_, &hist_frame); +} + +void PluginPlc::process_loss(audio::Frame& imp_lost_frame, + audio::Frame* imp_prev_frame, + audio::Frame* imp_next_frame) { + roc_panic_if(!plugin_); + roc_panic_if(!plugin_instance_); + + roc_frame lost_frame; + roc_frame next_frame; + + memset(&lost_frame, 0, sizeof(lost_frame)); + memset(&next_frame, 0, sizeof(next_frame)); + + sample_spec_.validate_frame(imp_lost_frame); + lost_frame.samples = imp_lost_frame.bytes(); + lost_frame.samples_size = imp_lost_frame.num_bytes(); + + if (imp_next_frame) { + sample_spec_.validate_frame(*imp_next_frame); + next_frame.samples = imp_next_frame->bytes(); + next_frame.samples_size = imp_next_frame->num_bytes(); + } + + plugin_->process_loss_cb(plugin_instance_, &lost_frame, &next_frame); +} + +} // namespace api +} // namespace roc diff --git a/src/public_api/src/plugin_plc.h b/src/public_api/src/plugin_plc.h new file mode 100644 index 000000000..57f95c1d9 --- /dev/null +++ b/src/public_api/src/plugin_plc.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2024 Roc Streaming authors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#ifndef ROC_PUBLIC_API_PLUGIN_PLC_H_ +#define ROC_PUBLIC_API_PLUGIN_PLC_H_ + +#include "roc/plugin.h" + +#include "roc_audio/iplc.h" +#include "roc_audio/plc_config.h" + +namespace roc { +namespace api { + +//! PLC backend using roc_plugin_plc function table. +class PluginPlc : public audio::IPlc { +public: + //! Valid plugin function table. + static bool validate(roc_plugin_plc* plugin); + + //! Construction function. + //! @p plugin is roc_plugin_plc. + static IPlc* construct(void* plugin, + core::IArena& arena, + audio::FrameFactory& frame_factory, + const audio::PlcConfig& config, + const audio::SampleSpec& sample_spec); + + //! Initialize. + PluginPlc(roc_plugin_plc* plugin, const audio::SampleSpec& sample_spec); + + virtual ~PluginPlc(); + + //! Check if the object was successfully constructed. + virtual status::StatusCode init_status() const; + + //! Sample specification expected by PLC. + virtual audio::SampleSpec sample_spec() const; + + //! How many samples before lost frame are needed for interpolation. + virtual packet::stream_timestamp_t lookbehind_len(); + + //! How many samples after lost frame are needed for interpolation. + virtual packet::stream_timestamp_t lookahead_len(); + + //! When next frame has no losses, PLC reader calls this method. + virtual void process_history(audio::Frame& hist_frame); + + //! When next frame is lost, PLC reader calls this method. + virtual void process_loss(audio::Frame& lost_frame, + audio::Frame* prev_frame, + audio::Frame* next_frame); + +private: + roc_plugin_plc* plugin_; + void* plugin_instance_; + const audio::SampleSpec sample_spec_; +}; + +} // namespace api +} // namespace roc + +#endif // ROC_PUBLIC_API_PLUGIN_PLC_H_ diff --git a/src/tests/public_api/test_helpers/context.h b/src/tests/public_api/test_helpers/context.h index f59fa2da6..1edf1417e 100644 --- a/src/tests/public_api/test_helpers/context.h +++ b/src/tests/public_api/test_helpers/context.h @@ -55,6 +55,10 @@ class Context : public core::NonCopyable<> { CHECK(roc_context_register_encoding(ctx_, encoding_id, &encoding) == 0); } + void register_plc_plugin(int plugin_id, roc_plugin_plc* plugin) { + CHECK(roc_context_register_plc(ctx_, plugin_id, plugin) == 0); + } + private: roc_context* ctx_; }; diff --git a/src/tests/public_api/test_plugin_plc.cpp b/src/tests/public_api/test_plugin_plc.cpp new file mode 100644 index 000000000..143b1b419 --- /dev/null +++ b/src/tests/public_api/test_plugin_plc.cpp @@ -0,0 +1,318 @@ +/* + * Copyright (c) 2024 Roc Streaming authors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include + +#include "test_helpers/context.h" +#include "test_helpers/proxy.h" +#include "test_helpers/receiver.h" +#include "test_helpers/sender.h" + +#include "roc_core/atomic.h" +#include "roc_core/panic.h" +#include "roc_fec/codec_map.h" + +#include "roc/config.h" +#include "roc/plugin.h" + +namespace roc { +namespace api { + +namespace { + +enum { + Magic = 123456789, + NumChans = 2, + LookaheadSamples = 10, + PluginID = ROC_PLUGIN_ID_MIN + 10 +}; + +const float SampleStep = 1. / 32768.; + +struct TestPlugin { + // First field, so we can cast roc_plugin_plc* to TestPlugin*. + roc_plugin_plc func_table; + + // Magic constant to ensure that all pointer casts work fine. + int magic; + + core::Atomic n_created; + core::Atomic n_deleted; + + core::Atomic n_hist_samples; + core::Atomic n_lost_samples; + + TestPlugin(); +}; + +struct TestPlc { + TestPlugin* plugin; + float last_sample; + + TestPlc(TestPlugin* plugin) + : plugin(plugin) + , last_sample(0) { + roc_panic_if_not(plugin); + roc_panic_if_not(plugin->magic == Magic); + + plugin->n_created++; + } + + ~TestPlc() { + roc_panic_if_not(plugin); + roc_panic_if_not(plugin->magic == Magic); + + plugin->n_deleted++; + } +}; + +void* test_plc_new(roc_plugin_plc* plugin) { + roc_panic_if_not(plugin); + + return new TestPlc((TestPlugin*)plugin); +} + +void test_plc_delete(void* plugin_instance) { + TestPlc* plc = (TestPlc*)plugin_instance; + + roc_panic_if_not(plc); + roc_panic_if_not(plc->plugin); + roc_panic_if_not(plc->plugin->magic == Magic); + + delete plc; +} + +unsigned int test_plc_lookahead_len(void* plugin_instance) { + TestPlc* plc = (TestPlc*)plugin_instance; + + roc_panic_if_not(plc); + roc_panic_if_not(plc->plugin); + roc_panic_if_not(plc->plugin->magic == Magic); + + return LookaheadSamples; +} + +void test_plc_process_history(void* plugin_instance, const roc_frame* history_frame) { + TestPlc* plc = (TestPlc*)plugin_instance; + + roc_panic_if_not(plc); + roc_panic_if_not(plc->plugin); + roc_panic_if_not(plc->plugin->magic == Magic); + + roc_panic_if_not(history_frame); + roc_panic_if_not(history_frame->samples != NULL); + roc_panic_if_not(history_frame->samples_size > 0); + + const float* hist_samples = (const float*)history_frame->samples; + const size_t hist_sample_count = + history_frame->samples_size / sizeof(float) / NumChans; + + plc->last_sample = hist_samples[hist_sample_count * NumChans - 1]; + + // update stats shared by all plugin instances + plc->plugin->n_hist_samples += hist_sample_count; +} + +void test_plc_process_loss(void* plugin_instance, + roc_frame* lost_frame, + const roc_frame* lookahead_frame) { + TestPlc* plc = (TestPlc*)plugin_instance; + + roc_panic_if_not(plc); + roc_panic_if_not(plc->plugin); + roc_panic_if_not(plc->plugin->magic == Magic); + + roc_panic_if_not(lost_frame); + roc_panic_if_not(lost_frame->samples != NULL); + roc_panic_if_not(lost_frame->samples_size > 0); + + roc_panic_if_not(lookahead_frame); + roc_panic_if_not( + (lost_frame->samples != NULL && lost_frame->samples_size > 0) + || (lookahead_frame->samples == NULL && lookahead_frame->samples_size == 0)); + + float* lost_samples = (float*)lost_frame->samples; + const size_t lost_sample_count = lost_frame->samples_size / sizeof(float) / NumChans; + + for (size_t ns = 0; ns < lost_sample_count; ns += NumChans) { + // test::Sender generates an incrementing sequence of samples, so + // we can easily restore original samples. + plc->last_sample = test::increment_sample_value(plc->last_sample, SampleStep); + + for (size_t nc = 0; nc < NumChans; nc++) { + lost_samples[ns + nc] = plc->last_sample; + } + } + + if (lookahead_frame->samples_size > 0) { + // check that lost frame fit perfectly + const float* lookahead_samples = (const float*)lookahead_frame->samples; + roc_panic_if_not(lookahead_samples[0] != plc->last_sample); + } + + // update stats shared by all plugin instances + plc->plugin->n_lost_samples += lost_sample_count; +} + +TestPlugin::TestPlugin() { + magic = Magic; + + func_table.new_cb = &test_plc_new; + func_table.delete_cb = &test_plc_delete; + func_table.lookahead_len_cb = &test_plc_lookahead_len; + func_table.process_history_cb = &test_plc_process_history; + func_table.process_loss_cb = &test_plc_process_loss; +} + +} // namespace + +TEST_GROUP(plugin_plc) { + roc_sender_config sender_conf; + roc_receiver_config receiver_conf; + + void setup() { + memset(&sender_conf, 0, sizeof(sender_conf)); + sender_conf.frame_encoding.rate = test::SampleRate; + sender_conf.frame_encoding.format = ROC_FORMAT_PCM_FLOAT32; + sender_conf.frame_encoding.channels = ROC_CHANNEL_LAYOUT_STEREO; + + sender_conf.packet_encoding = ROC_PACKET_ENCODING_AVP_L16_STEREO; + sender_conf.packet_length = + test::PacketSamples * 1000000000ull / test::SampleRate; + + sender_conf.fec_encoding = ROC_FEC_ENCODING_RS8M; + sender_conf.fec_block_source_packets = test::SourcePackets; + sender_conf.fec_block_repair_packets = test::RepairPackets; + + sender_conf.clock_source = ROC_CLOCK_SOURCE_INTERNAL; + + memset(&receiver_conf, 0, sizeof(receiver_conf)); + receiver_conf.frame_encoding.rate = test::SampleRate; + receiver_conf.frame_encoding.format = ROC_FORMAT_PCM_FLOAT32; + receiver_conf.frame_encoding.channels = ROC_CHANNEL_LAYOUT_STEREO; + + receiver_conf.clock_source = ROC_CLOCK_SOURCE_INTERNAL; + + // enable PLC plugin + receiver_conf.plc_backend = (roc_plc_backend)PluginID; + + receiver_conf.latency_tuner_profile = ROC_LATENCY_TUNER_PROFILE_INTACT; + receiver_conf.target_latency = test::Latency * 1000000000ull / test::SampleRate; + receiver_conf.no_playback_timeout = + test::Timeout * 1000000000ull / test::SampleRate; + } + + bool is_rs8m_supported() { + return fec::CodecMap::instance().has_scheme(packet::FEC_ReedSolomon_M8); + } +}; + +// Enable FEC + PLC (custom plugin). +// Lose some source packets. +// Check that all all packets were restored by FEC and not by PLC. +TEST(plugin_plc, losses_restored_by_fec) { + if (!is_rs8m_supported()) { + return; + } + + enum { Flags = test::FlagRS8M | test::FlagLoseSomePkts }; + + TestPlugin plugin; + + { + test::Context context; + + // register PLC plugin + context.register_plc_plugin(PluginID, &plugin.func_table); + + test::Receiver receiver(context, receiver_conf, SampleStep, NumChans, + test::FrameSamples, Flags); + + receiver.bind(); + + test::Proxy proxy(receiver.source_endpoint(), receiver.repair_endpoint(), + test::SourcePackets, test::RepairPackets, Flags); + + test::Sender sender(context, sender_conf, SampleStep, NumChans, + test::FrameSamples, Flags); + + sender.connect(proxy.source_endpoint(), proxy.repair_endpoint(), NULL); + + CHECK(sender.start()); + receiver.receive(); + sender.stop(); + sender.join(); + + // some packets were lost + CHECK(proxy.n_dropped_packets() > 0); + } + + // one plugin instance was created and deleted + LONGS_EQUAL(1, plugin.n_created); + LONGS_EQUAL(1, plugin.n_deleted); + + // PLC got history frames + CHECK(plugin.n_hist_samples > 0); + // but PLC was not asked to fill losses + CHECK(plugin.n_lost_samples == 0); +} + +// Enable FEC + PLC (custom plugin). +// Lose some source packets + lose all repair packets. +// Check that PLC was used to restore packets. +TEST(plugin_plc, losses_restored_by_plc) { + if (!is_rs8m_supported()) { + return; + } + + enum { + Flags = test::FlagRS8M | test::FlagLoseSomePkts | test::FlagLoseAllRepairPkts + }; + + TestPlugin plugin; + + { + test::Context context; + + // register PLC plugin + context.register_plc_plugin(PluginID, &plugin.func_table); + + test::Receiver receiver(context, receiver_conf, SampleStep, NumChans, + test::FrameSamples, Flags); + + receiver.bind(); + + test::Proxy proxy(receiver.source_endpoint(), receiver.repair_endpoint(), + test::SourcePackets, test::RepairPackets, Flags); + + test::Sender sender(context, sender_conf, SampleStep, NumChans, + test::FrameSamples, Flags); + + sender.connect(proxy.source_endpoint(), proxy.repair_endpoint(), NULL); + + CHECK(sender.start()); + receiver.receive(); + sender.stop(); + sender.join(); + + // some packets were lost + CHECK(proxy.n_dropped_packets() > 0); + } + + // one plugin instance was created and deleted + LONGS_EQUAL(1, plugin.n_created); + LONGS_EQUAL(1, plugin.n_deleted); + + // PLC got history frames + CHECK(plugin.n_hist_samples > 0); + // and PLC was asked to fill losses + CHECK(plugin.n_lost_samples > 0); +} + +} // namespace api +} // namespace roc