From d18d34269f4320066e7a4bbf11f1e13b666ffa98 Mon Sep 17 00:00:00 2001 From: Victor Gaydov Date: Sun, 28 Jan 2024 21:29:06 +0400 Subject: [PATCH] gh-675 Rework latency tuning - Extract LatencyTuner to be reused from LatencyMonitor and FeedbackMonitor - Also reuse LatencyConfig and LatencyMetrics - Refine LatencyConfig defaults - Use LinkMeter in LatencyMonitor to obtain jitter - Latency backend can't be disabled; there is now always a valid backend selected, however latency *tuning* can be disabled via latency tuner profile - Receiver reports NIQ length and stalling to sender via a non-standard RTCP XR report - API: replace roc_clock_sync_backend and roc_clock_sync_profile with roc_latency_tuner_backend and roc_latency_tuner_profile - API: add roc_latency_tuner_backend, roc_latency_tuner_profile, target_latency, latency_tolerance to sender - CLI: replace --clock-backend and --clock-profile with --latency-backend and --latency-profile - CLI: rename --sess-latency to --target-latency - CLI: add --latency-backend, --latency-profile, --target-latency, --latency-tolerance to sender --- docs/man/roc-copy.1 | 4 +- docs/man/roc-recv.1 | 24 +- docs/man/roc-send.1 | 34 +- docs/sphinx/api/reference.rst | 4 +- docs/sphinx/manuals/roc_recv.rst | 14 +- docs/sphinx/manuals/roc_send.rst | 16 + .../roc_audio/feedback_monitor.cpp | 255 ++++------- .../roc_audio/feedback_monitor.h | 114 ++--- .../roc_audio/freq_estimator.cpp | 38 +- .../roc_audio/freq_estimator.h | 28 +- .../roc_audio/latency_config.cpp | 50 --- .../roc_audio/latency_config.h | 62 --- .../roc_audio/latency_monitor.cpp | 318 ++++--------- .../roc_audio/latency_monitor.h | 95 ++-- .../roc_audio/latency_tuner.cpp | 425 ++++++++++++++++++ .../roc_audio/latency_tuner.h | 237 ++++++++++ .../roc_audio/resampler_config.cpp | 7 +- .../roc_audio/resampler_config.h | 5 +- src/internal_modules/roc_audio/watchdog.cpp | 6 +- .../target_libuv/roc_netio/udp_port.cpp | 8 +- src/internal_modules/roc_packet/packet.cpp | 8 + src/internal_modules/roc_packet/packet.h | 5 + .../target_libuv/roc_packet/udp.cpp | 21 + .../roc_packet/target_libuv/roc_packet/udp.h | 6 + src/internal_modules/roc_pipeline/config.cpp | 18 +- src/internal_modules/roc_pipeline/config.h | 6 +- src/internal_modules/roc_pipeline/metrics.h | 7 +- .../roc_pipeline/receiver_session.cpp | 7 +- .../roc_pipeline/sender_session.cpp | 19 +- .../roc_rtcp/communicator.cpp | 52 ++- src/internal_modules/roc_rtcp/communicator.h | 8 +- src/internal_modules/roc_rtcp/headers.h | 64 ++- .../roc_rtcp/print_packet.cpp | 13 +- src/internal_modules/roc_rtcp/reporter.cpp | 19 +- src/internal_modules/roc_rtcp/reports.h | 5 + src/public_api/include/roc/config.h | 263 ++++++++--- src/public_api/src/adapters.cpp | 62 +-- src/public_api/src/adapters.h | 8 +- .../test_sender_encoder_receiver_decoder.cpp | 2 +- src/tests/public_api/test_sender_receiver.cpp | 2 +- .../test_loopback_sink_2_source.cpp | 3 +- src/tests/roc_pipeline/test_receiver_loop.cpp | 3 +- .../roc_pipeline/test_receiver_source.cpp | 3 +- src/tests/roc_pipeline/test_sender_loop.cpp | 3 +- src/tests/roc_pipeline/test_sender_sink.cpp | 3 + src/tests/roc_rtcp/test_builder_traverser.cpp | 10 +- src/tests/roc_rtcp/test_communicator.cpp | 7 +- src/tests/roc_rtcp/test_headers.cpp | 105 +++-- src/tests/roc_rtcp/test_traverser.cpp | 10 +- src/tools/roc_recv/cmdline.ggo | 10 +- src/tools/roc_recv/main.cpp | 42 +- src/tools/roc_send/cmdline.ggo | 12 + src/tools/roc_send/main.cpp | 38 ++ 53 files changed, 1588 insertions(+), 1000 deletions(-) delete mode 100644 src/internal_modules/roc_audio/latency_config.cpp delete mode 100644 src/internal_modules/roc_audio/latency_config.h create mode 100644 src/internal_modules/roc_audio/latency_tuner.cpp create mode 100644 src/internal_modules/roc_audio/latency_tuner.h create mode 100644 src/internal_modules/roc_packet/target_libuv/roc_packet/udp.cpp diff --git a/docs/man/roc-copy.1 b/docs/man/roc-copy.1 index 79f3c0847..b33246699 100644 --- a/docs/man/roc-copy.1 +++ b/docs/man/roc-copy.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "ROC-COPY" "1" "2023" "Roc Toolkit 0.3" "Roc Toolkit" +.TH "ROC-COPY" "1" "2024" "Roc Toolkit 0.3" "Roc Toolkit" .SH NAME roc-copy \- copy local audio .SH SYNOPSIS @@ -187,6 +187,6 @@ Please report any bugs found via GitHub (\fI\%https://github.com/roc\-streaming/ .sp See authors page on the website for a list of maintainers and contributors (\fI\%https://roc\-streaming.org/toolkit/docs/about_project/authors.html\fP). .SH COPYRIGHT -2023, Roc Streaming authors +2024, Roc Streaming authors .\" Generated by docutils manpage writer. . diff --git a/docs/man/roc-recv.1 b/docs/man/roc-recv.1 index d2a1b88af..4c73feab0 100644 --- a/docs/man/roc-recv.1 +++ b/docs/man/roc-recv.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "ROC-RECV" "1" "2023" "Roc Toolkit 0.3" "Roc Toolkit" +.TH "ROC-RECV" "1" "2024" "Roc Toolkit 0.3" "Roc Toolkit" .SH NAME roc-recv \- receive real-time audio .SH SYNOPSIS @@ -78,8 +78,8 @@ IPv4 or IPv6 address of the network interface on which to join the multicast gro .B \-\-reuseaddr enable SO_REUSEADDR when binding sockets .TP -.BI \-\-sess\-latency\fB= STRING -Session target latency, TIME units +.BI \-\-target\-latency\fB= STRING +Target latency, TIME units .TP .BI \-\-io\-latency\fB= STRING Playback target latency, TIME units @@ -105,11 +105,11 @@ Maximum internal frame size, in SIZE units .BI \-\-rate\fB= INT Override output sample rate, Hz .TP -.BI \-\-clock\-backend\fB= ENUM -Clock synchronization backend (possible values=\(dqdisable\(dq, \(dqniq\(dq default=\(ganiq\(aq) +.BI \-\-latency\-backend\fB= ENUM +Which latency to use in latency tuner (possible values=\(dqniq\(dq default=\(ganiq\(aq) .TP -.BI \-\-clock\-profile\fB= ENUM -Clock synchronization profile (possible values=\(dqdefault\(dq, \(dqresponsive\(dq, \(dqgradual\(dq default=\(gadefault\(aq) +.BI \-\-latency\-profile\fB= ENUM +Latency tuning profile (possible values=\(dqdefault\(dq, \(dqresponsive\(dq, \(dqgradual\(dq, \(dqintact\(dq default=\(gadefault\(aq) .TP .BI \-\-resampler\-backend\fB= ENUM Resampler backend (possible values=\(dqdefault\(dq, \(dqbuiltin\(dq, \(dqspeex\(dq, \(dqspeexdec\(dq default=\(gadefault\(aq) @@ -480,7 +480,7 @@ Select lower session latency: .sp .nf .ft C -$ roc\-recv \-vv \-s rtp://0.0.0.0:10001 \-\-sess\-latency=50ms +$ roc\-recv \-vv \-s rtp://0.0.0.0:10001 \-\-target\-latency=50ms .ft P .fi .UNINDENT @@ -506,7 +506,7 @@ Manually specify thresholds and timeouts: .nf .ft C $ roc\-recv \-vv \-s rtp://0.0.0.0:10001 \e - \-\-sess\-latency=50ms \-\-latency\-tolerance=20ms \e + \-\-target\-latency=50ms \-\-latency\-tolerance=20ms \e \-\-no\-play\-timeout=200s \-\-choppy\-play\-timeout=500ms .ft P .fi @@ -526,14 +526,14 @@ $ roc\-recv \-vv \-s rtp://0.0.0.0:10001 \e .UNINDENT .UNINDENT .sp -Manually specify clock synchronization parameters: +Manually specify latency tuning parameters: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C $ roc\-recv \-vv \-s rtp://0.0.0.0:10001 \e - \-\-clock\-backend=niq \-\-clock\-profile=gradual + \-\-latency\-backend=niq \-\-latency\-profile=gradual .ft P .fi .UNINDENT @@ -559,6 +559,6 @@ Please report any bugs found via GitHub (\fI\%https://github.com/roc\-streaming/ .sp See authors page on the website for a list of maintainers and contributors (\fI\%https://roc\-streaming.org/toolkit/docs/about_project/authors.html\fP). .SH COPYRIGHT -2023, Roc Streaming authors +2024, Roc Streaming authors .\" Generated by docutils manpage writer. . diff --git a/docs/man/roc-send.1 b/docs/man/roc-send.1 index d7c0a07d5..5dae9941a 100644 --- a/docs/man/roc-send.1 +++ b/docs/man/roc-send.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "ROC-SEND" "1" "2023" "Roc Toolkit 0.3" "Roc Toolkit" +.TH "ROC-SEND" "1" "2024" "Roc Toolkit 0.3" "Roc Toolkit" .SH NAME roc-send \- send real-time audio .SH SYNOPSIS @@ -69,9 +69,15 @@ Remote control endpoint .B \-\-reuseaddr enable SO_REUSEADDR when binding sockets .TP +.BI \-\-target\-latency\fB= STRING +Target latency, TIME units +.TP .BI \-\-io\-latency\fB= STRING Recording target latency, TIME units .TP +.BI \-\-latency\-tolerance\fB= STRING +Maximum latency deviation, TIME units +.TP .BI \-\-nbsrc\fB= INT Number of source packets in FEC block .TP @@ -93,6 +99,12 @@ Maximum internal frame size, in SIZE units .BI \-\-rate\fB= INT Override input sample rate, Hz .TP +.BI \-\-latency\-backend\fB= ENUM +Which latency to use in latency tuner (possible values=\(dqniq\(dq default=\(ganiq\(aq) +.TP +.BI \-\-latency\-profile\fB= ENUM +Latency tuning profile (possible values=\(dqresponsive\(dq, \(dqgradual\(dq, \(dqintact\(dq default=\(gaintact\(aq) +.TP .BI \-\-resampler\-backend\fB= ENUM Resampler backend (possible values=\(dqdefault\(dq, \(dqbuiltin\(dq, \(dqspeex\(dq, \(dqspeexdec\(dq default=\(gadefault\(aq) .TP @@ -442,6 +454,24 @@ $ roc\-send \-vv \-s rtp://192.168.0.3:10001 \e .fi .UNINDENT .UNINDENT +.sp +Perform latency tuning on sender instead of receiver: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ roc\-recv \-vv \-o pulse://default \-s rtp+rs8m://0.0.0.0:10001 \e + \-r rs8m://0.0.0.0:10002 \-c rtcp://0.0.0.0:10003 \e + \-\-latency\-profile=intact \-\-target\-latency=200ms + +$ roc\-send \-vv \-i file:./input.wav \-s rtp+rs8m://192.168.0.3:10001 \e + \-r rs8m://192.168.0.3:10002 \-c rtcp://192.168.0.3:10003 \e + \-\-latency\-profile=gradual \-\-target\-latency=200ms +.ft P +.fi +.UNINDENT +.UNINDENT .SH ENVIRONMENT VARIABLES .sp The following environment variables are supported: @@ -463,6 +493,6 @@ Please report any bugs found via GitHub (\fI\%https://github.com/roc\-streaming/ .sp See authors page on the website for a list of maintainers and contributors (\fI\%https://roc\-streaming.org/toolkit/docs/about_project/authors.html\fP). .SH COPYRIGHT -2023, Roc Streaming authors +2024, Roc Streaming authors .\" Generated by docutils manpage writer. . diff --git a/docs/sphinx/api/reference.rst b/docs/sphinx/api/reference.rst index 92bb46588..eafa106d9 100644 --- a/docs/sphinx/api/reference.rst +++ b/docs/sphinx/api/reference.rst @@ -197,9 +197,9 @@ roc_config .. doxygenenum:: roc_clock_source -.. doxygenenum:: roc_clock_sync_backend +.. doxygenenum:: roc_latency_tuner_backend -.. doxygenenum:: roc_clock_sync_profile +.. doxygenenum:: roc_latency_tuner_profile .. doxygenenum:: roc_resampler_backend diff --git a/docs/sphinx/manuals/roc_recv.rst b/docs/sphinx/manuals/roc_recv.rst index cfa3525b5..dcd7d798a 100644 --- a/docs/sphinx/manuals/roc_recv.rst +++ b/docs/sphinx/manuals/roc_recv.rst @@ -27,7 +27,7 @@ Options -c, --control=ENDPOINT_URI Local control endpoint --miface=MIFACE IPv4 or IPv6 address of the network interface on which to join the multicast group --reuseaddr enable SO_REUSEADDR when binding sockets ---sess-latency=STRING Session target latency, TIME units +--target-latency=STRING Target latency, TIME units --io-latency=STRING Playback target latency, TIME units --latency-tolerance=STRING Maximum latency deviation, TIME units --no-play-timeout=STRING No playback timeout, TIME units @@ -36,8 +36,8 @@ Options --max-packet-size=SIZE Maximum packet size, in SIZE units --max-frame-size=SIZE Maximum internal frame size, in SIZE units --rate=INT Override output sample rate, Hz ---clock-backend=ENUM Clock synchronization backend (possible values="disable", "niq" default=`niq') ---clock-profile=ENUM Clock synchronization profile (possible values="default", "responsive", "gradual" default=`default') +--latency-backend=ENUM Which latency to use in latency tuner (possible values="niq" default=`niq') +--latency-profile=ENUM Latency tuning profile (possible values="default", "responsive", "gradual", "intact" default=`default') --resampler-backend=ENUM Resampler backend (possible values="default", "builtin", "speex", "speexdec" default=`default') --resampler-profile=ENUM Resampler profile (possible values="low", "medium", "high" default=`medium') -1, --oneshot Exit when last connected client disconnects (default=off) @@ -287,7 +287,7 @@ Select lower session latency: .. code:: - $ roc-recv -vv -s rtp://0.0.0.0:10001 --sess-latency=50ms + $ roc-recv -vv -s rtp://0.0.0.0:10001 --target-latency=50ms Select lower I/O latency and frame length: @@ -301,7 +301,7 @@ Manually specify thresholds and timeouts: .. code:: $ roc-recv -vv -s rtp://0.0.0.0:10001 \ - --sess-latency=50ms --latency-tolerance=20ms \ + --target-latency=50ms --latency-tolerance=20ms \ --no-play-timeout=200s --choppy-play-timeout=500ms Manually specify resampling parameters: @@ -311,12 +311,12 @@ Manually specify resampling parameters: $ roc-recv -vv -s rtp://0.0.0.0:10001 \ --resampler-backend=speex --resampler-profile=high -Manually specify clock synchronization parameters: +Manually specify latency tuning parameters: .. code:: $ roc-recv -vv -s rtp://0.0.0.0:10001 \ - --clock-backend=niq --clock-profile=gradual + --latency-backend=niq --latency-profile=gradual ENVIRONMENT VARIABLES ===================== diff --git a/docs/sphinx/manuals/roc_send.rst b/docs/sphinx/manuals/roc_send.rst index 4a6bf79ab..56d91d189 100644 --- a/docs/sphinx/manuals/roc_send.rst +++ b/docs/sphinx/manuals/roc_send.rst @@ -24,7 +24,9 @@ Options -r, --repair=ENDPOINT_URI Remote repair endpoint -c, --control=ENDPOINT_URI Remote control endpoint --reuseaddr enable SO_REUSEADDR when binding sockets +--target-latency=STRING Target latency, TIME units --io-latency=STRING Recording target latency, TIME units +--latency-tolerance=STRING Maximum latency deviation, TIME units --nbsrc=INT Number of source packets in FEC block --nbrpr=INT Number of repair packets in FEC block --packet-len=STRING Outgoing packet length, TIME units @@ -32,6 +34,8 @@ Options --max-packet-size=SIZE Maximum packet size, in SIZE units --max-frame-size=SIZE Maximum internal frame size, in SIZE units --rate=INT Override input sample rate, Hz +--latency-backend=ENUM Which latency to use in latency tuner (possible values="niq" default=`niq') +--latency-profile=ENUM Latency tuning profile (possible values="responsive", "gradual", "intact" default=`intact') --resampler-backend=ENUM Resampler backend (possible values="default", "builtin", "speex", "speexdec" default=`default') --resampler-profile=ENUM Resampler profile (possible values="low", "medium", "high" default=`medium') --interleaving Enable packet interleaving (default=off) @@ -262,6 +266,18 @@ Manually specify resampling parameters: $ roc-send -vv -s rtp://192.168.0.3:10001 \ --resampler-backend=speex --resampler-profile=high +Perform latency tuning on sender instead of receiver: + +.. code:: + + $ roc-recv -vv -o pulse://default -s rtp+rs8m://0.0.0.0:10001 \ + -r rs8m://0.0.0.0:10002 -c rtcp://0.0.0.0:10003 \ + --latency-profile=intact --target-latency=200ms + + $ roc-send -vv -i file:./input.wav -s rtp+rs8m://192.168.0.3:10001 \ + -r rs8m://192.168.0.3:10002 -c rtcp://192.168.0.3:10003 \ + --latency-profile=gradual --target-latency=200ms + ENVIRONMENT VARIABLES ===================== diff --git a/src/internal_modules/roc_audio/feedback_monitor.cpp b/src/internal_modules/roc_audio/feedback_monitor.cpp index e3086e333..74bcd7875 100644 --- a/src/internal_modules/roc_audio/feedback_monitor.cpp +++ b/src/internal_modules/roc_audio/feedback_monitor.cpp @@ -7,86 +7,36 @@ */ #include "roc_audio/feedback_monitor.h" -#include "roc_audio/freq_estimator.h" #include "roc_core/log.h" #include "roc_core/panic.h" +#include "roc_core/time.h" namespace roc { namespace audio { -namespace { - -const core::nanoseconds_t LogInterval = 5 * core::Second; - -double timestamp_to_ms(const SampleSpec& sample_spec, - packet::stream_timestamp_diff_t timestamp) { - return (double)sample_spec.stream_timestamp_delta_2_ns(timestamp) / core::Millisecond; -} - -} // namespace - FeedbackMonitor::FeedbackMonitor(IFrameWriter& writer, ResamplerWriter* resampler, - const SampleSpec& sample_spec, - const LatencyConfig& config) - : writer_(writer) + const FeedbackConfig& feedback_config, + const LatencyConfig& latency_config, + const SampleSpec& sample_spec) + : tuner_(latency_config, sample_spec) + , metrics_() + , has_metrics_(false) + , metrics_timeout_(feedback_config.source_timeout) + , metrics_ts_(0) + , writer_(writer) , resampler_(resampler) - , stream_pos_(0) - , update_interval_( - (packet::stream_timestamp_t)sample_spec.ns_2_stream_timestamp_delta( - config.fe_update_interval)) - , update_pos_(0) - , report_interval_((packet::stream_timestamp_t) - sample_spec.ns_2_stream_timestamp_delta(LogInterval)) - , report_pos_(0) - , first_report_(true) - , freq_coeff_(0) - , fe_input_(config.fe_input) - , fe_max_delta_(config.scaling_tolerance) - , niq_latency_(0) - , e2e_latency_(0) - , has_niq_latency_(false) - , has_e2e_latency_(false) - , jitter_(0) - , has_jitter_(false) - , target_latency_(fe_input_ != audio::FreqEstimatorInput_Disable - ? sample_spec.ns_2_stream_timestamp_delta(config.target_latency) - : 0) + , enable_scaling_(latency_config.tuner_profile != audio::LatencyTunerProfile_Intact) + , source_(0) + , source_change_limiter_(feedback_config.source_cooldown) , sample_spec_(sample_spec) , started_(false) , valid_(false) { - roc_log( - LogDebug, - "feedback monitor: initializing:" - " target=%lu(%.3fms) fe_input=%s fe_profile=%s fe_interval=%.3fms", - (unsigned long)target_latency_, timestamp_to_ms(sample_spec_, target_latency_), - fe_input_to_str(config.fe_input), fe_profile_to_str(config.fe_profile), - timestamp_to_ms(sample_spec_, (packet::stream_timestamp_diff_t)update_interval_)); - - if (fe_input_ != audio::FreqEstimatorInput_Disable) { - if (config.fe_update_interval <= 0) { - roc_log(LogError, "feedback monitor: invalid config: fe_update_interval=%ld", - (long)config.fe_update_interval); - return; - } - - if (config.target_latency <= 0) { - roc_log(LogError, "feedback monitor: invalid config: target_latency=%ldns", - (long)config.target_latency); - return; - } - - if (!resampler_) { - roc_panic( - "feedback monitor: freq estimator is enabled, but resampler is null"); - } - - fe_.reset(new (fe_) FreqEstimator(config.fe_profile, - (packet::stream_timestamp_t)target_latency_)); - if (!fe_) { - return; - } + if (!tuner_.is_valid()) { + return; + } + if (enable_scaling_) { if (!init_scaling_()) { return; } @@ -103,17 +53,6 @@ bool FeedbackMonitor::is_started() const { return started_; } -FeedbackMonitorMetrics FeedbackMonitor::metrics() const { - roc_panic_if(!is_valid()); - - FeedbackMonitorMetrics metrics; - metrics.jitter = sample_spec_.stream_timestamp_delta_2_ns(jitter_); - metrics.niq_latency = sample_spec_.stream_timestamp_delta_2_ns(niq_latency_); - metrics.e2e_latency = sample_spec_.stream_timestamp_delta_2_ns(e2e_latency_); - - return metrics; -} - void FeedbackMonitor::start() { roc_panic_if(!is_valid()); @@ -121,76 +60,92 @@ void FeedbackMonitor::start() { return; } - roc_log(LogDebug, "feedback monitor: starting"); + roc_log(LogDebug, "feedback monitor: start gathering feedback"); started_ = true; } -void FeedbackMonitor::store(const FeedbackMonitorMetrics& metrics) { +void FeedbackMonitor::process_feedback(packet::stream_source_t source_id, + const LatencyMetrics& metrics) { roc_panic_if(!is_valid()); if (!started_) { return; } - if (metrics.jitter != 0) { - jitter_ = sample_spec_.ns_2_stream_timestamp_delta(metrics.jitter); - has_jitter_ = true; + if (!has_metrics_) { + roc_log(LogInfo, "feedback monitor: got first report from receiver: source=%lu", + (unsigned long)source_id); + source_ = source_id; } - if (metrics.niq_latency != 0) { - niq_latency_ = sample_spec_.ns_2_stream_timestamp_delta(metrics.niq_latency); - has_niq_latency_ = true; - } + if (has_metrics_ && source_ != source_id) { + if (!source_change_limiter_.allow()) { + // Protection from inadequately frequent SSRC changes. + // Can happen is feedback monitor is mistakenly created when multiple + // receivers exists for a single sender, which is not supported. + // This also protects from outdated reports delivered from recently + // restarted receiver. + return; + } + + roc_log(LogInfo, + "feedback monitor: detected source change:" + " old_source=%lu new_source=%lu", + (unsigned long)source_, (unsigned long)source_id); - if (metrics.e2e_latency != 0) { - e2e_latency_ = sample_spec_.ns_2_stream_timestamp_delta(metrics.e2e_latency); - has_e2e_latency_ = true; + source_ = source_id; } + + metrics_ = metrics; + has_metrics_ = true; + metrics_ts_ = core::timestamp(core::ClockMonotonic); } void FeedbackMonitor::write(Frame& frame) { roc_panic_if(!is_valid()); - if (!update_()) { - // TODO(gh-183): return status code - roc_panic("feedback monitor: update failed"); + if (started_) { + if (!update_tuner_(frame.num_samples())) { + // TODO(gh-674): change sender SSRC to restart session + } + + if (enable_scaling_) { + if (!update_scaling_()) { + // TODO(gh-183): return status code + roc_panic("feedback monitor: update failed"); + } + } } writer_.write(frame); +} - stream_pos_ += frame.num_samples() / sample_spec_.num_channels(); - - report_(); +LatencyMetrics FeedbackMonitor::metrics() const { + return metrics_; } -bool FeedbackMonitor::update_() { - if (!started_) { +bool FeedbackMonitor::update_tuner_(size_t n_samples) { + if (!has_metrics_) { return true; } - if (!fe_) { + if (core::timestamp(core::ClockMonotonic) - metrics_ts_ > metrics_timeout_) { + roc_log(LogInfo, + "feedback monitor: no reports from receiver during timeout:" + " source=%lu timeout=%.3fms", + (unsigned long)source_, (double)metrics_timeout_ / core::Millisecond); + + has_metrics_ = false; + metrics_ts_ = 0; + source_ = 0; + return true; } - switch (fe_input_) { - case audio::FreqEstimatorInput_NiqLatency: - if (has_niq_latency_) { - if (!update_scaling_(niq_latency_)) { - return false; - } - } - break; + tuner_.write_metrics(metrics_); - case audio::FreqEstimatorInput_E2eLatency: - if (has_e2e_latency_) { - if (!update_scaling_(e2e_latency_)) { - return false; - } - } - break; - - default: - break; + if (!tuner_.advance_stream(n_samples)) { + return false; } return true; @@ -207,70 +162,22 @@ bool FeedbackMonitor::init_scaling_() { return true; } -bool FeedbackMonitor::update_scaling_(packet::stream_timestamp_diff_t latency) { +bool FeedbackMonitor::update_scaling_() { roc_panic_if_not(resampler_); - roc_panic_if_not(fe_); - if (latency < 0) { - latency = 0; - } - - if (stream_pos_ < update_pos_) { - return true; - } + const float scaling = tuner_.get_scaling(); - while (stream_pos_ >= update_pos_) { - fe_->update((packet::stream_timestamp_t)latency); - update_pos_ += update_interval_; - } - - freq_coeff_ = fe_->freq_coeff(); - freq_coeff_ = std::min(freq_coeff_, 1.0f + fe_max_delta_); - freq_coeff_ = std::max(freq_coeff_, 1.0f - fe_max_delta_); - - if (!resampler_->set_scaling(freq_coeff_)) { - roc_log(LogDebug, - "feedback monitor: scaling factor out of bounds: fe=%.6f trim_fe=%.6f", - (double)fe_->freq_coeff(), (double)freq_coeff_); - return false; + if (scaling > 0) { + if (!resampler_->set_scaling(scaling)) { + roc_log(LogDebug, + "feedback monitor: scaling factor out of bounds: scaling=%.6f", + (double)scaling); + return false; + } } return true; } -void FeedbackMonitor::report_() { - if (!started_) { - return; - } - - if (!has_e2e_latency_ && !has_niq_latency_ && !has_jitter_) { - return; - } - - if (first_report_) { - roc_log(LogInfo, "feedback monitor: got first report from receiver"); - first_report_ = false; - } - - if (stream_pos_ < report_pos_) { - return; - } - - while (stream_pos_ >= report_pos_) { - report_pos_ += report_interval_; - } - - roc_log(LogDebug, - "feedback monitor:" - " e2e_latency=%ld(%.3fms) niq_latency=%ld(%.3fms) target_latency=%ld(%.3fms)" - " jitter=%ld(%.3fms)" - " fe=%.6f trim_fe=%.6f", - (long)e2e_latency_, timestamp_to_ms(sample_spec_, e2e_latency_), - (long)niq_latency_, timestamp_to_ms(sample_spec_, niq_latency_), - (long)jitter_, timestamp_to_ms(sample_spec_, jitter_), (long)target_latency_, - timestamp_to_ms(sample_spec_, target_latency_), - (double)(fe_ ? fe_->freq_coeff() : 0), (double)freq_coeff_); -} - } // namespace audio } // namespace roc diff --git a/src/internal_modules/roc_audio/feedback_monitor.h b/src/internal_modules/roc_audio/feedback_monitor.h index e7bb2053d..f30471ca1 100644 --- a/src/internal_modules/roc_audio/feedback_monitor.h +++ b/src/internal_modules/roc_audio/feedback_monitor.h @@ -12,49 +12,60 @@ #ifndef ROC_AUDIO_FEEDBACK_MONITOR_H_ #define ROC_AUDIO_FEEDBACK_MONITOR_H_ -#include "roc_audio/freq_estimator.h" #include "roc_audio/iframe_writer.h" -#include "roc_audio/latency_config.h" +#include "roc_audio/latency_tuner.h" #include "roc_audio/resampler_writer.h" #include "roc_audio/sample_spec.h" #include "roc_core/noncopyable.h" -#include "roc_core/optional.h" -#include "roc_packet/units.h" +#include "roc_core/rate_limiter.h" +#include "roc_core/time.h" namespace roc { namespace audio { -//! Metrics for feedback monitor. -struct FeedbackMonitorMetrics { - //! Estimated interarrival jitter. - //! An estimate of the statistical variance of the RTP data packet - //! interarrival time. - core::nanoseconds_t jitter; - - //! Estimated network incoming queue latency. - //! An estimate of how much media is buffered in receiver packet queue. - core::nanoseconds_t niq_latency; - - //! Estimated end-to-end latency. - //! An estimate of the time from recording a frame on sender to playing it - //! on receiver. - core::nanoseconds_t e2e_latency; - - FeedbackMonitorMetrics() - : jitter(0) - , niq_latency(0) - , e2e_latency(0) { +//! Feedback monitor configuration. +struct FeedbackConfig { + //! Timeout for source feedback. + //! If there is no new feedback during timeout, feedback monitor resets state. + core::nanoseconds_t source_timeout; + + //! Cooldown period between source changes. + //! After source is change, another source change is now allowed during + //! this period and is ignored. + core::nanoseconds_t source_cooldown; + + FeedbackConfig() + : source_timeout(1500 * core::Millisecond) + , source_cooldown(50 * core::Millisecond) { } }; //! Feedback monitor. +//! +//! @b Features +//! +//! - handles latency metrics from receiver (obtained via RTCP) +//! - asks LatencyTuner to calculate scaling factor based on the actual and +//! target latencies +//! - passes calculated scaling factor to resampler +//! +//! @b Flow +//! +//! - when pipeline obtains RTCP report, it calls write_metrics() method +//! - pipeline periodically calls write() method; it passes latest metrics +//! to LatencyTuner, and obtains scaling factor for resampler +//! - feedback monitor has a reference to resampler, and periodically passes +//! updated scaling factor to it +//! - pipeline also can query feedback monitor for latency metrics on behalf of +//! request from user class FeedbackMonitor : public IFrameWriter, public core::NonCopyable<> { public: //! Constructor. FeedbackMonitor(IFrameWriter& writer, ResamplerWriter* resampler, - const SampleSpec& sample_spec, - const LatencyConfig& config); + const FeedbackConfig& feedback_config, + const LatencyConfig& latency_config, + const SampleSpec& sample_spec); //! Check if the object was initialized successfully. bool is_valid() const; @@ -62,53 +73,42 @@ class FeedbackMonitor : public IFrameWriter, public core::NonCopyable<> { //! Check if feedback monitoring is started. bool is_started() const; - //! Get metrics. - FeedbackMonitorMetrics metrics() const; - //! Enable feedback monitoring. void start(); - //! Update metrics. - void store(const FeedbackMonitorMetrics& metrics); + //! Process feedback from receiver. + void process_feedback(packet::stream_source_t source_id, + const LatencyMetrics& metrics); //! Write audio frame. + //! Passes frame to underlying writer. + //! If feedback monitoring is started, also performs latency tuning. virtual void write(Frame& frame); + //! Get back latest metrics. + LatencyMetrics metrics() const; + private: - bool update_(); + bool update_tuner_(size_t n_samples); bool init_scaling_(); - bool update_scaling_(packet::stream_timestamp_diff_t latency); - - void report_(); - - IFrameWriter& writer_; - - ResamplerWriter* resampler_; - core::Optional fe_; + bool update_scaling_(); - packet::stream_timestamp_t stream_pos_; + LatencyTuner tuner_; - const packet::stream_timestamp_t update_interval_; - packet::stream_timestamp_t update_pos_; + LatencyMetrics metrics_; + bool has_metrics_; - const packet::stream_timestamp_t report_interval_; - packet::stream_timestamp_t report_pos_; - bool first_report_; + const core::nanoseconds_t metrics_timeout_; + core::nanoseconds_t metrics_ts_; - float freq_coeff_; - const FreqEstimatorInput fe_input_; - const float fe_max_delta_; - - packet::stream_timestamp_diff_t niq_latency_; - packet::stream_timestamp_diff_t e2e_latency_; - bool has_niq_latency_; - bool has_e2e_latency_; + IFrameWriter& writer_; - packet::stream_timestamp_diff_t jitter_; - bool has_jitter_; + ResamplerWriter* resampler_; + const bool enable_scaling_; - const packet::stream_timestamp_diff_t target_latency_; + packet::stream_source_t source_; + core::RateLimiter source_change_limiter_; const SampleSpec sample_spec_; diff --git a/src/internal_modules/roc_audio/freq_estimator.cpp b/src/internal_modules/roc_audio/freq_estimator.cpp index 3ee3a30c5..82ebb97e3 100644 --- a/src/internal_modules/roc_audio/freq_estimator.cpp +++ b/src/internal_modules/roc_audio/freq_estimator.cpp @@ -33,9 +33,6 @@ FreqEstimatorConfig make_config(FreqEstimatorProfile profile) { config.decimation_factor1 = fe_decim_factor_max; config.decimation_factor2 = fe_decim_factor_max; break; - - case FreqEstimatorProfile_Default: - break; } return config; @@ -88,7 +85,7 @@ FreqEstimator::FreqEstimator(FreqEstimatorProfile profile, "freq estimator: invalid decimation factor 2: got=%lu expected=[0; %lu]", (unsigned long)config_.decimation_factor2, (unsigned long)fe_decim_factor_max); - roc_panic_if_msg(fe_decim_len % 2 != 0, + roc_panic_if_msg((fe_decim_len & (fe_decim_len - 1)) != 0, "freq estimator: decim_len should be power of two"); memset(dec1_casc_buff_, 0, sizeof(dec1_casc_buff_)); @@ -156,38 +153,5 @@ double FreqEstimator::run_controller_(double current) { return 1 + config_.P * error + config_.I * accum_; } -const char* fe_input_to_str(FreqEstimatorInput input) { - switch (input) { - case FreqEstimatorInput_Disable: - return "disable"; - - case FreqEstimatorInput_Default: - return "default"; - - case FreqEstimatorInput_NiqLatency: - return "niq"; - - case FreqEstimatorInput_E2eLatency: - return "e2e"; - } - - return ""; -} - -const char* fe_profile_to_str(FreqEstimatorProfile profile) { - switch (profile) { - case FreqEstimatorProfile_Default: - return "default"; - - case FreqEstimatorProfile_Responsive: - return "responsive"; - - case FreqEstimatorProfile_Gradual: - return "gradual"; - } - - return ""; -} - } // namespace audio } // namespace roc diff --git a/src/internal_modules/roc_audio/freq_estimator.h b/src/internal_modules/roc_audio/freq_estimator.h index 5a723e83b..68407721b 100644 --- a/src/internal_modules/roc_audio/freq_estimator.h +++ b/src/internal_modules/roc_audio/freq_estimator.h @@ -20,26 +20,8 @@ namespace roc { namespace audio { -//! FreqEstimator input value. -enum FreqEstimatorInput { - //! Don't use FreqEstimator. - FreqEstimatorInput_Disable, - - //! Use default input. - FreqEstimatorInput_Default, - - //! Feed FreqEstimator with Network Incoming Queue length. - FreqEstimatorInput_NiqLatency, - - //! Feed FreqEstimator with End-to-end latency. - FreqEstimatorInput_E2eLatency -}; - //! FreqEstimator paremeter preset. enum FreqEstimatorProfile { - //! Use default profile. - FreqEstimatorProfile_Default, - //! Fast and responsive tuning. //! Good for lower network latency and jitter. FreqEstimatorProfile_Responsive, @@ -84,8 +66,8 @@ class FreqEstimator : public core::NonCopyable<> { //! @b Parameters //! - @p profile defines configuration preset. //! - @p target_latency defines latency we want to archive. - explicit FreqEstimator(FreqEstimatorProfile profile, - packet::stream_timestamp_t target_latency); + FreqEstimator(FreqEstimatorProfile profile, + packet::stream_timestamp_t target_latency); //! Get current frequecy coefficient. float freq_coeff() const; @@ -112,12 +94,6 @@ class FreqEstimator : public core::NonCopyable<> { double coeff_; // Current frequency coefficient value. }; -//! Get string name of FreqEstimator input. -const char* fe_input_to_str(FreqEstimatorInput input); - -//! Get string name of FreqEstimator profile. -const char* fe_profile_to_str(FreqEstimatorProfile profile); - } // namespace audio } // namespace roc diff --git a/src/internal_modules/roc_audio/latency_config.cpp b/src/internal_modules/roc_audio/latency_config.cpp deleted file mode 100644 index f7a5dc2d9..000000000 --- a/src/internal_modules/roc_audio/latency_config.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2024 Roc Streaming authors - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -#include "roc_audio/latency_config.h" - -namespace roc { -namespace audio { - -void LatencyConfig::deduce_defaults(bool is_receiver) { - if (fe_input == audio::FreqEstimatorInput_Default) { - // On receiver, by default adjust clock based on NIQ latency. - // On sender, by default do not adjust clock. - fe_input = is_receiver ? audio::FreqEstimatorInput_NiqLatency - : audio::FreqEstimatorInput_Disable; - } - - if (fe_profile == audio::FreqEstimatorProfile_Default) { - fe_profile = target_latency < 30 * core::Millisecond - // Prefer responsive profile on low latencies, because gradual profile - // won't do it at all. - ? audio::FreqEstimatorProfile_Responsive - // Prefer gradual profile for higher latencies, because it can handle - // higher network jitter. - : audio::FreqEstimatorProfile_Gradual; - } - - if (latency_tolerance < 0) { - // This formula returns target_latency * N, where N starts with larger - // number and approaches 0.5 as target_latency grows. - // Examples: - // target=1ms -> tolerance=8ms (x8) - // target=10ms -> tolerance=20ms (x2) - // target=200ms -> tolerance=200ms (x1) - // target=2000ms -> tolerance=1444ms (x0.722) - const core::nanoseconds_t capped_latency = - std::max(target_latency, core::Millisecond); - - latency_tolerance = core::nanoseconds_t( - capped_latency - * (std::log((200 * core::Millisecond) * 2) / std::log(capped_latency * 2))); - } -} - -} // namespace audio -} // namespace roc diff --git a/src/internal_modules/roc_audio/latency_config.h b/src/internal_modules/roc_audio/latency_config.h deleted file mode 100644 index 291679b33..000000000 --- a/src/internal_modules/roc_audio/latency_config.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2015 Roc Streaming authors - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -//! @file roc_audio/latency_config.h -//! @brief Latency config. - -#ifndef ROC_AUDIO_LATENCY_CONFIG_H_ -#define ROC_AUDIO_LATENCY_CONFIG_H_ - -#include "roc_audio/freq_estimator.h" -#include "roc_core/time.h" - -namespace roc { -namespace audio { - -//! Parameters for latency and feedback monitors. -struct LatencyConfig { - //! FreqEstimator input. - FreqEstimatorInput fe_input; - - //! FreqEstimator profile. - FreqEstimatorProfile fe_profile; - - //! FreqEstimator update interval, nanoseconds. - //! How often to run FreqEstimator and update Resampler scaling. - core::nanoseconds_t fe_update_interval; - - //! Target latency, nanoseconds. - core::nanoseconds_t target_latency; - - //! Maximum allowed deviation from target latency, nanoseconds. - //! If the latency goes out of bounds, the session is terminated. - core::nanoseconds_t latency_tolerance; - - //! Maximum allowed deviation of freq_coeff from 1.0. - //! If the scaling goes out of bounds, it is trimmed. - //! For example, 0.01 allows freq_coeff values in range [0.99; 1.01]. - float scaling_tolerance; - - //! Initialize. - LatencyConfig(const core::nanoseconds_t default_target_latency) - : fe_input(FreqEstimatorInput_Default) - , fe_profile(FreqEstimatorProfile_Default) - , fe_update_interval(5 * core::Millisecond) - , target_latency(default_target_latency) - , latency_tolerance(-1) - , scaling_tolerance(0.005f) { - } - - //! Automatically fill missing settings. - void deduce_defaults(bool is_receiver); -}; - -} // namespace audio -} // namespace roc - -#endif // ROC_AUDIO_LATENCY_CONFIG_H_ diff --git a/src/internal_modules/roc_audio/latency_monitor.cpp b/src/internal_modules/roc_audio/latency_monitor.cpp index 4c01e1ce0..306b80cde 100644 --- a/src/internal_modules/roc_audio/latency_monitor.cpp +++ b/src/internal_modules/roc_audio/latency_monitor.cpp @@ -12,109 +12,36 @@ #include "roc_core/panic.h" #include "roc_core/stddefs.h" #include "roc_core/time.h" +#include "roc_rtp/link_meter.h" namespace roc { namespace audio { -namespace { - -const core::nanoseconds_t LogInterval = 5 * core::Second; - -double timestamp_to_ms(const SampleSpec& sample_spec, - packet::stream_timestamp_diff_t timestamp) { - return (double)sample_spec.stream_timestamp_delta_2_ns(timestamp) / core::Millisecond; -} - -} // namespace - LatencyMonitor::LatencyMonitor(IFrameReader& frame_reader, const packet::SortedQueue& incoming_queue, const Depacketizer& depacketizer, + const rtp::LinkMeter& link_meter, ResamplerReader* resampler, const LatencyConfig& config, - const SampleSpec& input_sample_spec, - const SampleSpec& output_sample_spec) - : frame_reader_(frame_reader) + const SampleSpec& packet_sample_spec, + const SampleSpec& frame_sample_spec) + : tuner_(config, frame_sample_spec) + , frame_reader_(frame_reader) , incoming_queue_(incoming_queue) , depacketizer_(depacketizer) + , link_meter_(link_meter) , resampler_(resampler) - , stream_pos_(0) - , stream_cts_(0) - , update_interval_( - (packet::stream_timestamp_t)input_sample_spec.ns_2_stream_timestamp_delta( - config.fe_update_interval)) - , update_pos_(0) - , report_interval_((packet::stream_timestamp_t) - input_sample_spec.ns_2_stream_timestamp_delta(LogInterval)) - , report_pos_(0) - , freq_coeff_(0) - , fe_input_(config.fe_input) - , fe_max_delta_(config.scaling_tolerance) - , niq_latency_(0) - , e2e_latency_(0) - , has_niq_latency_(false) - , has_e2e_latency_(false) - , target_latency_( - fe_input_ != audio::FreqEstimatorInput_Disable - ? input_sample_spec.ns_2_stream_timestamp_delta(config.target_latency) - : 0) - , min_latency_(fe_input_ != audio::FreqEstimatorInput_Disable - ? input_sample_spec.ns_2_stream_timestamp_delta( - config.target_latency - config.latency_tolerance) - : 0) - , max_latency_(fe_input_ != audio::FreqEstimatorInput_Disable - ? input_sample_spec.ns_2_stream_timestamp_delta( - config.target_latency + config.latency_tolerance) - : 0) - , input_sample_spec_(input_sample_spec) - , output_sample_spec_(output_sample_spec) + , enable_scaling_(config.tuner_profile != audio::LatencyTunerProfile_Intact) + , capture_ts_(0) + , packet_sample_spec_(packet_sample_spec) , alive_(true) , valid_(false) { - roc_log( - LogDebug, - "latency monitor: initializing:" - " target=%lu(%.3fms) min=%lu(%.3fms) max=%lu(%.3fms)" - " in_rate=%lu out_rate=%lu" - " fe_input=%s fe_profile=%s fe_interval=%.3fms", - (unsigned long)target_latency_, - timestamp_to_ms(input_sample_spec_, target_latency_), (unsigned long)min_latency_, - timestamp_to_ms(input_sample_spec_, min_latency_), (unsigned long)max_latency_, - timestamp_to_ms(input_sample_spec_, max_latency_), - (unsigned long)input_sample_spec_.sample_rate(), - (unsigned long)output_sample_spec_.sample_rate(), - fe_input_to_str(config.fe_input), fe_profile_to_str(config.fe_profile), - timestamp_to_ms(input_sample_spec_, - (packet::stream_timestamp_diff_t)update_interval_)); - - if (fe_input_ != audio::FreqEstimatorInput_Disable) { - if (config.fe_update_interval <= 0) { - roc_log(LogError, "latency monitor: invalid config: fe_update_interval=%ld", - (long)config.fe_update_interval); - return; - } - - if (config.target_latency <= 0 || config.latency_tolerance <= 0 - || target_latency_ < min_latency_ || target_latency_ > max_latency_) { - roc_log(LogError, - "latency monitor: invalid config:" - " target_latency=%ldns latency_tolerance=%ldns", - (long)config.target_latency, (long)config.latency_tolerance); - return; - } - - if (!resampler_) { - roc_panic( - "latency monitor: freq estimator is enabled, but resampler is null"); - } - - fe_.reset(new (fe_) FreqEstimator(config.fe_profile, - (packet::stream_timestamp_t)target_latency_)); - if (!fe_) { - return; - } + if (!tuner_.is_valid()) { + return; + } - if (!init_scaling_(input_sample_spec.sample_rate(), - output_sample_spec.sample_rate())) { + if (enable_scaling_) { + if (!init_scaling_()) { return; } } @@ -132,35 +59,32 @@ bool LatencyMonitor::is_alive() const { return alive_; } -LatencyMonitorMetrics LatencyMonitor::metrics() const { +LatencyMetrics LatencyMonitor::metrics() const { roc_panic_if(!is_valid()); - LatencyMonitorMetrics metrics; - metrics.niq_latency = input_sample_spec_.stream_timestamp_delta_2_ns(niq_latency_); - metrics.e2e_latency = input_sample_spec_.stream_timestamp_delta_2_ns(e2e_latency_); - - return metrics; + return metrics_; } bool LatencyMonitor::read(Frame& frame) { roc_panic_if(!is_valid()); - if (frame.num_samples() % input_sample_spec_.num_channels() != 0) { - roc_panic("latency monitor: unexpected frame size"); + if (!alive_) { + return false; } compute_niq_latency_(); + query_link_meter_(); - update_(); + if (!pre_process_(frame)) { + alive_ = false; + return false; + } if (!frame_reader_.read(frame)) { return false; } - stream_pos_ += frame.num_samples() / input_sample_spec_.num_channels(); - stream_cts_ = frame.capture_timestamp(); - - report_(); + post_process_(frame); return true; } @@ -168,10 +92,6 @@ bool LatencyMonitor::read(Frame& frame) { bool LatencyMonitor::reclock(const core::nanoseconds_t playback_timestamp) { roc_panic_if(!is_valid()); - if (playback_timestamp < 0) { - roc_panic("latency monitor: unexpected playback timestamp"); - } - // this method is called when playback time of last frame was reported // now we can update e2e latency based on it compute_e2e_latency_(playback_timestamp); @@ -179,6 +99,29 @@ bool LatencyMonitor::reclock(const core::nanoseconds_t playback_timestamp) { return true; } +bool LatencyMonitor::pre_process_(const Frame& frame) { + tuner_.write_metrics(metrics_); + + if (!tuner_.advance_stream(frame.num_samples())) { + // TODO(gh-183): forward status code + return false; + } + + if (enable_scaling_) { + if (!update_scaling_()) { + // TODO(gh-183): forward status code + return false; + } + } + + return true; +} + +void LatencyMonitor::post_process_(const Frame& frame) { + // for end-2-end latency calculations + capture_ts_ = frame.capture_timestamp(); +} + void LatencyMonitor::compute_niq_latency_() { if (!depacketizer_.is_started()) { return; @@ -199,168 +142,71 @@ void LatencyMonitor::compute_niq_latency_() { // packet pipeline length // includes incoming queue and packets buffered inside other packet // pipeline elements, e.g. in FEC reader - niq_latency_ = packet::stream_timestamp_diff(niq_tail, niq_head); - has_niq_latency_ = true; + const packet::stream_timestamp_diff_t niq_latency = + packet::stream_timestamp_diff(niq_tail, niq_head); + + metrics_.niq_latency = packet_sample_spec_.stream_timestamp_delta_2_ns(niq_latency); + + // compute delay since last packet + const core::nanoseconds_t rts = latest_packet->receive_timestamp(); + const core::nanoseconds_t now = core::timestamp(core::ClockUnix); + + if (rts > 0 && rts < now) { + metrics_.niq_stalling = now - rts; + } } void LatencyMonitor::compute_e2e_latency_(const core::nanoseconds_t playback_timestamp) { - if (stream_cts_ == 0) { + if (capture_ts_ == 0) { + return; + } + + if (playback_timestamp <= 0) { return; } // delta between time when first sample of last frame is played on receiver and // time when first sample of that frame was captured on sender // (both timestamps are in receiver clock domain) - e2e_latency_ = - input_sample_spec_.ns_2_stream_timestamp_delta(playback_timestamp - stream_cts_); - has_e2e_latency_ = true; + metrics_.e2e_latency = playback_timestamp - capture_ts_; } -bool LatencyMonitor::update_() { - if (!alive_) { - return false; - } - - switch (fe_input_) { - case audio::FreqEstimatorInput_NiqLatency: - if (has_niq_latency_) { - if (!check_bounds_(niq_latency_)) { - alive_ = false; - return false; - } - if (fe_) { - if (!update_scaling_(niq_latency_)) { - alive_ = false; - return false; - } - } - } - break; - - case audio::FreqEstimatorInput_E2eLatency: - if (has_e2e_latency_) { - if (!check_bounds_(e2e_latency_)) { - alive_ = false; - return false; - } - if (fe_) { - if (!update_scaling_(e2e_latency_)) { - alive_ = false; - return false; - } - } - } - break; - - default: - break; - } - - return true; -} - -bool LatencyMonitor::check_bounds_(const packet::stream_timestamp_diff_t latency) const { - if (latency < min_latency_ && incoming_queue_.size() == 0) { - // Sharp latency decrease often appears on burst packet delays or drops, - // when depacketizer goes quite ahead of the last retrieved packet, and there - // are no new packets. If such burst is short, pipeline can easily recover from - // it, and terminating session would only harm. Hence, while the queue is empty, - // latency monitor does not terminate the session and leaves decision to the - // watchdog. If the burst was short, watchdog will keep session, otherwise - // no_playback_timeout will trigger and watchdog will terminate session. - return true; +void LatencyMonitor::query_link_meter_() { + if (!link_meter_.has_metrics()) { + return; } - if (latency < min_latency_ || latency > max_latency_) { - roc_log(LogDebug, - "latency monitor: latency out of bounds:" - " latency=%ld(%.3fms) target=%ld(%.3fms)" - " min=%ld(%.3fms) max=%ld(%.3fms) q_size=%lu", - (long)latency, timestamp_to_ms(input_sample_spec_, latency), - (long)target_latency_, - timestamp_to_ms(input_sample_spec_, target_latency_), (long)min_latency_, - timestamp_to_ms(input_sample_spec_, min_latency_), (long)max_latency_, - timestamp_to_ms(input_sample_spec_, max_latency_), - (unsigned long)incoming_queue_.size()); - return false; - } + const rtp::LinkMetrics link_metrics = link_meter_.metrics(); - return true; + metrics_.jitter = link_metrics.jitter; } -bool LatencyMonitor::init_scaling_(const size_t input_sample_rate, - const size_t output_sample_rate) { +bool LatencyMonitor::init_scaling_() { roc_panic_if_not(resampler_); - if (input_sample_rate == 0 || output_sample_rate == 0) { - roc_log(LogError, "latency monitor: invalid sample rates: input=%lu output=%lu", - (unsigned long)input_sample_rate, (unsigned long)output_sample_rate); - return false; - } - if (!resampler_->set_scaling(1.0f)) { - roc_log(LogError, - "latency monitor: scaling factor out of bounds: input=%lu output=%lu", - (unsigned long)input_sample_rate, (unsigned long)output_sample_rate); + roc_log(LogError, "latency monitor: can't set initial scaling"); return false; } return true; } -bool LatencyMonitor::update_scaling_(packet::stream_timestamp_diff_t latency) { +bool LatencyMonitor::update_scaling_() { roc_panic_if_not(resampler_); - roc_panic_if_not(fe_); - - if (latency < 0) { - latency = 0; - } - - if (stream_pos_ < update_pos_) { - return true; - } - while (stream_pos_ >= update_pos_) { - fe_->update((packet::stream_timestamp_t)latency); - update_pos_ += update_interval_; - } - - freq_coeff_ = fe_->freq_coeff(); - freq_coeff_ = std::min(freq_coeff_, 1.0f + fe_max_delta_); - freq_coeff_ = std::max(freq_coeff_, 1.0f - fe_max_delta_); - - if (!resampler_->set_scaling(freq_coeff_)) { - roc_log(LogDebug, - "latency monitor: scaling factor out of bounds: fe=%.6f trim_fe=%.6f", - (double)fe_->freq_coeff(), (double)freq_coeff_); - return false; + const float scaling = tuner_.get_scaling(); + if (scaling > 0) { + if (!resampler_->set_scaling(scaling)) { + roc_log(LogDebug, + "latency monitor: scaling factor out of bounds: scaling=%.6f", + (double)scaling); + return false; + } } return true; } -void LatencyMonitor::report_() { - if (!has_e2e_latency_ && !has_niq_latency_) { - return; - } - - if (stream_pos_ < report_pos_) { - return; - } - - while (stream_pos_ >= report_pos_) { - report_pos_ += report_interval_; - } - - roc_log(LogDebug, - "latency monitor:" - " e2e_latency=%ld(%.3fms) niq_latency=%ld(%.3fms) target_latency=%ld(%.3fms)" - " fe=%.6f trim_fe=%.6f", - (long)e2e_latency_, timestamp_to_ms(input_sample_spec_, e2e_latency_), - (long)niq_latency_, timestamp_to_ms(input_sample_spec_, niq_latency_), - (long)target_latency_, timestamp_to_ms(input_sample_spec_, target_latency_), - (double)(fe_ ? fe_->freq_coeff() : 0), (double)freq_coeff_); -} - } // namespace audio } // namespace roc diff --git a/src/internal_modules/roc_audio/latency_monitor.h b/src/internal_modules/roc_audio/latency_monitor.h index 1dccec633..5953d4fb6 100644 --- a/src/internal_modules/roc_audio/latency_monitor.h +++ b/src/internal_modules/roc_audio/latency_monitor.h @@ -15,7 +15,7 @@ #include "roc_audio/depacketizer.h" #include "roc_audio/freq_estimator.h" #include "roc_audio/iframe_reader.h" -#include "roc_audio/latency_config.h" +#include "roc_audio/latency_tuner.h" #include "roc_audio/resampler_reader.h" #include "roc_audio/sample_spec.h" #include "roc_core/attributes.h" @@ -24,30 +24,11 @@ #include "roc_core/time.h" #include "roc_packet/sorted_queue.h" #include "roc_packet/units.h" +#include "roc_rtp/link_meter.h" namespace roc { namespace audio { -//! Metrics of latency monitor. -struct LatencyMonitorMetrics { - //! Estimated NIQ latency. - //! NIQ = network incoming queue. - //! Defines how many samples are buffered in receiver packet queue and - //! receiver pipeline before depacketizer (packet part of pipeline). - core::nanoseconds_t niq_latency; - - //! Estimated E2E latency. - //! E2E = end-to-end. - //! Defines how much time passed between frame entered sender pipeline - //! (when it is captured) and leaved received pipeline (when it is played). - core::nanoseconds_t e2e_latency; - - LatencyMonitorMetrics() - : niq_latency(0) - , e2e_latency(0) { - } -}; - //! Latency monitor. //! //! @b Features @@ -57,13 +38,9 @@ struct LatencyMonitorMetrics { //! - calculates E2E latency (end-to-end) - how much time passed between frame //! was captured on sender and played on receiver (this is based on capture //! timestamps, which are populated with the help of RTCP) -//! - monitors how close actual latency and target latency are (which one to -//! check, NIQ or E2E, depends on config) -//! - assuming that the difference between actual latency and target latency is -//! caused by the clock drift between sender and receiver, calculates scaling -//! factor for resampler to compensate that clock drift +//! - asks LatencyTuner to calculate scaling factor based on the actual and +//! target latencies //! - passes calculated scaling factor to resampler -//! - shuts down session if the actual latency goes out of bounds //! //! @b Flow //! @@ -74,12 +51,12 @@ struct LatencyMonitorMetrics { //! - after adding frame to playback buffer, pipeline invokes reclock() method; //! it calculates difference between capture and playback time of the frame, //! which is E2E latency -//! - latency monitor has an instance of FreqEstimator (FE); it continously passes -//! calculated latency to FE, and FE calculates scaling factor for resampler +//! - latency monitor has an instance of LatencyTuner; it continously passes +//! calculated latencies to it, and obtains scaling factor for resampler //! - latency monitor has a reference to resampler, and periodically passes -//! updated scaling factor to it; -//! - pipeline also can query latency monitor for latency statistics on behalf of -//! request from user +//! updated scaling factor to it +//! - pipeline also can query latency monitor for latency metrics on behalf of +//! request from user or to report them to sender via RTCP class LatencyMonitor : public IFrameReader, public core::NonCopyable<> { public: //! Constructor. @@ -87,20 +64,21 @@ class LatencyMonitor : public IFrameReader, public core::NonCopyable<> { //! @b Parameters //! - @p frame_reader is inner frame reader for E2E latency calculation //! - @p incoming_queue and @p depacketizer are used to NIQ latency calculation + //! - @p link_meter is used to obtain link metrics //! - @p resampler is used to set the scaling factor to compensate clock //! drift according to calculated latency //! - @p config defines calculation parameters - //! - @p target_latency defines target latency that we should maintain - //! - @p input_sample_spec is the sample spec of the input packets - //! - @p output_sample_spec is the sample spec of the output frames (after + //! - @p packet_sample_spec is the sample spec of the input packets + //! - @p frame_sample_spec is the sample spec of the output frames (after //! resampling) LatencyMonitor(IFrameReader& frame_reader, const packet::SortedQueue& incoming_queue, const Depacketizer& depacketizer, + const rtp::LinkMeter& link_meter, ResamplerReader* resampler, const LatencyConfig& config, - const SampleSpec& input_sample_spec, - const SampleSpec& output_sample_spec); + const SampleSpec& packet_sample_spec, + const SampleSpec& frame_sample_spec); //! Check if the object was initialized successfully. bool is_valid() const; @@ -109,7 +87,7 @@ class LatencyMonitor : public IFrameReader, public core::NonCopyable<> { bool is_alive() const; //! Get metrics. - LatencyMonitorMetrics metrics() const; + LatencyMetrics metrics() const; //! Read audio frame from a pipeline. //! @remarks @@ -127,48 +105,29 @@ class LatencyMonitor : public IFrameReader, public core::NonCopyable<> { private: void compute_niq_latency_(); void compute_e2e_latency_(core::nanoseconds_t playback_timestamp); + void query_link_meter_(); - bool update_(); + bool pre_process_(const Frame& frame); + void post_process_(const Frame& frame); - bool check_bounds_(packet::stream_timestamp_diff_t latency) const; + bool init_scaling_(); + bool update_scaling_(); - bool init_scaling_(size_t input_sample_rate, size_t output_sample_rate); - bool update_scaling_(packet::stream_timestamp_diff_t latency); - - void report_(); + LatencyTuner tuner_; + LatencyMetrics metrics_; IFrameReader& frame_reader_; const packet::SortedQueue& incoming_queue_; const Depacketizer& depacketizer_; + const rtp::LinkMeter& link_meter_; ResamplerReader* resampler_; - core::Optional fe_; - - packet::stream_timestamp_t stream_pos_; - core::nanoseconds_t stream_cts_; - - const packet::stream_timestamp_t update_interval_; - packet::stream_timestamp_t update_pos_; - - const packet::stream_timestamp_t report_interval_; - packet::stream_timestamp_t report_pos_; - - float freq_coeff_; - const FreqEstimatorInput fe_input_; - const float fe_max_delta_; - - packet::stream_timestamp_diff_t niq_latency_; - packet::stream_timestamp_diff_t e2e_latency_; - bool has_niq_latency_; - bool has_e2e_latency_; + const bool enable_scaling_; - const packet::stream_timestamp_diff_t target_latency_; - const packet::stream_timestamp_diff_t min_latency_; - const packet::stream_timestamp_diff_t max_latency_; + core::nanoseconds_t capture_ts_; - const SampleSpec input_sample_spec_; - const SampleSpec output_sample_spec_; + const SampleSpec packet_sample_spec_; bool alive_; bool valid_; diff --git a/src/internal_modules/roc_audio/latency_tuner.cpp b/src/internal_modules/roc_audio/latency_tuner.cpp new file mode 100644 index 000000000..42f9287ee --- /dev/null +++ b/src/internal_modules/roc_audio/latency_tuner.cpp @@ -0,0 +1,425 @@ +/* + * Copyright (c) 2017 Roc Streaming authors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include "roc_audio/latency_tuner.h" +#include "roc_core/log.h" +#include "roc_core/panic.h" +#include "roc_core/time.h" + +namespace roc { +namespace audio { + +namespace { + +const core::nanoseconds_t LogInterval = 5 * core::Second; + +double timestamp_to_ms(const SampleSpec& sample_spec, + packet::stream_timestamp_diff_t timestamp) { + return (double)sample_spec.stream_timestamp_delta_2_ns(timestamp) / core::Millisecond; +} + +} // namespace + +void LatencyConfig::deduce_defaults(core::nanoseconds_t default_target_latency, + bool is_receiver) { + if (target_latency < 0) { + if (is_receiver) { + if (tuner_profile != LatencyTunerProfile_Intact) { + // Latency tuning is enabled on receiver. + // Use default if target latency is not specified. + target_latency = default_target_latency; + } else { + // Latency tuning is disabled on receiver. + // Most likely, it is enabled on sender. To make tuning work on sender, + // user should configure exactly same target latency on both sides. + // To make this requirement clear, in this case we don't use any default + // and require target latency to be specified explicitly on both sides. + target_latency = 0; + } + } else { + // See comment above. + // If latency tuning is enabled on sender, we don't use any default + // and require target latency to be specified explicitly. + target_latency = 0; + } + } + + if (latency_tolerance < 0) { + if (target_latency != 0) { + // This formula returns target_latency * N, where N starts with larger + // number and approaches 0.5 as target_latency grows. + // Examples (for multiplier = 1): + // target=1ms -> tolerance=8ms (x8) + // target=10ms -> tolerance=20ms (x2) + // target=200ms -> tolerance=200ms (x1) + // target=2000ms -> tolerance=1444ms (x0.722) + const core::nanoseconds_t capped_latency = + std::max(target_latency, core::Millisecond); + + // On sender, apply multiplier to make default tolerance a bit higher than + // on receiver. This way, if tolerance is enabled on both sides, receiver + // will always trigger first. + const int multiplier = is_receiver ? 1 : 2; + + latency_tolerance = core::nanoseconds_t( + capped_latency + * (std::log((200 * core::Millisecond) * 2 * multiplier) + / std::log(capped_latency * 2))); + } else { + // If default is not deduced for target latency, then it's also + // not deduced for latency tolerance. + latency_tolerance = 0; + } + } + + if (stale_tolerance < 0) { + if (latency_tolerance != 0) { + // Consider queue "stalling" if at least 1/4 of the missing latency + // is caused by lack of new packets. + stale_tolerance = latency_tolerance / 4; + } else { + // If default is not deduced for latency tolerance, then it's also + // not deduced for stale tolerance. + stale_tolerance = 0; + } + } + + if (tuner_backend == LatencyTunerBackend_Default) { + tuner_backend = LatencyTunerBackend_Niq; + } + + if (tuner_profile == LatencyTunerProfile_Default) { + if (is_receiver) { + if (tuner_backend == LatencyTunerBackend_Niq) { + // If latency is low, we assume network jitter is also low. In this + // case we use responsive profile. Gradual profile could cause + // oscillations comparable with the latency and break playback. + // + // If latency is high, we assume the jitter may be also high. In + // this case use gradual profile because it can handle high jitter + // mich better. + tuner_profile = target_latency < 30 * core::Millisecond + ? LatencyTunerProfile_Responsive + : LatencyTunerProfile_Gradual; + } else { + // E2E backend is not affected by network jitter that much, so + // we can just always use responsive profile. + tuner_profile = LatencyTunerProfile_Responsive; + } + } else { + // On sender, by default disable latency tuning. + // Typically latency tuning is done on receiver. + tuner_profile = LatencyTunerProfile_Intact; + } + } +} + +LatencyTuner::LatencyTuner(const LatencyConfig& config, const SampleSpec& sample_spec) + : stream_pos_(0) + , update_interval_( + (packet::stream_timestamp_t)sample_spec.ns_2_stream_timestamp_delta( + config.scaling_interval)) + , update_pos_(0) + , report_interval_((packet::stream_timestamp_t) + sample_spec.ns_2_stream_timestamp_delta(LogInterval)) + , report_pos_(0) + , freq_coeff_(0) + , freq_coeff_max_delta_(config.scaling_tolerance) + , backend_(config.tuner_backend) + , profile_(config.tuner_profile) + , enable_checking_(config.latency_tolerance > 0) + , enable_tuning_(profile_ != audio::LatencyTunerProfile_Intact) + , has_niq_latency_(false) + , niq_latency_(0) + , niq_stalling_(0) + , has_e2e_latency_(false) + , e2e_latency_(0) + , has_jitter_(false) + , jitter_(0) + , target_latency_(0) + , min_latency_(0) + , max_latency_(0) + , max_stalling_(0) + , sample_spec_(sample_spec) + , valid_(false) { + roc_log(LogDebug, + "latency tuner: initializing:" + " target_latency=%ld(%.3fms) latency_tolerance=%ld(%.3fms)" + " stale_tolerance=%ld(%.3fms)" + " scaling_interval=%ld(%.3fms) scaling_tolerance=%f" + " backend=%s profile=%s", + (long)sample_spec.ns_2_stream_timestamp_delta(config.target_latency), + (double)config.target_latency / core::Millisecond, + (long)sample_spec.ns_2_stream_timestamp_delta(config.latency_tolerance), + (double)config.latency_tolerance / core::Millisecond, + (long)sample_spec.ns_2_stream_timestamp_delta(config.stale_tolerance), + (double)config.stale_tolerance / core::Millisecond, + (long)sample_spec.ns_2_stream_timestamp_delta(config.scaling_interval), + (double)config.scaling_interval / core::Millisecond, + (double)config.scaling_tolerance, latency_tuner_backend_to_str(backend_), + latency_tuner_profile_to_str(profile_)); + + if (enable_checking_ || enable_tuning_) { + target_latency_ = sample_spec.ns_2_stream_timestamp_delta(config.target_latency); + + if (target_latency_ <= 0) { + roc_log( + LogError, + "latency tuner: invalid config: target_latency not set or out of bounds"); + return; + } + + if (enable_checking_) { + min_latency_ = sample_spec.ns_2_stream_timestamp_delta( + config.target_latency - config.latency_tolerance); + max_latency_ = sample_spec.ns_2_stream_timestamp_delta( + config.target_latency + config.latency_tolerance); + max_stalling_ = + sample_spec.ns_2_stream_timestamp_delta(config.stale_tolerance); + + if (target_latency_ < min_latency_ || target_latency_ > max_latency_) { + roc_log(LogError, + "latency tuner: invalid config: latency_tolerance out of bounds"); + return; + } + + if (max_stalling_ < 0) { + roc_log( + LogError, + "latency tuner: invalid config: stalling_tolerance out of bounds"); + return; + } + } + + if (enable_tuning_) { + if (freq_coeff_max_delta_ <= 0 || freq_coeff_max_delta_ >= 1) { + roc_log(LogError, + "latency tuner: invalid config: scaling_tolerance out of bounds"); + return; + } + + if (update_interval_ <= 0) { + roc_log(LogError, + "latency tuner: invalid config: scaling_interval out of bounds"); + return; + } + + fe_.reset(new (fe_) + FreqEstimator(profile_ == LatencyTunerProfile_Responsive + ? FreqEstimatorProfile_Responsive + : FreqEstimatorProfile_Gradual, + (packet::stream_timestamp_t)target_latency_)); + if (!fe_) { + return; + } + } + } + + valid_ = true; +} + +bool LatencyTuner::is_valid() const { + return valid_; +} + +void LatencyTuner::write_metrics(const LatencyMetrics& metrics) { + roc_panic_if(!is_valid()); + + if (metrics.niq_latency > 0 || metrics.niq_stalling > 0 || has_niq_latency_) { + niq_latency_ = sample_spec_.ns_2_stream_timestamp_delta(metrics.niq_latency); + niq_stalling_ = sample_spec_.ns_2_stream_timestamp_delta(metrics.niq_stalling); + has_niq_latency_ = true; + } + + if (metrics.e2e_latency > 0 || has_e2e_latency_) { + e2e_latency_ = sample_spec_.ns_2_stream_timestamp_delta(metrics.e2e_latency); + has_e2e_latency_ = true; + } + + if (metrics.jitter > 0 || has_jitter_) { + jitter_ = sample_spec_.ns_2_stream_timestamp_delta(metrics.jitter); + has_jitter_ = true; + } +} + +bool LatencyTuner::advance_stream(size_t n_samples) { + roc_panic_if(!is_valid()); + + const bool success = update_(); + + if (n_samples % sample_spec_.num_channels() != 0) { + roc_panic("latency tuner: unexpected frame size"); + } + + stream_pos_ += packet::stream_timestamp_t(n_samples / sample_spec_.num_channels()); + + report_(); + + return success; +} + +float LatencyTuner::get_scaling() const { + roc_panic_if(!is_valid()); + + return freq_coeff_; +} + +bool LatencyTuner::update_() { + packet::stream_timestamp_diff_t latency = 0; + + switch (backend_) { + case audio::LatencyTunerBackend_Niq: + if (!has_niq_latency_) { + return true; + } + latency = niq_latency_; + break; + + case audio::LatencyTunerBackend_E2e: + if (!has_e2e_latency_) { + return true; + } + latency = e2e_latency_; + break; + + default: + break; + } + + if (enable_checking_) { + if (!check_bounds_(latency)) { + return false; + } + } + + if (enable_tuning_) { + compute_scaling_(latency); + } + + return true; +} + +bool LatencyTuner::check_bounds_(const packet::stream_timestamp_diff_t latency) { + // Queue is considered "stalling" if there were no new packets for + // some period of time. + const bool is_stalling = backend_ == audio::LatencyTunerBackend_Niq + && niq_stalling_ > max_stalling_ && max_stalling_ > 0; + + if (latency < min_latency_ && is_stalling) { + // There are two possible reasons why queue latency becomes lower than mininmum: + // 1. either we were not able to compensate clock drift (or compensation is + // disabled) and queue slowly exhausted, + // 2. or there is a burst packet delay or drop, which led to sharp decrease + // of the latency. + // + // In the first case we normally want to terminate/restart session, but the + // second case is often not a big deal. If the burst is short, pipeline + // can easily recover from it, and terminating session would be worse. + // In this case, we want to keep things as is and leave decision to the + // watchdog. If the burst was short, watchdog will keep session, otherwise + // no_playback_timeout will trigger and watchdog will terminate session. + // + // To distinguish between the cases, we check network queue stalling metric, + // which shows delay since last received packet. If there were no packets + // during notabe amount of time, we assume that the second case takes place. + return true; + } + + if (latency < min_latency_ || latency > max_latency_) { + roc_log(LogDebug, + "latency tuner: latency out of bounds:" + " latency=%ld(%.3fms) target=%ld(%.3fms)" + " min=%ld(%.3fms) max=%ld(%.3fms) stalling=%ld(%.3fms)", + (long)latency, timestamp_to_ms(sample_spec_, latency), + (long)target_latency_, timestamp_to_ms(sample_spec_, target_latency_), + (long)min_latency_, timestamp_to_ms(sample_spec_, min_latency_), + (long)max_latency_, timestamp_to_ms(sample_spec_, max_latency_), + (long)niq_stalling_, timestamp_to_ms(sample_spec_, niq_stalling_)); + return false; + } + + return true; +} + +void LatencyTuner::compute_scaling_(packet::stream_timestamp_diff_t latency) { + if (latency < 0) { + latency = 0; + } + + if (stream_pos_ < update_pos_) { + return; + } + + while (stream_pos_ >= update_pos_) { + fe_->update((packet::stream_timestamp_t)latency); + update_pos_ += update_interval_; + } + + freq_coeff_ = fe_->freq_coeff(); + freq_coeff_ = std::min(freq_coeff_, 1.0f + freq_coeff_max_delta_); + freq_coeff_ = std::max(freq_coeff_, 1.0f - freq_coeff_max_delta_); +} + +void LatencyTuner::report_() { + if (stream_pos_ < report_pos_) { + return; + } + + while (stream_pos_ >= report_pos_) { + report_pos_ += report_interval_; + } + + roc_log(LogDebug, + "latency tuner:" + " e2e_latency=%ld(%.3fms) niq_latency=%ld(%.3fms) target_latency=%ld(%.3fms)" + " jitter=%ld(%.3fms) stale=%ld(%.3fms)" + " fe=%.6f eff_fe=%.6f", + (long)e2e_latency_, timestamp_to_ms(sample_spec_, e2e_latency_), + (long)niq_latency_, timestamp_to_ms(sample_spec_, niq_latency_), + (long)target_latency_, timestamp_to_ms(sample_spec_, target_latency_), + (long)jitter_, timestamp_to_ms(sample_spec_, jitter_), (long)niq_stalling_, + timestamp_to_ms(sample_spec_, niq_stalling_), + (double)(fe_ && freq_coeff_ > 0 ? fe_->freq_coeff() : 0), + (double)freq_coeff_); +} + +const char* latency_tuner_backend_to_str(LatencyTunerBackend backend) { + switch (backend) { + case LatencyTunerBackend_Default: + return "default"; + + case LatencyTunerBackend_Niq: + return "niq"; + + case LatencyTunerBackend_E2e: + return "e2e"; + } + + return ""; +} + +const char* latency_tuner_profile_to_str(LatencyTunerProfile profile) { + switch (profile) { + case LatencyTunerProfile_Default: + return "default"; + + case LatencyTunerProfile_Intact: + return "intact"; + + case LatencyTunerProfile_Responsive: + return "responsive"; + + case LatencyTunerProfile_Gradual: + return "gradual"; + } + + return ""; +} + +} // namespace audio +} // namespace roc diff --git a/src/internal_modules/roc_audio/latency_tuner.h b/src/internal_modules/roc_audio/latency_tuner.h new file mode 100644 index 000000000..c470151d7 --- /dev/null +++ b/src/internal_modules/roc_audio/latency_tuner.h @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2017 Roc Streaming authors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +//! @file roc_audio/latency_tuner.h +//! @brief Latency tuner. + +#ifndef ROC_AUDIO_LATENCY_TUNER_H_ +#define ROC_AUDIO_LATENCY_TUNER_H_ + +#include "roc_audio/freq_estimator.h" +#include "roc_audio/sample_spec.h" +#include "roc_core/noncopyable.h" +#include "roc_core/optional.h" +#include "roc_core/time.h" +#include "roc_status/status_code.h" + +namespace roc { +namespace audio { + +//! Latency tuner backend. +//! Defines which latency we monitor and tune to achieve target. +enum LatencyTunerBackend { + //! Deduce best default for given settings. + LatencyTunerBackend_Default, + + //! Latency is Network Incoming Queue length. + //! Calculated on receiver without use of any signaling protocol. + //! Reported back to sender via RTCP XR. + LatencyTunerBackend_Niq, + + //! Latency is End-to-end delay. + //! Can on receiver if RTCP XR is supported by both sides. + //! Reported back to sender via RTCP XR. + LatencyTunerBackend_E2e +}; + +//! Latency tuner profile. +//! Defines whether and how we tune latency on fly to compensate clock +//! drift and jitter. +enum LatencyTunerProfile { + //! Deduce best default for given settings. + LatencyTunerProfile_Default, + + //! Do not tune latency. + LatencyTunerProfile_Intact, + + //! Fast and responsive tuning. + //! Good for lower network latency and jitter. + LatencyTunerProfile_Responsive, + + //! Slow and smooth tuning. + //! Good for higher network latency and jitter. + LatencyTunerProfile_Gradual +}; + +//! Latency settings. +struct LatencyConfig { + //! Latency tuner backend to use. + LatencyTunerBackend tuner_backend; + + //! Latency tuner profile to use. + LatencyTunerProfile tuner_profile; + + //! Target latency. + //! If zero, latency tuning should be disabled, otherwise an error occurs. + //! If negative, default value is used if possible. + core::nanoseconds_t target_latency; + + //! Maximum allowed deviation from target latency. + //! If the latency goes out of bounds, the session is terminated. + //! If zero, bounds checks are disabled. + //! If negative, default value is used if possible. + core::nanoseconds_t latency_tolerance; + + //! Maximum delay since last packet before queue is considered stalling. + //! If niq_stalling becomes larger than stalling_tolerance, latency + //! tolerance checks are temporary disabled. + //! If zero, stalling checks are disabled. + //! If negative, default value is used if possible. + core::nanoseconds_t stale_tolerance; + + //! Scaling update interval. + //! How often to run FreqEstimator and update Resampler scaling. + core::nanoseconds_t scaling_interval; + + //! Maximum allowed deviation of freq_coeff from 1.0. + //! If the scaling goes out of bounds, it is trimmed. + //! For example, 0.01 allows freq_coeff values in range [0.99; 1.01]. + float scaling_tolerance; + + //! Initialize. + LatencyConfig() + : tuner_backend(LatencyTunerBackend_Default) + , tuner_profile(LatencyTunerProfile_Default) + , target_latency(-1) + , latency_tolerance(-1) + , stale_tolerance(-1) + , scaling_interval(5 * core::Millisecond) + , scaling_tolerance(0.005f) { + } + + //! Automatically fill missing settings. + void deduce_defaults(core::nanoseconds_t default_target_latency, bool is_receiver); +}; + +//! Latency metrics. +struct LatencyMetrics { + //! Estimated network incoming queue latency. + //! An estimate of how much media is buffered in receiver packet queue. + core::nanoseconds_t niq_latency; + + //! Delay since last received packet. + //! In other words, how long there were no new packets in network incoming queue. + core::nanoseconds_t niq_stalling; + + //! Estimated end-to-end latency. + //! An estimate of the time from recording a frame on sender to playing it + //! on receiver. + core::nanoseconds_t e2e_latency; + + //! Estimated interarrival jitter. + //! An estimate of the statistical variance of the RTP data packet + //! interarrival time. + core::nanoseconds_t jitter; + + LatencyMetrics() + : niq_latency(0) + , niq_stalling(0) + , e2e_latency(0) + , jitter(0) { + } +}; + +//! Latency tuner. +//! +//! On receiver, LatencyMonitor computes local metrics and passes them to LatencyTuner. +//! On sender, FeedbackMonitor obtains remote metrics and passes them to LatencyTuner. +//! In both cases, LatencyTuner processes metrics and computes scaling factor that +//! should be passed to resampler. +//! +//! Features: +//! - monitors how close actual latency and target latency are +//! - monitors whether latency goes out of bounds +//! - assuming that the difference between actual latency and target latency is +//! caused by the clock drift between sender and receiver, calculates scaling +//! factor for resampler to compensate it +class LatencyTuner : public core::NonCopyable<> { +public: + //! Initialize. + LatencyTuner(const LatencyConfig& config, const SampleSpec& sample_spec); + + //! Check if the object was initialized successfully. + bool is_valid() const; + + //! Pass updated metrics to tuner. + //! Tuner will use new metrics next time when advance() is called. + void write_metrics(const LatencyMetrics& metrics); + + //! Advance stream by given number of samples. + //! This method performs all actual work: + //! - depending on configured backend, selects which latency from + //! metrics to use + //! - check if latency goes out of bounds and session should be + //! terminated; if so, returns false + //! - computes updated scaling based on latency history and configured + //! profile + bool advance_stream(size_t n_samples); + + //! Get computed scaling. + //! Latency tuner expects that this scaling will applied to the stream + //! resampler, so that the latency will slowly achieve target value. + //! Returned value is close to 1.0. + //! If no scaling was computed yet, it returns 0.0. + float get_scaling() const; + +private: + bool update_(); + + bool check_bounds_(packet::stream_timestamp_diff_t latency); + void compute_scaling_(packet::stream_timestamp_diff_t latency); + + void report_(); + + core::Optional fe_; + + packet::stream_timestamp_t stream_pos_; + + const packet::stream_timestamp_t update_interval_; + packet::stream_timestamp_t update_pos_; + + const packet::stream_timestamp_t report_interval_; + packet::stream_timestamp_t report_pos_; + + float freq_coeff_; + const float freq_coeff_max_delta_; + + const LatencyTunerBackend backend_; + const LatencyTunerProfile profile_; + + const bool enable_checking_; + const bool enable_tuning_; + + bool has_niq_latency_; + packet::stream_timestamp_diff_t niq_latency_; + packet::stream_timestamp_diff_t niq_stalling_; + + bool has_e2e_latency_; + packet::stream_timestamp_diff_t e2e_latency_; + + bool has_jitter_; + packet::stream_timestamp_diff_t jitter_; + + packet::stream_timestamp_diff_t target_latency_; + packet::stream_timestamp_diff_t min_latency_; + packet::stream_timestamp_diff_t max_latency_; + packet::stream_timestamp_diff_t max_stalling_; + + const SampleSpec sample_spec_; + + bool valid_; +}; + +//! Get string name of latency backend. +const char* latency_tuner_backend_to_str(LatencyTunerBackend backend); + +//! Get string name of latency tuner. +const char* latency_tuner_profile_to_str(LatencyTunerProfile tuner); + +} // namespace audio +} // namespace roc + +#endif // ROC_AUDIO_LATENCY_TUNER_H_ diff --git a/src/internal_modules/roc_audio/resampler_config.cpp b/src/internal_modules/roc_audio/resampler_config.cpp index c7ed9c377..8d483bc85 100644 --- a/src/internal_modules/roc_audio/resampler_config.cpp +++ b/src/internal_modules/roc_audio/resampler_config.cpp @@ -12,13 +12,12 @@ namespace roc { namespace audio { -void ResamplerConfig::deduce_defaults(FreqEstimatorInput fe_input, - FreqEstimatorProfile fe_profile) { +void ResamplerConfig::deduce_defaults(LatencyTunerBackend latency_backend, + LatencyTunerProfile latency_tuner) { if (backend == ResamplerBackend_Default) { // If responsive profile is set, use builtin backend instead of speex, // since it has higher scaling precision. - const bool need_builtin_backend = fe_input != FreqEstimatorInput_Disable - && fe_profile == FreqEstimatorProfile_Responsive; + const bool need_builtin_backend = latency_tuner == LatencyTunerProfile_Responsive; // If speex backend is not available, fallback to builtin backend. const bool force_builtin_backend = diff --git a/src/internal_modules/roc_audio/resampler_config.h b/src/internal_modules/roc_audio/resampler_config.h index b1f32e1c9..cb2fdf490 100644 --- a/src/internal_modules/roc_audio/resampler_config.h +++ b/src/internal_modules/roc_audio/resampler_config.h @@ -12,7 +12,7 @@ #ifndef ROC_AUDIO_RESAMPLER_CONFIG_H_ #define ROC_AUDIO_RESAMPLER_CONFIG_H_ -#include "roc_audio/freq_estimator.h" +#include "roc_audio/latency_tuner.h" namespace roc { namespace audio { @@ -65,7 +65,8 @@ struct ResamplerConfig { } //! Automatically fill missing settings. - void deduce_defaults(FreqEstimatorInput fe_input, FreqEstimatorProfile fe_profile); + void deduce_defaults(LatencyTunerBackend latency_backend, + LatencyTunerProfile latency_tuner); }; //! Get string name of resampler backend. diff --git a/src/internal_modules/roc_audio/watchdog.cpp b/src/internal_modules/roc_audio/watchdog.cpp index 36bc61d36..5267eddd8 100644 --- a/src/internal_modules/roc_audio/watchdog.cpp +++ b/src/internal_modules/roc_audio/watchdog.cpp @@ -12,7 +12,11 @@ namespace roc { namespace audio { -void WatchdogConfig::deduce_defaults(const core::nanoseconds_t target_latency) { +void WatchdogConfig::deduce_defaults(core::nanoseconds_t target_latency) { + if (target_latency <= 0) { + target_latency = 200 * core::Millisecond; + } + if (no_playback_timeout < 0) { no_playback_timeout = target_latency * 4 / 3; } diff --git a/src/internal_modules/roc_netio/target_libuv/roc_netio/udp_port.cpp b/src/internal_modules/roc_netio/target_libuv/roc_netio/udp_port.cpp index b86cc24cd..1d3f9ca5b 100644 --- a/src/internal_modules/roc_netio/target_libuv/roc_netio/udp_port.cpp +++ b/src/internal_modules/roc_netio/target_libuv/roc_netio/udp_port.cpp @@ -12,6 +12,7 @@ #include "roc_core/panic.h" #include "roc_core/shared_ptr.h" #include "roc_core/string_builder.h" +#include "roc_core/time.h" #include "roc_netio/socket_ops.h" #include "roc_status/code_to_str.h" @@ -333,13 +334,16 @@ void UdpPort::recv_cb_(uv_udp_t* handle, pp->udp()->src_addr = src_addr; pp->udp()->dst_addr = self.config_.bind_address; + pp->udp()->receive_timestamp = core::timestamp(core::ClockUnix); pp->set_buffer(core::Slice(*bp, 0, (size_t)nread)); if (self.inbound_writer_) { const status::StatusCode code = self.inbound_writer_->write(pp); - // TODO(gh-183): handle or log status - roc_panic_if(code != status::StatusOK); + if (code != status::StatusOK) { + roc_panic("udp port: %s: can't writer packet: status=%s", self.descriptor(), + status::code_to_str(code)); + } } } diff --git a/src/internal_modules/roc_packet/packet.cpp b/src/internal_modules/roc_packet/packet.cpp index 8ddbb1b24..aad7e0d2d 100644 --- a/src/internal_modules/roc_packet/packet.cpp +++ b/src/internal_modules/roc_packet/packet.cpp @@ -162,6 +162,14 @@ core::nanoseconds_t Packet::capture_timestamp() const { return 0; } +core::nanoseconds_t Packet::receive_timestamp() const { + if (const UDP* u = udp()) { + return u->receive_timestamp; + } + + return 0; +} + int Packet::compare(const Packet& other) const { if (const RTP* ra = rtp()) { if (const RTP* rb = other.rtp()) { diff --git a/src/internal_modules/roc_packet/packet.h b/src/internal_modules/roc_packet/packet.h index e11af2d25..0c9fd6838 100644 --- a/src/internal_modules/roc_packet/packet.h +++ b/src/internal_modules/roc_packet/packet.h @@ -131,6 +131,11 @@ class Packet : public core::RefCounted, //! Returns number of nanoseconds since Unix epoch. core::nanoseconds_t capture_timestamp() const; + //! Get receive timestamp (RTS) of the packet. + //! @remarks + //! Returns number of nanoseconds since Unix epoch. + core::nanoseconds_t receive_timestamp() const; + //! Determine packet ordering. //! @returns //! * -1 if this packet precedes @p other packet diff --git a/src/internal_modules/roc_packet/target_libuv/roc_packet/udp.cpp b/src/internal_modules/roc_packet/target_libuv/roc_packet/udp.cpp new file mode 100644 index 000000000..de7fb976f --- /dev/null +++ b/src/internal_modules/roc_packet/target_libuv/roc_packet/udp.cpp @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2017 Roc Streaming authors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include "roc_packet/udp.h" +#include "roc_core/stddefs.h" + +namespace roc { +namespace packet { + +UDP::UDP() + : receive_timestamp(0) { + memset(&request, 0, sizeof(request)); +} + +} // namespace packet +} // namespace roc diff --git a/src/internal_modules/roc_packet/target_libuv/roc_packet/udp.h b/src/internal_modules/roc_packet/target_libuv/roc_packet/udp.h index 58291cad5..1c16d6bd0 100644 --- a/src/internal_modules/roc_packet/target_libuv/roc_packet/udp.h +++ b/src/internal_modules/roc_packet/target_libuv/roc_packet/udp.h @@ -17,6 +17,7 @@ #include "roc_address/socket_addr.h" #include "roc_core/slice.h" #include "roc_core/stddefs.h" +#include "roc_core/time.h" namespace roc { namespace packet { @@ -29,8 +30,13 @@ struct UDP { //! Destination address. address::SocketAddr dst_addr; + //! Packet receive timestamp (RTS), nanoseconds since Unix epoch. + core::nanoseconds_t receive_timestamp; + //! Sender request state. uv_udp_send_t request; + + UDP(); }; } // namespace packet diff --git a/src/internal_modules/roc_pipeline/config.cpp b/src/internal_modules/roc_pipeline/config.cpp index a91664d58..dcbce8115 100644 --- a/src/internal_modules/roc_pipeline/config.cpp +++ b/src/internal_modules/roc_pipeline/config.cpp @@ -13,8 +13,7 @@ namespace roc { namespace pipeline { SenderConfig::SenderConfig() - : latency(DefaultLatency) - , input_sample_spec(DefaultSampleSpec) + : input_sample_spec(DefaultSampleSpec) , packet_length(DefaultPacketLength) , payload_type(rtp::PayloadType_L16_Stereo) , enable_interleaving(false) @@ -24,19 +23,18 @@ SenderConfig::SenderConfig() } void SenderConfig::deduce_defaults() { - latency.deduce_defaults(false); - resampler.deduce_defaults(latency.fe_input, latency.fe_profile); + latency.deduce_defaults(DefaultLatency, false); + resampler.deduce_defaults(latency.tuner_backend, latency.tuner_profile); } ReceiverSessionConfig::ReceiverSessionConfig() - : payload_type(0) - , latency(DefaultLatency) { + : payload_type(0) { } void ReceiverSessionConfig::deduce_defaults() { - latency.deduce_defaults(true); + latency.deduce_defaults(DefaultLatency, true); watchdog.deduce_defaults(latency.target_latency); - resampler.deduce_defaults(latency.fe_input, latency.fe_profile); + resampler.deduce_defaults(latency.tuner_backend, latency.tuner_profile); } ReceiverCommonConfig::ReceiverCommonConfig() @@ -65,8 +63,8 @@ TranscoderConfig::TranscoderConfig() } void TranscoderConfig::deduce_defaults() { - resampler.deduce_defaults(audio::FreqEstimatorInput_Disable, - audio::FreqEstimatorProfile_Default); + resampler.deduce_defaults(audio::LatencyTunerBackend_Default, + audio::LatencyTunerProfile_Default); } } // namespace pipeline diff --git a/src/internal_modules/roc_pipeline/config.h b/src/internal_modules/roc_pipeline/config.h index 7be84a078..124110a52 100644 --- a/src/internal_modules/roc_pipeline/config.h +++ b/src/internal_modules/roc_pipeline/config.h @@ -13,7 +13,8 @@ #define ROC_PIPELINE_CONFIG_H_ #include "roc_address/protocol.h" -#include "roc_audio/latency_config.h" +#include "roc_audio/feedback_monitor.h" +#include "roc_audio/latency_tuner.h" #include "roc_audio/profiler.h" #include "roc_audio/resampler_config.h" #include "roc_audio/sample_spec.h" @@ -61,6 +62,9 @@ struct SenderConfig { //! Latency parameters. audio::LatencyConfig latency; + //! Feedback parameters. + audio::FeedbackConfig feedback; + //! Resampler parameters. audio::ResamplerConfig resampler; diff --git a/src/internal_modules/roc_pipeline/metrics.h b/src/internal_modules/roc_pipeline/metrics.h index 85217da35..5059711d1 100644 --- a/src/internal_modules/roc_pipeline/metrics.h +++ b/src/internal_modules/roc_pipeline/metrics.h @@ -12,8 +12,7 @@ #ifndef ROC_PIPELINE_METRICS_H_ #define ROC_PIPELINE_METRICS_H_ -#include "roc_audio/feedback_monitor.h" -#include "roc_audio/latency_monitor.h" +#include "roc_audio/latency_tuner.h" #include "roc_audio/packetizer.h" #include "roc_core/stddefs.h" #include "roc_rtp/link_meter.h" @@ -27,7 +26,7 @@ struct SenderSessionMetrics { audio::PacketizerMetrics packets; //! Receiver feedback. - audio::FeedbackMonitorMetrics feedback; + audio::LatencyMetrics latency; SenderSessionMetrics() { } @@ -49,7 +48,7 @@ struct ReceiverSessionMetrics { rtp::LinkMetrics link; //! Latency metrics. - audio::LatencyMonitorMetrics latency; + audio::LatencyMetrics latency; ReceiverSessionMetrics() { } diff --git a/src/internal_modules/roc_pipeline/receiver_session.cpp b/src/internal_modules/roc_pipeline/receiver_session.cpp index 8375bf5d5..b943ccee4 100644 --- a/src/internal_modules/roc_pipeline/receiver_session.cpp +++ b/src/internal_modules/roc_pipeline/receiver_session.cpp @@ -167,7 +167,7 @@ ReceiverSession::ReceiverSession( areader = channel_mapper_reader_.get(); } - if (session_config.latency.fe_input != audio::FreqEstimatorInput_Disable + if (session_config.latency.tuner_profile != audio::LatencyTunerProfile_Intact || encoding->sample_spec.sample_rate() != common_config.output_sample_spec.sample_rate()) { resampler_poisoner_.reset(new (resampler_poisoner_) @@ -208,7 +208,7 @@ ReceiverSession::ReceiverSession( areader = session_poisoner_.get(); latency_monitor_.reset(new (latency_monitor_) audio::LatencyMonitor( - *areader, *source_queue_, *depacketizer_, resampler_reader_.get(), + *areader, *source_queue_, *depacketizer_, *source_meter_, resampler_reader_.get(), session_config.latency, encoding->sample_spec, common_config.output_sample_spec)); if (!latency_monitor_ || !latency_monitor_->is_valid()) { return; @@ -295,7 +295,7 @@ void ReceiverSession::generate_reports(const char* report_cname, if (n_reports > 0 && packet_router_->has_source_id(packet::Packet::FlagAudio) && source_meter_->has_metrics()) { - const audio::LatencyMonitorMetrics latency_metrics = latency_monitor_->metrics(); + const audio::LatencyMetrics latency_metrics = latency_monitor_->metrics(); const rtp::LinkMetrics link_metrics = source_meter_->metrics(); rtcp::RecvReport& report = *reports; @@ -312,6 +312,7 @@ void ReceiverSession::generate_reports(const char* report_cname, report.cum_loss = link_metrics.cum_loss; report.jitter = link_metrics.jitter; report.niq_latency = latency_metrics.niq_latency; + report.niq_stalling = latency_metrics.niq_stalling; report.e2e_latency = latency_metrics.e2e_latency; reports++; diff --git a/src/internal_modules/roc_pipeline/sender_session.cpp b/src/internal_modules/roc_pipeline/sender_session.cpp index 7e50f4120..92f4b89ca 100644 --- a/src/internal_modules/roc_pipeline/sender_session.cpp +++ b/src/internal_modules/roc_pipeline/sender_session.cpp @@ -141,7 +141,7 @@ bool SenderSession::create_transport_pipeline(SenderEndpoint* source_endpoint, awriter = channel_mapper_writer_.get(); } - if (config_.latency.fe_input != audio::FreqEstimatorInput_Disable + if (config_.latency.tuner_profile != audio::LatencyTunerProfile_Intact || encoding->sample_spec.sample_rate() != config_.input_sample_spec.sample_rate()) { resampler_.reset(audio::ResamplerMap::instance().new_resampler( @@ -167,7 +167,8 @@ bool SenderSession::create_transport_pipeline(SenderEndpoint* source_endpoint, } feedback_monitor_.reset(new (feedback_monitor_) audio::FeedbackMonitor( - *awriter, resampler_writer_.get(), config_.input_sample_spec, config_.latency)); + *awriter, resampler_writer_.get(), config_.feedback, config_.latency, + config_.input_sample_spec)); if (!feedback_monitor_ || !feedback_monitor_->is_valid()) { return false; } @@ -244,7 +245,7 @@ SenderSessionMetrics SenderSession::get_metrics() const { metrics.packets = packetizer_->metrics(); } if (feedback_monitor_) { - metrics.feedback = feedback_monitor_->metrics(); + metrics.latency = feedback_monitor_->metrics(); } return metrics; @@ -292,12 +293,13 @@ SenderSession::notify_send_stream(packet::stream_source_t recv_source_id, roc_panic_if(!has_send_stream()); if (feedback_monitor_ && feedback_monitor_->is_started()) { - audio::FeedbackMonitorMetrics metrics; - metrics.jitter = recv_report.jitter; + audio::LatencyMetrics metrics; metrics.niq_latency = recv_report.niq_latency; + metrics.niq_stalling = recv_report.niq_stalling; metrics.e2e_latency = recv_report.e2e_latency; + metrics.jitter = recv_report.jitter; - feedback_monitor_->store(metrics); + feedback_monitor_->process_feedback(recv_source_id, metrics); } return status::StatusOK; @@ -320,6 +322,11 @@ void SenderSession::start_feedback_monitor_() { return; } + if (feedback_monitor_->is_started()) { + // Already started. + return; + } + feedback_monitor_->start(); } diff --git a/src/internal_modules/roc_rtcp/communicator.cpp b/src/internal_modules/roc_rtcp/communicator.cpp index f089a0f45..18f0302ad 100644 --- a/src/internal_modules/roc_rtcp/communicator.cpp +++ b/src/internal_modules/roc_rtcp/communicator.cpp @@ -21,7 +21,7 @@ namespace rtcp { namespace { -const core::nanoseconds_t ReportInterval = core::Second * 30; +const core::nanoseconds_t LogInterval = core::Second * 30; } // namespace @@ -49,7 +49,9 @@ Communicator::Communicator(const Config& config, , cur_pkt_send_stream_(0) , cur_pkt_recv_stream_(0) , error_count_(0) - , error_limiter_(ReportInterval) + , processed_packet_count_(0) + , generated_packet_count_(0) + , log_limiter_(LogInterval) , valid_(false) { if (!reporter_.is_valid()) { return; @@ -80,10 +82,12 @@ status::StatusCode Communicator::process_packet(const packet::PacketPtr& packet, roc_log(LogTrace, "rtcp communicator: processing incoming packet"); + processed_packet_count_++; + Traverser traverser(packet->rtcp()->payload); if (!traverser.parse()) { roc_log(LogTrace, "rtcp communicator: error when parsing compound packet"); - record_error_(); + error_count_++; return status::StatusOK; } @@ -128,7 +132,7 @@ void Communicator::process_all_descriptions_(const Traverser& traverser) { SdesTraverser sdes = iter.get_sdes(); if (!sdes.parse()) { roc_log(LogTrace, "rtcp communicator: error when parsing SDES packet"); - record_error_(); + error_count_++; break; } process_description_(sdes); @@ -141,7 +145,7 @@ void Communicator::process_all_descriptions_(const Traverser& traverser) { if (iter.error()) { roc_log(LogTrace, "rtcp communicator: error when traversing compound packet"); - record_error_(); + error_count_++; } } @@ -163,7 +167,7 @@ void Communicator::process_all_reports_(const Traverser& traverser) { XrTraverser xr = iter.get_xr(); if (!xr.parse()) { roc_log(LogTrace, "rtcp communicator: error when parsing XR packet"); - record_error_(); + error_count_++; break; } process_extended_report_(xr); @@ -176,7 +180,7 @@ void Communicator::process_all_reports_(const Traverser& traverser) { if (iter.error()) { roc_log(LogTrace, "rtcp communicator: error when traversing compound packet"); - record_error_(); + error_count_++; } } @@ -190,7 +194,7 @@ void Communicator::process_all_goodbyes_(const Traverser& traverser) { ByeTraverser bye = iter.get_bye(); if (!bye.parse()) { roc_log(LogTrace, "rtcp communicator: error when parsing BYE packet"); - record_error_(); + error_count_++; break; } process_goodbye_(bye); @@ -203,7 +207,7 @@ void Communicator::process_all_goodbyes_(const Traverser& traverser) { if (iter.error()) { roc_log(LogTrace, "rtcp communicator: error when traversing compound packet"); - record_error_(); + error_count_++; } } @@ -237,7 +241,7 @@ void Communicator::process_description_(const SdesTraverser& sdes) { if (iter.error()) { roc_log(LogTrace, "rtcp communicator: error when traversing SDES packet"); - record_error_(); + error_count_++; } } @@ -258,7 +262,7 @@ void Communicator::process_goodbye_(const ByeTraverser& bye) { if (iter.error()) { roc_log(LogTrace, "rtcp communicator: error when traversing BYE packet"); - record_error_(); + error_count_++; } } @@ -323,7 +327,7 @@ void Communicator::process_extended_report_(const XrTraverser& xr) { if (iter.error()) { roc_log(LogTrace, "rtcp communicator: error when traversing XR packet"); - record_error_(); + error_count_++; } } @@ -414,6 +418,8 @@ status::StatusCode Communicator::generate_packets_(core::nanoseconds_t current_t if (status != status::StatusOK) { break; } + + generated_packet_count_++; } status::StatusCode e_status = end_packet_generation_(); @@ -421,6 +427,8 @@ status::StatusCode Communicator::generate_packets_(core::nanoseconds_t current_t status = e_status; } + log_stats_(); + return status; } @@ -835,14 +843,20 @@ void Communicator::generate_goodbye_(Builder& bld) { bld.end_bye(); } -void Communicator::record_error_() { - error_count_++; - - if (error_limiter_.allow()) { - roc_log(LogDebug, "rtcp communicator: dropped %lu malformed packets", - (unsigned long)error_count_); - error_count_ = 0; +void Communicator::log_stats_() { + if (!log_limiter_.allow()) { + return; } + + roc_log(LogDebug, + "rtcp communicator:" + " generated_pkts=%lu processed_pkts=%lu proc_errs=%lu", + (unsigned long)generated_packet_count_, + (unsigned long)processed_packet_count_, (unsigned long)error_count_); + + error_count_ = 0; + processed_packet_count_ = 0; + generated_packet_count_ = 0; } } // namespace rtcp diff --git a/src/internal_modules/roc_rtcp/communicator.h b/src/internal_modules/roc_rtcp/communicator.h index 5dca01c16..53bd137b3 100644 --- a/src/internal_modules/roc_rtcp/communicator.h +++ b/src/internal_modules/roc_rtcp/communicator.h @@ -137,7 +137,7 @@ class Communicator : public core::NonCopyable<> { void generate_description_(Builder& bld); void generate_goodbye_(Builder& bld); - void record_error_(); + void log_stats_(); packet::PacketFactory& packet_factory_; core::BufferFactory& buffer_factory_; @@ -165,9 +165,11 @@ class Communicator : public core::NonCopyable<> { size_t cur_pkt_send_stream_; size_t cur_pkt_recv_stream_; - // Dropped malformed incoming packets. + // Statistics. size_t error_count_; - core::RateLimiter error_limiter_; + size_t processed_packet_count_; + size_t generated_packet_count_; + core::RateLimiter log_limiter_; bool valid_; }; diff --git a/src/internal_modules/roc_rtcp/headers.h b/src/internal_modules/roc_rtcp/headers.h index 6ca27766a..9652d4efa 100644 --- a/src/internal_modules/roc_rtcp/headers.h +++ b/src/internal_modules/roc_rtcp/headers.h @@ -1525,7 +1525,7 @@ ROC_ATTR_PACKED_BEGIN class XrDelayMetricsBlock { NtpTimestamp32 mean_rtt_; NtpTimestamp32 min_rtt_; NtpTimestamp32 max_rtt_; - NtpTimestamp64 e2e_delay_; + NtpTimestamp64 e2e_latency_; public: XrDelayMetricsBlock() { @@ -1539,7 +1539,7 @@ ROC_ATTR_PACKED_BEGIN class XrDelayMetricsBlock { mean_rtt_.set_value(MetricUnavail_32); min_rtt_.set_value(MetricUnavail_32); max_rtt_.set_value(MetricUnavail_32); - e2e_delay_.set_value(MetricUnavail_64); + e2e_latency_.set_value(MetricUnavail_64); } //! Get common block header. @@ -1624,18 +1624,18 @@ ROC_ATTR_PACKED_BEGIN class XrDelayMetricsBlock { } //! Check if End System Delay is set. - bool has_e2e_delay() const { - return e2e_delay_.value() != MetricUnavail_64; + bool has_e2e_latency() const { + return e2e_latency_.value() != MetricUnavail_64; } //! Get End System Delay. - packet::ntp_timestamp_t e2e_delay() const { - return e2e_delay_.value(); + packet::ntp_timestamp_t e2e_latency() const { + return e2e_latency_.value(); } //! Set End System Delay. - void set_e2e_delay(const packet::ntp_timestamp_t t) { - e2e_delay_.set_value(ntp_clamp_64(t, MetricUnavail_64 - 1)); + void set_e2e_latency(const packet::ntp_timestamp_t t) { + e2e_latency_.set_value(ntp_clamp_64(t, MetricUnavail_64 - 1)); } } ROC_ATTR_PACKED_END; @@ -1643,15 +1643,26 @@ ROC_ATTR_PACKED_BEGIN class XrDelayMetricsBlock { //! //! Non-standard. //! +//! Reports two metrics from receiver to sender: +//! +//! - niq_latency: current length of network incoming queue, measured via timestamp +//! difference between last received packet and last decoded packet +//! (expressed in units of 1/65536 seconds) +//! +//! - niq_stalling: current elapsed time since last received packet +//! (expressed in units of 1/65536 seconds) +//! //! @code //! 0 1 2 3 //! 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! | BT=220 | I | resv. | block length = 6 | +//! | BT=220 | I | resv. | block length = 3 | //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ //! | SSRC of Source | //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! | Network Incoming Queue Delay | +//! | Network Incoming Queue Latency | +//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +//! | Network Incoming Queue Stalling | //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ //! @endcode ROC_ATTR_PACKED_BEGIN class XrQueueMetricsBlock { @@ -1664,7 +1675,8 @@ ROC_ATTR_PACKED_BEGIN class XrQueueMetricsBlock { XrBlockHeader header_; uint32_t ssrc_; - NtpTimestamp32 niq_delay_; + NtpTimestamp32 niq_latency_; + NtpTimestamp32 niq_stalling_; public: XrQueueMetricsBlock() { @@ -1675,7 +1687,8 @@ ROC_ATTR_PACKED_BEGIN class XrQueueMetricsBlock { void reset() { header_.reset(XR_QUEUE_METRICS); ssrc_ = 0; - niq_delay_.set_value(MetricUnavail_32); + niq_latency_.set_value(MetricUnavail_32); + niq_stalling_.set_value(MetricUnavail_32); } //! Get common block header. @@ -1712,18 +1725,33 @@ ROC_ATTR_PACKED_BEGIN class XrQueueMetricsBlock { } //! Check if Network Incoming Queue Delay is set. - bool has_niq_delay() const { - return niq_delay_.value() != MetricUnavail_32; + bool has_niq_latency() const { + return niq_latency_.value() != MetricUnavail_32; } //! Get Network Incoming Queue Delay. - packet::ntp_timestamp_t niq_delay() const { - return niq_delay_.value(); + packet::ntp_timestamp_t niq_latency() const { + return niq_latency_.value(); } //! Set Network Incoming Queue Delay. - void set_niq_delay(const packet::ntp_timestamp_t t) { - niq_delay_.set_value(ntp_clamp_32(t, MetricUnavail_32 - 1)); + void set_niq_latency(const packet::ntp_timestamp_t t) { + niq_latency_.set_value(ntp_clamp_32(t, MetricUnavail_32 - 1)); + } + + //! Check if Network Incoming Queue Stalling is set. + bool has_niq_stalling() const { + return niq_stalling_.value() != MetricUnavail_32; + } + + //! Get Network Incoming Queue Stalling. + packet::ntp_timestamp_t niq_stalling() const { + return niq_stalling_.value(); + } + + //! Set Network Incoming Queue Stalling. + void set_niq_stalling(const packet::ntp_timestamp_t t) { + niq_stalling_.set_value(ntp_clamp_32(t, MetricUnavail_32 - 1)); } } ROC_ATTR_PACKED_END; diff --git a/src/internal_modules/roc_rtcp/print_packet.cpp b/src/internal_modules/roc_rtcp/print_packet.cpp index 60d5407ad..0af9cd548 100644 --- a/src/internal_modules/roc_rtcp/print_packet.cpp +++ b/src/internal_modules/roc_rtcp/print_packet.cpp @@ -182,8 +182,9 @@ void print_xr_delay_metrics(core::Printer& p, const header::XrDelayMetricsBlock& (long long)packet::ntp_2_nanoseconds(blk.min_rtt())); p.writef("|--- rtt_max: %016llx (unix %lld)\n", (unsigned long long)blk.max_rtt(), (long long)packet::ntp_2_nanoseconds(blk.max_rtt())); - p.writef("|--- e2e_delay: %016llx (unix %lld)\n", (unsigned long long)blk.e2e_delay(), - (long long)packet::ntp_2_nanoseconds(blk.e2e_delay())); + p.writef("|--- e2e_latency: %016llx (unix %lld)\n", + (unsigned long long)blk.e2e_latency(), + (long long)packet::ntp_2_nanoseconds(blk.e2e_latency())); } void print_xr_queue_metrics(core::Printer& p, const header::XrQueueMetricsBlock& blk) { @@ -194,8 +195,12 @@ void print_xr_queue_metrics(core::Printer& p, const header::XrQueueMetricsBlock& p.writef("|-- block body:\n"); print_metric_flag(p, blk.metric_flag()); p.writef("|--- ssrc: %lu\n", (unsigned long)blk.ssrc()); - p.writef("|--- niq_delay: %016llx (unix %lld)\n", (unsigned long long)blk.niq_delay(), - (long long)packet::ntp_2_nanoseconds(blk.niq_delay())); + p.writef("|--- niq_latency: %016llx (unix %lld)\n", + (unsigned long long)blk.niq_latency(), + (long long)packet::ntp_2_nanoseconds(blk.niq_latency())); + p.writef("|--- niq_stalling: %016llx (unix %lld)\n", + (unsigned long long)blk.niq_stalling(), + (long long)packet::ntp_2_nanoseconds(blk.niq_stalling())); } void print_xr(core::Printer& p, const XrTraverser& xr) { diff --git a/src/internal_modules/roc_rtcp/reporter.cpp b/src/internal_modules/roc_rtcp/reporter.cpp index 80de8550b..3ae210511 100644 --- a/src/internal_modules/roc_rtcp/reporter.cpp +++ b/src/internal_modules/roc_rtcp/reporter.cpp @@ -495,9 +495,9 @@ void Reporter::process_delay_metrics_block(const header::XrPacket& xr, "rtcp reporter: processing Delay Metrics block: send_ssrc=%lu recv_ssrc=%lu", (unsigned long)send_source_id, (unsigned long)recv_source_id); - if (blk.has_e2e_delay()) { + if (blk.has_e2e_latency()) { stream->remote_recv_report.e2e_latency = - packet::ntp_2_nanoseconds(blk.e2e_delay()); + packet::ntp_2_nanoseconds(blk.e2e_latency()); } update_stream_(*stream); @@ -533,9 +533,14 @@ void Reporter::process_queue_metrics_block(const header::XrPacket& xr, "rtcp reporter: processing Queue Metrics block: send_ssrc=%lu recv_ssrc=%lu", (unsigned long)send_source_id, (unsigned long)recv_source_id); - if (blk.has_niq_delay()) { + if (blk.has_niq_latency()) { stream->remote_recv_report.niq_latency = - packet::ntp_2_nanoseconds(blk.niq_delay()); + packet::ntp_2_nanoseconds(blk.niq_latency()); + } + + if (blk.has_niq_stalling()) { + stream->remote_recv_report.niq_stalling = + packet::ntp_2_nanoseconds(blk.niq_stalling()); } update_stream_(*stream); @@ -820,7 +825,7 @@ void Reporter::generate_delay_metrics_block(size_t addr_index, } if (stream->local_recv_report->e2e_latency > 0) { - blk.set_e2e_delay( + blk.set_e2e_latency( packet::nanoseconds_2_ntp(stream->local_recv_report->e2e_latency)); } } @@ -841,8 +846,10 @@ void Reporter::generate_queue_metrics_block(size_t addr_index, blk.set_metric_flag(header::MetricFlag_SampledValue); if (stream->local_recv_report->niq_latency > 0) { - blk.set_niq_delay( + blk.set_niq_latency( packet::nanoseconds_2_ntp(stream->local_recv_report->niq_latency)); + blk.set_niq_stalling( + packet::nanoseconds_2_ntp(stream->local_recv_report->niq_stalling)); } } diff --git a/src/internal_modules/roc_rtcp/reports.h b/src/internal_modules/roc_rtcp/reports.h index 9c59d62de..aae0cf285 100644 --- a/src/internal_modules/roc_rtcp/reports.h +++ b/src/internal_modules/roc_rtcp/reports.h @@ -150,6 +150,10 @@ struct RecvReport { //! An estimate of how much media is buffered in receiver packet queue. core::nanoseconds_t niq_latency; + //! Network incoming queue stalling. + //! How much time elapsed since last received packet. + core::nanoseconds_t niq_stalling; + //! Estimated end-to-end latency. //! An estimate of the time from recording a frame on sender to playing it //! on receiver. @@ -179,6 +183,7 @@ struct RecvReport { , cum_loss(0) , jitter(0) , niq_latency(0) + , niq_stalling(0) , e2e_latency(0) , clock_offset(0) , rtt(0) { diff --git a/src/public_api/include/roc/config.h b/src/public_api/include/roc/config.h index 3df384966..f0e5aac19 100644 --- a/src/public_api/include/roc/config.h +++ b/src/public_api/include/roc/config.h @@ -407,30 +407,31 @@ typedef enum roc_clock_source { ROC_CLOCK_SOURCE_INTERNAL = 1 } roc_clock_source; -/** Clock synchronization algorithm. - * Defines how sender and receiver clocks are synchronized. +/** Latency tuner backend. + * Defines which latency is monitored and tuned by latency tuner. */ -typedef enum roc_clock_sync_backend { - /** Disable clock synchronization. - * - * In this mode, sender and receiver clocks are not synchronized. This mode is - * generally not recommended, since clock drift will lead to periodic playback - * disruptions caused by underruns and overruns. - */ - ROC_CLOCK_SYNC_BACKEND_DISABLE = -1, - +typedef enum roc_latency_tuner_backend { /** Default backend. - * Current default is \c ROC_CLOCK_SYNC_BACKEND_NIQ. + * Current default is \c ROC_LATENCY_TUNER_BACKEND_NIQ. */ - ROC_CLOCK_SYNC_BACKEND_DEFAULT = 0, + ROC_LATENCY_TUNER_BACKEND_DEFAULT = 0, - /** Clock synchronization based on network incoming queue size. + /** Latency tuning is based on network incoming queue length. + * + * In this mode, latency is defined as incoming queue length (in nanoseconds). + * Latency tuner monitors queue length and and adjusts playback clock speed + * to keep queue length close to configured target latency. + * + * Keeping constant queue length allows to match playback clock speed with the + * capture clock speed and to keep the overall latency constant (yet unknown). * - * In this mode, receiver monitors incoming queue size and adjusts playback clock - * speed to match the estimated capture clock speed. + * On receiver, this backend is always available, without any protocol help. + * On sender, this backend works only if RTCP is enabled and both sender and + * receiver are implemented with roc-toolkit, as it relies on a non-standard + * RTCP extension to report receiver queue length to sender. * * Pros: - * - works with any protocol (does not require RTCP or NTP) + * - works with any protocol if used on receiver (does not require RTCP or NTP) * * Cons: * - synchronizes only clock speed, but not position; different receivers will @@ -438,22 +439,40 @@ typedef enum roc_clock_sync_backend { * - affected by network jitter; spikes in packet delivery will cause slow * oscillations in clock speed */ - ROC_CLOCK_SYNC_BACKEND_NIQ = 2 -} roc_clock_sync_backend; + ROC_LATENCY_TUNER_BACKEND_NIQ = 2 +} roc_latency_tuner_backend; -/** Clock synchronization profile. - * Defines what latency and jitter are tolerated by clock synchronization algorithm. +/** Latency tuner profile. + * Defines which algorithm is used by latency tuner. */ -typedef enum roc_clock_sync_profile { +typedef enum roc_latency_tuner_profile { /** Default profile. * - * When \ref ROC_CLOCK_SYNC_BACKEND_NIQ is used, selects \ref - * ROC_CLOCK_SYNC_PROFILE_RESPONSIVE if target latency is low, and \ref - * ROC_CLOCK_SYNC_PROFILE_GRADUAL if target latency is high. + * On receiver, when \ref ROC_LATENCY_TUNER_BACKEND_NIQ is used, selects \ref + * ROC_LATENCY_TUNER_PROFILE_RESPONSIVE if target latency is low, and \ref + * ROC_LATENCY_TUNER_PROFILE_GRADUAL if target latency is high. + * + * On sender, selects \ref ROC_LATENCY_TUNER_PROFILE_INTACT. + */ + ROC_LATENCY_TUNER_PROFILE_DEFAULT = 0, + + /** No latency tuning. + * + * In this mode, clock speed is not adjusted. Default on sender. + * + * You can set this mode on receiver, and set some other mode on sender, to + * do latency tuning on sender side instead of recever side. It's useful + * when receiver is CPU-constrained and sender is not, because latency tuner + * relies on resampling, which is CPU-demanding. + * + * You can also set this mode on both sender and receiver if you don't need + * latency tuning at all. However, if sender and receiver have independent + * clocks (which is typically the case), clock drift will lead to periodic + * playback disruptions caused by underruns and overruns. */ - ROC_CLOCK_SYNC_PROFILE_DEFAULT = 0, + ROC_LATENCY_TUNER_PROFILE_INTACT = 1, - /** Responsive clock adjustment. + /** Responsive latency tuning. * * Clock speed is adjusted quickly and accurately. * @@ -461,16 +480,16 @@ typedef enum roc_clock_sync_profile { * \ref ROC_RESAMPLER_BACKEND_BUILTIN. * * Pros: - * - allows very low latency or synchronization error + * - allows very low latency and synchronization error * * Cons: * - does not work well with some resampler backends - * - does not work well with \ref ROC_CLOCK_SYNC_BACKEND_NIQ + * - does not work well with \ref ROC_LATENCY_TUNER_BACKEND_NIQ * if network jitter is high */ - ROC_CLOCK_SYNC_PROFILE_RESPONSIVE = 1, + ROC_LATENCY_TUNER_PROFILE_RESPONSIVE = 2, - /** Gradual clock adjustment. + /** Gradual latency tuning. * * Clock speed is adjusted slowly and smoothly. * @@ -479,10 +498,10 @@ typedef enum roc_clock_sync_profile { * - works well with any resampler backend * * Cons: - * - does not allow very low latency or synchronization error + * - does not allow very low latency and synchronization error */ - ROC_CLOCK_SYNC_PROFILE_GRADUAL = 2 -} roc_clock_sync_profile; + ROC_LATENCY_TUNER_PROFILE_GRADUAL = 3 +} roc_latency_tuner_profile; /** Resampler backend. * Affects CPU usage, quality, and clock synchronization precision. @@ -492,7 +511,7 @@ typedef enum roc_resampler_backend { /** Default backend. * * Selects \ref ROC_RESAMPLER_BACKEND_BUILTIN when using \ref - * ROC_CLOCK_SYNC_PROFILE_RESPONSIVE, or when SpeexDSP is disabled. + * ROC_LATENCY_TUNER_PROFILE_RESPONSIVE, or when SpeexDSP is disabled. * * Otherwise, selects \ref ROC_RESAMPLER_BACKEND_SPEEX. */ @@ -510,7 +529,7 @@ typedef enum roc_resampler_backend { * * This backend is always available. * - * Recommended for \ref ROC_CLOCK_SYNC_PROFILE_RESPONSIVE and on good CPUs. + * Recommended for \ref ROC_LATENCY_TUNER_PROFILE_RESPONSIVE and on good CPUs. */ ROC_RESAMPLER_BACKEND_BUILTIN = 1, @@ -523,7 +542,7 @@ typedef enum roc_resampler_backend { * * This backend is available only when SpeexDSP was enabled at build time. * - * Recommended for \ref ROC_CLOCK_SYNC_PROFILE_GRADUAL and on cheap CPUs. + * Recommended for \ref ROC_LATENCY_TUNER_PROFILE_GRADUAL and on cheap CPUs. */ ROC_RESAMPLER_BACKEND_SPEEX = 2, @@ -582,15 +601,19 @@ typedef enum roc_resampler_profile { */ typedef struct roc_context_config { /** Maximum size in bytes of a network packet. + * * Defines the amount of bytes allocated per network packet. * Sender and receiver won't handle packets larger than this. + * * If zero, default value is used. */ unsigned int max_packet_size; /** Maximum size in bytes of an audio frame. + * * Defines the amount of bytes allocated per intermediate internal frame in the * pipeline. Does not limit the size of the frames provided by user. + * * If zero, default value is used. */ unsigned int max_frame_size; @@ -606,9 +629,11 @@ typedef struct roc_context_config { */ typedef struct roc_sender_config { /** The encoding used in frames passed to sender. + * * Frame encoding defines sample format, channel layout, and sample rate in local * frames created by user and passed to sender. - * Should be set (zero value is invalid). + * + * Should be set explicitly (zero value is invalid). */ roc_media_encoding frame_encoding; @@ -632,24 +657,30 @@ typedef struct roc_sender_config { roc_packet_encoding packet_encoding; /** The length of the packets produced by sender, in nanoseconds. + * * Number of nanoseconds encoded per packet. * The samples written to the sender are buffered until the full packet is * accumulated or the sender is flushed or closed. Larger number reduces * packet overhead but also increases latency. + * * If zero, default value is used. */ unsigned long long packet_length; /** Enable packet interleaving. + * * If non-zero, the sender shuffles packets before sending them. This * may increase robustness but also increases latency. */ unsigned int packet_interleaving; /** FEC encoding to use. - * If non-zero, the sender employs a FEC encoding to generate redundant packets - * which may be used on receiver to restore lost packets. This requires both - * sender and receiver to use two separate source and repair endpoints. + * + * If FEC is enabled, the sender employs a FEC encoding to generate redundant + * packet which may be used on receiver to restore lost packets. This requires + * both sender and receiver to use two separate source and repair endpoints. + * + * If zero, default encoding is used (\ref ROC_FEC_ENCODING_DEFAULT). */ roc_fec_encoding fec_encoding; @@ -684,20 +715,74 @@ typedef struct roc_sender_config { /** Clock source to use. * Defines whether write operation will be blocking or non-blocking. - * If zero, default value is used (\c ROC_CLOCK_SOURCE_EXTERNAL). + * + * If zero, \ref ROC_CLOCK_SOURCE_EXTERNAL is used. */ roc_clock_source clock_source; - /** Resampler backend to use. - * If zero, default value is used. + /** Latency tuner backend. + * Defines which latency is monitored and controlled by latency tuner. + * Defines semantics of \c target_latency field. + * + * If zero, default backend is used (\ref ROC_LATENCY_TUNER_BACKEND_DEFAULT). + */ + roc_latency_tuner_backend latency_tuner_backend; + + /** Latency tuner profile. + * Defines which algorithm is used by latency tuner. + * + * If zero, default profile is used (\ref ROC_LATENCY_TUNER_PROFILE_DEFAULT). + * + * By default, latency tuning is disabled on sender. If you enable it on sender, + * you need to disable it on receiver. In that case you also need to set + * \c target_latency to the same value on both sides. + */ + roc_latency_tuner_profile latency_tuner_profile; + + /** Resampler backend. + * Affects CPU usage, quality, and clock synchronization precision + * (if latency tuning is enabled). + * + * If zero, default backend is used (\ref ROC_RESAMPLER_BACKEND_DEFAULT). */ roc_resampler_backend resampler_backend; - /** Resampler profile to use. - * If non-zero, the sender employs resampler if the frame sample rate differs - * from the packet sample rate. + /** Resampler profile. + * Affects CPU usage and quality. + * + * If zero, default profile is used (\ref ROC_RESAMPLER_PROFILE_DEFAULT). */ roc_resampler_profile resampler_profile; + + /** Target latency, in nanoseconds. + * + * Exact semantics of this field, i.e. how latency is defined, depends on + * \c latency_tuner_backend field. + * + * If latency tuning is enabled on sender (\c latency_tuner_profile), it will + * adjust its clock to keep actual latency as close as possible to the target. + * + * If latency tolerance is set on sender (\c latency_tolerance), it will + * restart session when actual latency differs from target too much. + * + * If latency tuning or latency tolerance is enabled on sender, you should + * explicitly set target latency to a non-zero value, there is no default. + * + * By default, latency tuning is disabled on sender. If you enable it on sender, + * you need to disable it on receiver. You also need to set \c target_latency + * to the same value on both sides. + */ + unsigned long long target_latency; + + /** Maximum allowed delta between current and target latency, in nanoseconds. + * + * When set, if delta between actual latency and target latency becomes larger + * than the tolerance, the session is restarted. + * + * If zero, default value is used based on \c target_latency, if it's set, or + * checks are disabled if it's not set. If negative, the checks are disabled. + */ + long long latency_tolerance; } roc_sender_config; /** Receiver configuration. @@ -710,75 +795,105 @@ typedef struct roc_sender_config { */ typedef struct roc_receiver_config { /** The encoding used in frames returned by receiver. + * * Frame encoding defines sample format, channel layout, and sample rate in local * frames returned by receiver to user. + * * Should be set (zero value is invalid). */ roc_media_encoding frame_encoding; /** Clock source. * Defines whether read operation will be blocking or non-blocking. + * * If zero, \ref ROC_CLOCK_SOURCE_EXTERNAL is used. */ roc_clock_source clock_source; - /** Clock synchronization backend. - * Defines how sender and receiver clocks are synchronized. - * If zero, default value is used. + /** Latency tuner backend. + * Defines which latency is monitored and controlled by latency tuner. + * Defines semantics of \c target_latency field. + * + * If zero, default backend is used (\ref ROC_LATENCY_TUNER_BACKEND_DEFAULT). */ - roc_clock_sync_backend clock_sync_backend; + roc_latency_tuner_backend latency_tuner_backend; - /** Clock synchronization profile. - * Defines what latency and network jitter are tolerated. - * If zero, default value is used. + /** Latency tuner profile. + * Defines which algorithm is used by latency tuner. + * + * If zero, default profile is used (\ref ROC_LATENCY_TUNER_PROFILE_DEFAULT). + * + * By default, latency tuning is enabled on receiver. If you disable it on receiver, + * you usually need to enable it on sender. In that case you also need to set + * \c target_latency to the same value on both sides. */ - roc_clock_sync_profile clock_sync_profile; + roc_latency_tuner_profile latency_tuner_profile; /** Resampler backend. - * Affects CPU usage, quality, and clock synchronization precision. - * If zero, default value is used. + * Affects CPU usage, quality, and clock synchronization precision + * (if latency tuning is enabled). + * + * If zero, default backend is used (\ref ROC_RESAMPLER_BACKEND_DEFAULT). */ roc_resampler_backend resampler_backend; /** Resampler profile. * Affects CPU usage and quality. - * If zero, default value is used. + * + * If zero, default profile is used (\ref ROC_RESAMPLER_PROFILE_DEFAULT). */ roc_resampler_profile resampler_profile; /** Target latency, in nanoseconds. - * The session will not start playing until it accumulates the requested latency. - * Then, if clock synchronization is enabled, the session will adjust its clock to - * keep actual latency as close as possible to the target latency. - * If zero, default value is used. + * + * Exact semantics of this field, i.e. how latency is defined, depends on + * \c latency_tuner_backend field. + * + * If latency tuning is enabled on receiver (\c latency_tuner_profile), it will + * adjust its clock to keep actual latency as close as possible to the target. + * + * If latency tolerance is set on receiver (\c latency_tolerance), it will + * restart session when actual latency differs from target too much. + * + * If zero, default value is used, unless latency tuning is disabled, in which + * case it should be specified explicitly. */ unsigned long long target_latency; /** Maximum allowed delta between current and target latency, in nanoseconds. - * If session latency differs from the target latency by more than given value, the - * session is terminated (it can then automatically restart). Receiver itself is - * not terminated; if there are no sessions, it will produce zeros. - * If zero, default value is used. + * + * If the actual latency differs from the target by more than the tolerance, the + * session is terminated (it can then restart if traffic keeps coming). Receiver + * itself is not terminated; if there are no sessions, it produces zeros. + * + * If zero, default value is used based on \c target_latency. If negative, + * the checks are disabled. */ - unsigned long long latency_tolerance; + long long latency_tolerance; /** Timeout for the lack of playback, in nanoseconds. + * * If there is no playback during this period, the session is terminated (it can - * then automatically restart). Receiver itself is not terminated; if there are - * no sessions, it will produce zeros. + * then restart if traffic keeps coming). Receiver itself is not terminated; if + * there are no sessions, it produces zeros. + * * This mechanism allows to detect dead, hanging, or incompatible clients that * generate unparseable packets. - * If zero, default value is used. If negative, the timeout is disabled. + * + * If zero, default value is used. If negative, the checks are disabled. */ long long no_playback_timeout; /** Timeout for choppy playback, in nanoseconds. - * If there is constant stuttering during this period, the session is terminated (it - * can then automatically restart). Receiver itself is not terminated; if there are - * no sessions, it will produce zeros. + * + * If there is constant stuttering during this period, the session is terminated + * (it can then restart if traffic keeps coming). Receiver itself is not terminated; + * if there are no sessions, it produces zeros. + * * This mechanism allows to detect situations when playback continues but there * are frequent glitches, for example because there is a high ratio of late packets. - * If zero, default value is used. If negative, the timeout is disabled. + * + * If zero, default value is used. If negative, the checks are disabled. */ long long choppy_playback_timeout; } roc_receiver_config; diff --git a/src/public_api/src/adapters.cpp b/src/public_api/src/adapters.cpp index ff02d06a4..0c607f569 100644 --- a/src/public_api/src/adapters.cpp +++ b/src/public_api/src/adapters.cpp @@ -94,6 +94,16 @@ bool sender_config_from_user(node::Context& context, out.packet_length = (core::nanoseconds_t)in.packet_length; } + if (in.target_latency != 0) { + out.latency.target_latency = (core::nanoseconds_t)in.target_latency; + } + + if (in.latency_tolerance < 0) { + out.latency.latency_tolerance = 0; + } else if (in.latency_tolerance > 0) { + out.latency.latency_tolerance = (core::nanoseconds_t)in.latency_tolerance; + } + out.enable_timing = false; out.enable_auto_cts = true; @@ -144,7 +154,9 @@ bool receiver_config_from_user(node::Context&, (core::nanoseconds_t)in.target_latency; } - if (in.latency_tolerance != 0) { + if (in.latency_tolerance < 0) { + out.default_session.latency.latency_tolerance = 0; + } else if (in.latency_tolerance > 0) { out.default_session.latency.latency_tolerance = (core::nanoseconds_t)in.latency_tolerance; } @@ -177,18 +189,18 @@ bool receiver_config_from_user(node::Context&, return false; } - if (!clock_sync_backend_from_user(out.default_session.latency.fe_input, - in.clock_sync_backend)) { + if (!latency_tuner_backend_from_user(out.default_session.latency.tuner_backend, + in.latency_tuner_backend)) { roc_log(LogError, - "bad configuration: invalid roc_receiver_config.clock_sync_backend:" + "bad configuration: invalid roc_receiver_config.latency_tuner_backend:" " should be valid enum value"); return false; } - if (!clock_sync_profile_from_user(out.default_session.latency.fe_profile, - in.clock_sync_profile)) { + if (!latency_tuner_profile_from_user(out.default_session.latency.tuner_profile, + in.latency_tuner_profile)) { roc_log(LogError, - "bad configuration: invalid roc_receiver_config.clock_sync_profile:" + "bad configuration: invalid roc_receiver_config.latency_tuner_profile:" " should be valid enum value"); return false; } @@ -369,19 +381,15 @@ bool clock_source_from_user(bool& out_timing, roc_clock_source in) { } ROC_ATTR_NO_SANITIZE_UB -bool clock_sync_backend_from_user(audio::FreqEstimatorInput& out, - roc_clock_sync_backend in) { +bool latency_tuner_backend_from_user(audio::LatencyTunerBackend& out, + roc_latency_tuner_backend in) { switch (enum_from_user(in)) { - case ROC_CLOCK_SYNC_BACKEND_DISABLE: - out = audio::FreqEstimatorInput_Disable; + case ROC_LATENCY_TUNER_BACKEND_DEFAULT: + out = audio::LatencyTunerBackend_Default; return true; - case ROC_CLOCK_SYNC_BACKEND_DEFAULT: - out = audio::FreqEstimatorInput_Default; - return true; - - case ROC_CLOCK_SYNC_BACKEND_NIQ: - out = audio::FreqEstimatorInput_NiqLatency; + case ROC_LATENCY_TUNER_BACKEND_NIQ: + out = audio::LatencyTunerBackend_Niq; return true; } @@ -389,19 +397,23 @@ bool clock_sync_backend_from_user(audio::FreqEstimatorInput& out, } ROC_ATTR_NO_SANITIZE_UB -bool clock_sync_profile_from_user(audio::FreqEstimatorProfile& out, - roc_clock_sync_profile in) { +bool latency_tuner_profile_from_user(audio::LatencyTunerProfile& out, + roc_latency_tuner_profile in) { switch (enum_from_user(in)) { - case ROC_CLOCK_SYNC_PROFILE_DEFAULT: - out = audio::FreqEstimatorProfile_Default; + case ROC_LATENCY_TUNER_PROFILE_DEFAULT: + out = audio::LatencyTunerProfile_Default; + return true; + + case ROC_LATENCY_TUNER_PROFILE_INTACT: + out = audio::LatencyTunerProfile_Intact; return true; - case ROC_CLOCK_SYNC_PROFILE_RESPONSIVE: - out = audio::FreqEstimatorProfile_Responsive; + case ROC_LATENCY_TUNER_PROFILE_RESPONSIVE: + out = audio::LatencyTunerProfile_Responsive; return true; - case ROC_CLOCK_SYNC_PROFILE_GRADUAL: - out = audio::FreqEstimatorProfile_Gradual; + case ROC_LATENCY_TUNER_PROFILE_GRADUAL: + out = audio::LatencyTunerProfile_Gradual; return true; } diff --git a/src/public_api/src/adapters.h b/src/public_api/src/adapters.h index c9815de0e..473d82d2e 100644 --- a/src/public_api/src/adapters.h +++ b/src/public_api/src/adapters.h @@ -45,10 +45,10 @@ bool channel_set_from_user(audio::ChannelSet& out, bool clock_source_from_user(bool& out_timing, roc_clock_source in); -bool clock_sync_backend_from_user(audio::FreqEstimatorInput& out, - roc_clock_sync_backend in); -bool clock_sync_profile_from_user(audio::FreqEstimatorProfile& out, - roc_clock_sync_profile in); +bool latency_tuner_backend_from_user(audio::LatencyTunerBackend& out, + roc_latency_tuner_backend in); +bool latency_tuner_profile_from_user(audio::LatencyTunerProfile& out, + roc_latency_tuner_profile in); bool resampler_backend_from_user(audio::ResamplerBackend& out, roc_resampler_backend in); bool resampler_profile_from_user(audio::ResamplerProfile& out, roc_resampler_profile in); diff --git a/src/tests/public_api/test_sender_encoder_receiver_decoder.cpp b/src/tests/public_api/test_sender_encoder_receiver_decoder.cpp index fc1c36d8d..3c3207b90 100644 --- a/src/tests/public_api/test_sender_encoder_receiver_decoder.cpp +++ b/src/tests/public_api/test_sender_encoder_receiver_decoder.cpp @@ -60,7 +60,7 @@ TEST_GROUP(sender_encoder_receiver_decoder) { receiver_conf.frame_encoding.format = ROC_FORMAT_PCM_FLOAT32; receiver_conf.frame_encoding.channels = ROC_CHANNEL_LAYOUT_STEREO; receiver_conf.clock_source = ROC_CLOCK_SOURCE_EXTERNAL; - receiver_conf.clock_sync_backend = ROC_CLOCK_SYNC_BACKEND_DISABLE; + receiver_conf.latency_tuner_profile = ROC_LATENCY_TUNER_PROFILE_INTACT; receiver_conf.target_latency = test::Latency * 1000000000ull / test::SampleRate; receiver_conf.no_playback_timeout = test::Timeout * 1000000000ull / test::SampleRate; diff --git a/src/tests/public_api/test_sender_receiver.cpp b/src/tests/public_api/test_sender_receiver.cpp index 62791a1ad..e4a4fadfb 100644 --- a/src/tests/public_api/test_sender_receiver.cpp +++ b/src/tests/public_api/test_sender_receiver.cpp @@ -112,7 +112,7 @@ TEST_GROUP(sender_receiver) { } receiver_conf.clock_source = ROC_CLOCK_SOURCE_INTERNAL; - receiver_conf.clock_sync_backend = ROC_CLOCK_SYNC_BACKEND_DISABLE; + receiver_conf.latency_tuner_profile = ROC_LATENCY_TUNER_PROFILE_INTACT; receiver_conf.target_latency = test::Latency * 1000000000ull / test::SampleRate; receiver_conf.no_playback_timeout = test::Timeout * 1000000000ull / test::SampleRate; diff --git a/src/tests/roc_pipeline/test_loopback_sink_2_source.cpp b/src/tests/roc_pipeline/test_loopback_sink_2_source.cpp index e0cc509bc..43259bc43 100644 --- a/src/tests/roc_pipeline/test_loopback_sink_2_source.cpp +++ b/src/tests/roc_pipeline/test_loopback_sink_2_source.cpp @@ -166,7 +166,8 @@ ReceiverConfig make_receiver_config(audio::ChannelMask frame_channels, config.common.rtcp.inactivity_timeout = Timeout * core::Second / SampleRate; - config.default_session.latency.fe_input = audio::FreqEstimatorInput_Disable; + config.default_session.latency.tuner_backend = audio::LatencyTunerBackend_Niq; + config.default_session.latency.tuner_profile = audio::LatencyTunerProfile_Intact; config.default_session.latency.target_latency = Latency * core::Second / SampleRate; config.default_session.watchdog.no_playback_timeout = Timeout * core::Second / SampleRate; diff --git a/src/tests/roc_pipeline/test_receiver_loop.cpp b/src/tests/roc_pipeline/test_receiver_loop.cpp index fb4e2d089..873139e7b 100644 --- a/src/tests/roc_pipeline/test_receiver_loop.cpp +++ b/src/tests/roc_pipeline/test_receiver_loop.cpp @@ -105,7 +105,8 @@ TEST_GROUP(receiver_loop) { ReceiverConfig config; void setup() { - config.default_session.latency.fe_input = audio::FreqEstimatorInput_Disable; + config.default_session.latency.tuner_backend = audio::LatencyTunerBackend_Niq; + config.default_session.latency.tuner_profile = audio::LatencyTunerProfile_Intact; } }; diff --git a/src/tests/roc_pipeline/test_receiver_source.cpp b/src/tests/roc_pipeline/test_receiver_source.cpp index e61f0c99d..aacb21751 100644 --- a/src/tests/roc_pipeline/test_receiver_source.cpp +++ b/src/tests/roc_pipeline/test_receiver_source.cpp @@ -125,7 +125,8 @@ TEST_GROUP(receiver_source) { config.common.enable_timing = false; config.common.enable_profiling = true; - config.default_session.latency.fe_input = audio::FreqEstimatorInput_Disable; + config.default_session.latency.tuner_backend = audio::LatencyTunerBackend_Niq; + config.default_session.latency.tuner_profile = audio::LatencyTunerProfile_Intact; config.default_session.latency.target_latency = Latency * core::Second / (int)output_sample_spec.sample_rate(); config.default_session.latency.latency_tolerance = diff --git a/src/tests/roc_pipeline/test_sender_loop.cpp b/src/tests/roc_pipeline/test_sender_loop.cpp index fe1876428..923f79941 100644 --- a/src/tests/roc_pipeline/test_sender_loop.cpp +++ b/src/tests/roc_pipeline/test_sender_loop.cpp @@ -110,7 +110,8 @@ TEST_GROUP(sender_loop) { SenderConfig config; void setup() { - config.latency.fe_input = audio::FreqEstimatorInput_Disable; + config.latency.tuner_backend = audio::LatencyTunerBackend_Niq; + config.latency.tuner_profile = audio::LatencyTunerProfile_Intact; } }; diff --git a/src/tests/roc_pipeline/test_sender_sink.cpp b/src/tests/roc_pipeline/test_sender_sink.cpp index 8b648e39f..d726b8598 100644 --- a/src/tests/roc_pipeline/test_sender_sink.cpp +++ b/src/tests/roc_pipeline/test_sender_sink.cpp @@ -115,6 +115,9 @@ TEST_GROUP(sender_sink) { config.enable_timing = false; config.enable_profiling = true; + config.latency.tuner_backend = audio::LatencyTunerBackend_Niq; + config.latency.tuner_profile = audio::LatencyTunerProfile_Intact; + return config; } diff --git a/src/tests/roc_rtcp/test_builder_traverser.cpp b/src/tests/roc_rtcp/test_builder_traverser.cpp index 76d29435c..82008eef2 100644 --- a/src/tests/roc_rtcp/test_builder_traverser.cpp +++ b/src/tests/roc_rtcp/test_builder_traverser.cpp @@ -379,11 +379,12 @@ TEST(builder_traverser, rr_sdes_xr) { delay_metrics.set_mean_rtt(0x9100000); delay_metrics.set_min_rtt(0x9200000); delay_metrics.set_max_rtt(0x9300000); - delay_metrics.set_e2e_delay(0x9400000000000049); + delay_metrics.set_e2e_latency(0x9400000000000049); header::XrQueueMetricsBlock queue_metrics; queue_metrics.set_metric_flag(header::MetricFlag_SampledValue); queue_metrics.set_ssrc(1010); - queue_metrics.set_niq_delay(0xA100000); + queue_metrics.set_niq_latency(0xA100000); + queue_metrics.set_niq_stalling(0xA200000); // Synthesize part @@ -508,12 +509,13 @@ TEST(builder_traverser, rr_sdes_xr) { CHECK_EQUAL(0x9100000, xr_it.get_delay_metrics().mean_rtt()); CHECK_EQUAL(0x9200000, xr_it.get_delay_metrics().min_rtt()); CHECK_EQUAL(0x9300000, xr_it.get_delay_metrics().max_rtt()); - CHECK_EQUAL(0x9400000000000049, xr_it.get_delay_metrics().e2e_delay()); + CHECK_EQUAL(0x9400000000000049, xr_it.get_delay_metrics().e2e_latency()); CHECK_EQUAL(XrTraverser::Iterator::QUEUE_METRICS_BLOCK, xr_it.next()); CHECK_EQUAL(header::MetricFlag_SampledValue, xr_it.get_queue_metrics().metric_flag()); CHECK_EQUAL(1010, xr_it.get_queue_metrics().ssrc()); - CHECK_EQUAL(0xA100000, xr_it.get_queue_metrics().niq_delay()); + CHECK_EQUAL(0xA100000, xr_it.get_queue_metrics().niq_latency()); + CHECK_EQUAL(0xA200000, xr_it.get_queue_metrics().niq_stalling()); CHECK_EQUAL(XrTraverser::Iterator::END, xr_it.next()); CHECK_FALSE(xr_it.error()); diff --git a/src/tests/roc_rtcp/test_communicator.cpp b/src/tests/roc_rtcp/test_communicator.cpp index 06bb1be56..a32c017d8 100644 --- a/src/tests/roc_rtcp/test_communicator.cpp +++ b/src/tests/roc_rtcp/test_communicator.cpp @@ -380,7 +380,8 @@ RecvReport make_recv_report(core::nanoseconds_t time, report.cum_loss = (long)seed * 3000; report.jitter = seed * 400000; report.niq_latency = seed * 500000; - report.e2e_latency = seed * 6000; + report.niq_stalling = seed * 600000; + report.e2e_latency = seed * 7000; return report; } @@ -412,11 +413,13 @@ void expect_recv_report(const RecvReport& report, expect_timestamp("jitter", seed * 400000, report.jitter, TimestampEpsilon); if (expect_xr) { expect_timestamp("niq_latency", seed * 500000, report.niq_latency, RttEpsilon); - expect_timestamp("e2e_latency", seed * 6000, report.e2e_latency, + expect_timestamp("niq_stalling", seed * 600000, report.niq_stalling, RttEpsilon); + expect_timestamp("e2e_latency", seed * 7000, report.e2e_latency, TimestampEpsilon); CHECK(report.rtt >= 0); } else { CHECK(report.niq_latency == 0); + CHECK(report.niq_stalling == 0); CHECK(report.e2e_latency == 0); CHECK(report.rtt == 0); CHECK(report.clock_offset == 0); diff --git a/src/tests/roc_rtcp/test_headers.cpp b/src/tests/roc_rtcp/test_headers.cpp index 0f0dddd06..4c789d149 100644 --- a/src/tests/roc_rtcp/test_headers.cpp +++ b/src/tests/roc_rtcp/test_headers.cpp @@ -318,63 +318,94 @@ TEST(headers, metrics) { CHECK(!blk.has_max_rtt()); CHECK_EQUAL(0x0000FFFFFFFF0000, blk.max_rtt()); } - { // e2e_delay + { // e2e_latency header::XrDelayMetricsBlock blk; - CHECK(!blk.has_e2e_delay()); - CHECK_EQUAL(0xFFFFFFFFFFFFFFFF, blk.e2e_delay()); + CHECK(!blk.has_e2e_latency()); + CHECK_EQUAL(0xFFFFFFFFFFFFFFFF, blk.e2e_latency()); - blk.set_e2e_delay(0x0000AABBCCDD0000); - CHECK(blk.has_e2e_delay()); - CHECK_EQUAL(0x0000AABBCCDD0000, blk.e2e_delay()); + blk.set_e2e_latency(0x0000AABBCCDD0000); + CHECK(blk.has_e2e_latency()); + CHECK_EQUAL(0x0000AABBCCDD0000, blk.e2e_latency()); - blk.set_e2e_delay(0x0000AABBCCDD1111); - CHECK(blk.has_e2e_delay()); - CHECK_EQUAL(0x0000AABBCCDD1111, blk.e2e_delay()); + blk.set_e2e_latency(0x0000AABBCCDD1111); + CHECK(blk.has_e2e_latency()); + CHECK_EQUAL(0x0000AABBCCDD1111, blk.e2e_latency()); - blk.set_e2e_delay(0x1111AABBCCDD0000); - CHECK(blk.has_e2e_delay()); - CHECK_EQUAL(0x1111AABBCCDD0000, blk.e2e_delay()); + blk.set_e2e_latency(0x1111AABBCCDD0000); + CHECK(blk.has_e2e_latency()); + CHECK_EQUAL(0x1111AABBCCDD0000, blk.e2e_latency()); - blk.set_e2e_delay(0xFFFFFFFFFFFFFFFF); - CHECK(blk.has_e2e_delay()); - CHECK_EQUAL(0xFFFFFFFFFFFFFFFE, blk.e2e_delay()); + blk.set_e2e_latency(0xFFFFFFFFFFFFFFFF); + CHECK(blk.has_e2e_latency()); + CHECK_EQUAL(0xFFFFFFFFFFFFFFFE, blk.e2e_latency()); blk.reset(); - CHECK(!blk.has_e2e_delay()); - CHECK_EQUAL(0xFFFFFFFFFFFFFFFF, blk.e2e_delay()); + CHECK(!blk.has_e2e_latency()); + CHECK_EQUAL(0xFFFFFFFFFFFFFFFF, blk.e2e_latency()); } - { // niq_delay + { // niq_latency header::XrQueueMetricsBlock blk; - CHECK(!blk.has_niq_delay()); - CHECK_EQUAL(0x0000FFFFFFFF0000, blk.niq_delay()); + CHECK(!blk.has_niq_latency()); + CHECK_EQUAL(0x0000FFFFFFFF0000, blk.niq_latency()); - blk.set_niq_delay(0x0000AABBCCDD0000); - CHECK(blk.has_niq_delay()); - CHECK_EQUAL(0x0000AABBCCDD0000, blk.niq_delay()); + blk.set_niq_latency(0x0000AABBCCDD0000); + CHECK(blk.has_niq_latency()); + CHECK_EQUAL(0x0000AABBCCDD0000, blk.niq_latency()); - blk.set_niq_delay(0x0000AABBCCDD1111); - CHECK(blk.has_niq_delay()); - CHECK_EQUAL(0x0000AABBCCDD0000, blk.niq_delay()); + blk.set_niq_latency(0x0000AABBCCDD1111); + CHECK(blk.has_niq_latency()); + CHECK_EQUAL(0x0000AABBCCDD0000, blk.niq_latency()); - blk.set_niq_delay(0x0000AABBCCDD8888); - CHECK(blk.has_niq_delay()); - CHECK_EQUAL(0x0000AABBCCDE0000, blk.niq_delay()); + blk.set_niq_latency(0x0000AABBCCDD8888); + CHECK(blk.has_niq_latency()); + CHECK_EQUAL(0x0000AABBCCDE0000, blk.niq_latency()); - blk.set_niq_delay(0x1111AABBCCDD0000); - CHECK(blk.has_niq_delay()); - CHECK_EQUAL(0x0000FFFFFFFE0000, blk.niq_delay()); + blk.set_niq_latency(0x1111AABBCCDD0000); + CHECK(blk.has_niq_latency()); + CHECK_EQUAL(0x0000FFFFFFFE0000, blk.niq_latency()); - blk.set_niq_delay(0x0000FFFFFFFE8000); - CHECK(blk.has_niq_delay()); - CHECK_EQUAL(0x0000FFFFFFFE0000, blk.niq_delay()); + blk.set_niq_latency(0x0000FFFFFFFE8000); + CHECK(blk.has_niq_latency()); + CHECK_EQUAL(0x0000FFFFFFFE0000, blk.niq_latency()); blk.reset(); - CHECK(!blk.has_niq_delay()); - CHECK_EQUAL(0x0000FFFFFFFF0000, blk.niq_delay()); + CHECK(!blk.has_niq_latency()); + CHECK_EQUAL(0x0000FFFFFFFF0000, blk.niq_latency()); + } + { // niq_stalling + header::XrQueueMetricsBlock blk; + + CHECK(!blk.has_niq_stalling()); + CHECK_EQUAL(0x0000FFFFFFFF0000, blk.niq_stalling()); + + blk.set_niq_stalling(0x0000AABBCCDD0000); + CHECK(blk.has_niq_stalling()); + CHECK_EQUAL(0x0000AABBCCDD0000, blk.niq_stalling()); + + blk.set_niq_stalling(0x0000AABBCCDD1111); + CHECK(blk.has_niq_stalling()); + CHECK_EQUAL(0x0000AABBCCDD0000, blk.niq_stalling()); + + blk.set_niq_stalling(0x0000AABBCCDD8888); + CHECK(blk.has_niq_stalling()); + CHECK_EQUAL(0x0000AABBCCDE0000, blk.niq_stalling()); + + blk.set_niq_stalling(0x1111AABBCCDD0000); + CHECK(blk.has_niq_stalling()); + CHECK_EQUAL(0x0000FFFFFFFE0000, blk.niq_stalling()); + + blk.set_niq_stalling(0x0000FFFFFFFE8000); + CHECK(blk.has_niq_stalling()); + CHECK_EQUAL(0x0000FFFFFFFE0000, blk.niq_stalling()); + + blk.reset(); + + CHECK(!blk.has_niq_stalling()); + CHECK_EQUAL(0x0000FFFFFFFF0000, blk.niq_stalling()); } } diff --git a/src/tests/roc_rtcp/test_traverser.cpp b/src/tests/roc_rtcp/test_traverser.cpp index 72785e463..6c742ceb0 100644 --- a/src/tests/roc_rtcp/test_traverser.cpp +++ b/src/tests/roc_rtcp/test_traverser.cpp @@ -1200,13 +1200,14 @@ TEST(traverser, xr_fields) { delay_metrics.set_mean_rtt(0x600000); delay_metrics.set_min_rtt(0x700000); delay_metrics.set_max_rtt(0x800000); - delay_metrics.set_e2e_delay(0x9000000000000009); + delay_metrics.set_e2e_latency(0x9000000000000009); header::XrQueueMetricsBlock queue_metrics; queue_metrics.header().set_len_bytes(sizeof(header::XrQueueMetricsBlock)); queue_metrics.set_metric_flag(header::MetricFlag_SampledValue); queue_metrics.set_ssrc(666); - queue_metrics.set_niq_delay(0xA00000); + queue_metrics.set_niq_latency(0xA00000); + queue_metrics.set_niq_stalling(0xB00000); append_buffer(buff, &xr, sizeof(xr)); append_buffer(buff, &rrtr, sizeof(rrtr)); @@ -1263,13 +1264,14 @@ TEST(traverser, xr_fields) { CHECK_EQUAL(0x600000, xr_it.get_delay_metrics().mean_rtt()); CHECK_EQUAL(0x700000, xr_it.get_delay_metrics().min_rtt()); CHECK_EQUAL(0x800000, xr_it.get_delay_metrics().max_rtt()); - CHECK_EQUAL(0x9000000000000009, xr_it.get_delay_metrics().e2e_delay()); + CHECK_EQUAL(0x9000000000000009, xr_it.get_delay_metrics().e2e_latency()); CHECK_EQUAL(XrTraverser::Iterator::QUEUE_METRICS_BLOCK, xr_it.next()); CHECK_EQUAL(header::MetricFlag_SampledValue, xr_it.get_queue_metrics().metric_flag()); CHECK_EQUAL(666, xr_it.get_queue_metrics().ssrc()); - CHECK_EQUAL(0xA00000, xr_it.get_queue_metrics().niq_delay()); + CHECK_EQUAL(0xA00000, xr_it.get_queue_metrics().niq_latency()); + CHECK_EQUAL(0xB00000, xr_it.get_queue_metrics().niq_stalling()); CHECK_EQUAL(XrTraverser::Iterator::END, xr_it.next()); CHECK_FALSE(xr_it.error()); diff --git a/src/tools/roc_recv/cmdline.ggo b/src/tools/roc_recv/cmdline.ggo index 5128d7429..9f3d4659e 100644 --- a/src/tools/roc_recv/cmdline.ggo +++ b/src/tools/roc_recv/cmdline.ggo @@ -29,7 +29,7 @@ section "Options" option "reuseaddr" - "enable SO_REUSEADDR when binding sockets" optional - option "sess-latency" - "Session target latency, TIME units" + option "target-latency" - "Target latency, TIME units" string optional option "io-latency" - "Playback target latency, TIME units" @@ -56,11 +56,11 @@ section "Options" option "rate" - "Override output sample rate, Hz" int optional - option "clock-backend" - "Clock synchronization backend" - values="disable","niq" default="niq" enum optional + option "latency-backend" - "Which latency to use in latency tuner" + values="niq" default="niq" enum optional - option "clock-profile" - "Clock synchronization profile" - values="default","responsive","gradual" default="default" enum optional + option "latency-profile" - "Latency tuning profile" + values="default","responsive","gradual","intact" default="default" enum optional option "resampler-backend" - "Resampler backend" values="default","builtin","speex","speexdec" default="default" enum optional diff --git a/src/tools/roc_recv/main.cpp b/src/tools/roc_recv/main.cpp index 8ca0ff04a..dc3460f74 100644 --- a/src/tools/roc_recv/main.cpp +++ b/src/tools/roc_recv/main.cpp @@ -104,11 +104,11 @@ int main(int argc, char** argv) { sndio::BackendMap::instance().set_frame_size( io_config.frame_length, receiver_config.common.output_sample_spec); - if (args.sess_latency_given) { + if (args.target_latency_given) { if (!core::parse_duration( - args.sess_latency_arg, + args.target_latency_arg, receiver_config.default_session.latency.target_latency)) { - roc_log(LogError, "invalid --sess-latency"); + roc_log(LogError, "invalid --target-latency"); return 1; } } @@ -140,31 +140,31 @@ int main(int argc, char** argv) { } } - switch (args.clock_backend_arg) { - case clock_backend_arg_disable: - receiver_config.default_session.latency.fe_input = - audio::FreqEstimatorInput_Disable; - break; - case clock_backend_arg_niq: - receiver_config.default_session.latency.fe_input = - audio::FreqEstimatorInput_NiqLatency; + switch (args.latency_backend_arg) { + case latency_backend_arg_niq: + receiver_config.default_session.latency.tuner_backend = + audio::LatencyTunerBackend_Niq; break; default: break; } - switch (args.clock_profile_arg) { - case clock_profile_arg_default: - receiver_config.default_session.latency.fe_profile = - audio::FreqEstimatorProfile_Default; + switch (args.latency_profile_arg) { + case latency_profile_arg_default: + receiver_config.default_session.latency.tuner_profile = + audio::LatencyTunerProfile_Default; + break; + case latency_profile_arg_responsive: + receiver_config.default_session.latency.tuner_profile = + audio::LatencyTunerProfile_Responsive; break; - case clock_profile_arg_responsive: - receiver_config.default_session.latency.fe_profile = - audio::FreqEstimatorProfile_Responsive; + case latency_profile_arg_gradual: + receiver_config.default_session.latency.tuner_profile = + audio::LatencyTunerProfile_Gradual; break; - case clock_profile_arg_gradual: - receiver_config.default_session.latency.fe_profile = - audio::FreqEstimatorProfile_Gradual; + case latency_profile_arg_intact: + receiver_config.default_session.latency.tuner_profile = + audio::LatencyTunerProfile_Intact; break; default: break; diff --git a/src/tools/roc_send/cmdline.ggo b/src/tools/roc_send/cmdline.ggo index 2f67fb0d6..cb5f04939 100644 --- a/src/tools/roc_send/cmdline.ggo +++ b/src/tools/roc_send/cmdline.ggo @@ -20,9 +20,15 @@ section "Options" option "reuseaddr" - "enable SO_REUSEADDR when binding sockets" optional + option "target-latency" - "Target latency, TIME units" + string optional + option "io-latency" - "Recording target latency, TIME units" string optional + option "latency-tolerance" - "Maximum latency deviation, TIME units" + string optional + option "nbsrc" - "Number of source packets in FEC block" int optional @@ -44,6 +50,12 @@ section "Options" option "rate" - "Override input sample rate, Hz" int optional + option "latency-backend" - "Which latency to use in latency tuner" + values="niq" default="niq" enum optional + + option "latency-profile" - "Latency tuning profile" + values="responsive","gradual","intact" default="intact" enum optional + option "resampler-backend" - "Resampler backend" values="default","builtin","speex","speexdec" default="default" enum optional diff --git a/src/tools/roc_send/main.cpp b/src/tools/roc_send/main.cpp index 03a732f2a..3ac3a27cd 100644 --- a/src/tools/roc_send/main.cpp +++ b/src/tools/roc_send/main.cpp @@ -146,6 +146,44 @@ int main(int argc, char** argv) { sender_config.fec_writer.n_repair_packets = (size_t)args.nbrpr_arg; } + if (args.target_latency_given) { + if (!core::parse_duration(args.target_latency_arg, + sender_config.latency.target_latency)) { + roc_log(LogError, "invalid --target-latency"); + return 1; + } + } + + if (args.latency_tolerance_given) { + if (!core::parse_duration(args.latency_tolerance_arg, + sender_config.latency.latency_tolerance)) { + roc_log(LogError, "invalid --latency-tolerance"); + return 1; + } + } + + switch (args.latency_backend_arg) { + case latency_backend_arg_niq: + sender_config.latency.tuner_backend = audio::LatencyTunerBackend_Niq; + break; + default: + break; + } + + switch (args.latency_profile_arg) { + case latency_profile_arg_responsive: + sender_config.latency.tuner_profile = audio::LatencyTunerProfile_Responsive; + break; + case latency_profile_arg_gradual: + sender_config.latency.tuner_profile = audio::LatencyTunerProfile_Gradual; + break; + case latency_profile_arg_intact: + sender_config.latency.tuner_profile = audio::LatencyTunerProfile_Intact; + break; + default: + break; + } + switch (args.resampler_backend_arg) { case resampler_backend_arg_default: sender_config.resampler.backend = audio::ResamplerBackend_Default;