diff --git a/src/internal_modules/roc_audio/target_speexdsp/roc_audio/speex_resampler.cpp b/src/internal_modules/roc_audio/target_speexdsp/roc_audio/speex_resampler.cpp
index 67611c155..27ae9086a 100644
--- a/src/internal_modules/roc_audio/target_speexdsp/roc_audio/speex_resampler.cpp
+++ b/src/internal_modules/roc_audio/target_speexdsp/roc_audio/speex_resampler.cpp
@@ -102,6 +102,8 @@ SpeexResampler::SpeexResampler(core::IArena& arena,
     }
 
     startup_countdown_ = (size_t)speex_resampler_get_output_latency(speex_state_);
+    initial_input_latency_ = (size_t)speex_resampler_get_input_latency(speex_state_);
+    current_input_latency_diff_ = 0;
 
     valid_ = true;
 }
@@ -185,6 +187,9 @@ bool SpeexResampler::set_scaling(size_t input_rate, size_t output_rate, float mu
         return false;
     }
 
+    const ssize_t latency = (ssize_t)speex_resampler_get_input_latency(speex_state_);
+    current_input_latency_diff_ = latency - (ssize_t)initial_input_latency_;
+
     return true;
 }
 
@@ -254,7 +259,7 @@ size_t SpeexResampler::pop_output(sample_t* out_buf, size_t out_bufsz) {
 float SpeexResampler::n_left_to_process() const {
     roc_panic_if_not(is_valid());
 
-    return float(in_frame_size_ - in_frame_pos_);
+    return float(in_frame_size_ - in_frame_pos_) + float(current_input_latency_diff_);
 }
 
 void SpeexResampler::report_stats_() {
@@ -275,14 +280,13 @@ void SpeexResampler::report_stats_() {
     speex_resampler_get_rate(speex_state_, &in_rate, &out_rate);
 
     const int in_latency = speex_resampler_get_input_latency(speex_state_);
-    const int out_latency = speex_resampler_get_output_latency(speex_state_);
 
     roc_log(
         LogDebug,
         "speex resampler:"
-        " ratio_num=%u ratio_den=%u in_rate=%u out_rate=%u in_latency=%d out_latency=%d",
+        " ratio_num=%u ratio_den=%u in_rate=%u out_rate=%u in_latency=%d latency_diff=%d",
         (unsigned int)ratio_num, (unsigned int)ratio_den, (unsigned int)in_rate,
-        (unsigned int)out_rate, (int)in_latency, (int)out_latency);
+        (unsigned int)out_rate, (int)in_latency, (int)current_input_latency_diff_);
 }
 
 } // namespace audio
diff --git a/src/internal_modules/roc_audio/target_speexdsp/roc_audio/speex_resampler.h b/src/internal_modules/roc_audio/target_speexdsp/roc_audio/speex_resampler.h
index c31bb4e13..9ea8e2551 100644
--- a/src/internal_modules/roc_audio/target_speexdsp/roc_audio/speex_resampler.h
+++ b/src/internal_modules/roc_audio/target_speexdsp/roc_audio/speex_resampler.h
@@ -80,6 +80,12 @@ class SpeexResampler : public IResampler, public core::NonCopyable<> {
     // Counts how many output samples to throw away in order to
     // compensate resampler's inner latency.
     size_t startup_countdown_;
+    // Stores initial latency in order to track its further changes.
+    size_t initial_input_latency_;
+    // Stores how much speex resampler latency changed from the start, in order to
+    // reflect it in n_left_to_process() for better precision in capture timestamp
+    // calculations.
+    ssize_t current_input_latency_diff_;
 
     core::RateLimiter rate_limiter_;
 
diff --git a/src/tests/roc_audio/test_resampler.cpp b/src/tests/roc_audio/test_resampler.cpp
index 67c2b88c7..393147cd1 100644
--- a/src/tests/roc_audio/test_resampler.cpp
+++ b/src/tests/roc_audio/test_resampler.cpp
@@ -207,7 +207,7 @@ double timestamp_allowance(ResamplerBackend backend) {
     case ResamplerBackend_Builtin:
         return 0.1;
     case ResamplerBackend_Speex:
-        return 1;
+        return 5;
     case ResamplerBackend_SpeexDec:
         return 2;
     default:
@@ -653,11 +653,6 @@ TEST(resampler, timestamp_passthrough_reader) {
                         SampleSpec(supported_rates[n_orate], ChanLayout_Surround,
                                    ChanOrder_Smpte, ChMask);
 
-                    // FIXME: test fails if we're downsampling
-                    if (in_spec.sample_rate() >= out_spec.sample_rate()) {
-                        continue;
-                    }
-
                     core::SharedPtr<IResampler> resampler =
                         ResamplerMap::instance().new_resampler(
                             backend, arena, buffer_factory, supported_profiles[n_prof],
@@ -688,7 +683,10 @@ TEST(resampler, timestamp_passthrough_reader) {
                         {
                             Frame frame(samples, ROC_ARRAY_SIZE(samples));
                             CHECK(rreader.read(frame));
-                            CHECK(frame.capture_timestamp() >= start_ts);
+                            // Since CTS is estimated based scaling, it can happen
+                            // to be in past relative to the very first frame, but only
+                            // within allowed epsilon.
+                            CHECK(frame.capture_timestamp() >= start_ts - epsilon);
                             cur_ts = frame.capture_timestamp();
                         }
                         for (size_t i = 0; i < NumIterations; i++) {
@@ -768,11 +766,6 @@ TEST(resampler, timestamp_passthrough_writer) {
                         SampleSpec(supported_rates[n_orate], ChanLayout_Surround,
                                    ChanOrder_Smpte, ChMask);
 
-                    // FIXME: test fails if we're downsampling
-                    if (in_spec.sample_rate() >= out_spec.sample_rate()) {
-                        continue;
-                    }
-
                     core::SharedPtr<IResampler> resampler =
                         ResamplerMap::instance().new_resampler(
                             backend, arena, buffer_factory, supported_profiles[n_prof],