diff --git a/llo/plugin_observation.go b/llo/plugin_observation.go
index e25a126..1ad1a07 100644
--- a/llo/plugin_observation.go
+++ b/llo/plugin_observation.go
@@ -149,8 +149,10 @@ func (p *Plugin) observation(ctx context.Context, outctx ocr3types.OutcomeContex
 			// limit the call to Observe to 25ms less than that, to allow some
 			// headroom for serialization and other operations.
 			maxDurationObserve := p.MaxDurationObservation - 25*time.Millisecond
-			if maxDurationObserve < 0 {
-				maxDurationObserve = 0
+			if maxDurationObserve < 50*time.Millisecond {
+				// Don't ever allow LESS than 50ms for Observe even if it would
+				// log a warning
+				maxDurationObserve = 50 * time.Millisecond
 			}
 			observationCtx, cancel := context.WithTimeout(ctx, maxDurationObserve)
 			defer cancel()