Merge branch 'ccip-develop' into ccip-develop

.changeset/tricky-candles-matter.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#bugfix Memory leak fix on promwrapper

core/chains/evm/config/toml/defaults/BOB_Mainnet.toml

@@ -3,8 +3,8 @@ ChainID = '60808'
 ChainType = 'optimismBedrock'
 # finality_depth was: ~850
 FinalityDepth = 900
-# block_time was:  2s
-LogPollInterval = '5s'
+# block_time was:  2s, adding 1 second buffer 
+LogPollInterval = '3s'
 
 # finality_depth * block_time / 60 secs = ~30  min (finality time)
 NoNewFinalizedHeadsThreshold = '35m'
@@ -16,8 +16,8 @@ EIP1559DynamicFees = true
 Mode = 'FeeHistory'
 
 [GasEstimator.FeeHistory]
-# block_time was: 2s
-CacheTimeout = '2s'
+# block_time was: 2s, per recommendation skip 1-2 blocks
+CacheTimeout = '4s'
 
 [GasEstimator.BlockHistory]
 BlockHistorySize = 100

core/chains/evm/config/toml/defaults/BOB_Testnet.toml

@@ -3,8 +3,8 @@ ChainID = '808813'
 ChainType = 'optimismBedrock'
 # finality_depth was: ~850
 FinalityDepth = 900
-# block_time was:  2s
-LogPollInterval = '5s'  
+# block_time was:  2s, adding 1 second buffer 
+LogPollInterval = '3s'  
 
 # finality_depth * block_time / 60 secs = ~30  min (finality time)
 NoNewFinalizedHeadsThreshold = '35m'
@@ -16,8 +16,8 @@ EIP1559DynamicFees = true
 Mode = 'FeeHistory'
 
 [GasEstimator.FeeHistory]
-# block_time was: 2s
-CacheTimeout = '2s'
+# block_time was: 2s, per recommendation skip 1-2 blocks
+CacheTimeout = '4s'
 
 [GasEstimator.BlockHistory]
 BlockHistorySize = 100

core/chains/evm/config/toml/defaults/Berachain_Testnet.toml

@@ -1,8 +1,8 @@
 ChainID = '80084'
 # finality_depth: instant
 FinalityDepth = 10 
-# block_time: 5s 
-LogPollInterval = '10s' 
+# block_time: 5s, adding 1 second buffer 
+LogPollInterval = '6s' 
 
 # finality_depth * block_time / 60 secs = ~0.8  min (finality time)
 NoNewFinalizedHeadsThreshold = '5m'
@@ -12,8 +12,8 @@ EIP1559DynamicFees = true
 Mode = 'FeeHistory'
 
 [GasEstimator.FeeHistory]
-# block_time was: 5s
-CacheTimeout = '5s'
+# block_time was: 5s, per recommendation skip 1-2 blocks
+CacheTimeout = '10s'
 
 [GasEstimator.BlockHistory]
 BlockHistorySize = 100

core/chains/evm/config/toml/defaults/Bsquared_Mainnet.toml

@@ -3,8 +3,8 @@ ChainID = '223'
 ChainType = 'optimismBedrock'
 # finality_depth was: ~1900
 FinalityDepth = 2000
-# block_time:  ~2s 
-LogPollInterval = '5s' 
+# block_time:  ~2s, adding 1 second buffer 
+LogPollInterval = '3s' 
 
 # finality_depth * block_time / 60 secs = ~66  min (finality time)
 NoNewFinalizedHeadsThreshold = '70m' 
@@ -16,8 +16,8 @@ EIP1559DynamicFees = true
 Mode = 'FeeHistory'
 
 [GasEstimator.FeeHistory]
-# block_time was: 2s
-CacheTimeout = '2s'
+# block_time was: 2s, per recommendation skip 1-2 blocks
+CacheTimeout = '4s'
 
 [GasEstimator.BlockHistory]
 BlockHistorySize = 100

core/chains/evm/config/toml/defaults/Bsquared_Testnet.toml

@@ -3,8 +3,8 @@ ChainID = '1123'
 ChainType = 'optimismBedrock'
 # finality_depth was: ~1900
 FinalityDepth = 2000
-# block_time:  ~2s 
-LogPollInterval = '5s'
+# block_time:  ~2s, adding 1 second buffer  
+LogPollInterval = '3s'
 
 # finality_depth * block_time / 60 secs = ~66  min (finality time)
 NoNewFinalizedHeadsThreshold = '70m'
@@ -16,8 +16,8 @@ EIP1559DynamicFees = true
 Mode = 'FeeHistory'
 
 [GasEstimator.FeeHistory]
-# block_time was: 2s
-CacheTimeout = '2s'
+# block_time was: 2s, per recommendation skip 1-2 blocks
+CacheTimeout = '4s'
 
 [GasEstimator.BlockHistory]
 BlockHistorySize = 100

core/chains/evm/config/toml/defaults/Unichain_Testnet.toml

@@ -3,12 +3,13 @@ ChainID = '1301'
 ChainType = 'optimismBedrock'
 # finality_depth was: ~1900
 FinalityDepth = 2000
-# block_time was:  ~1s 
-LogPollInterval = '5s'
+# block_time was:  ~1s, adding 1 second buffer 
+LogPollInterval = '2s'
 
 # batching_size_finalization_percentage = 30% according to the explorer batching view
-# ( batching_size_finalization_percentage * finality_depth) * block_time / 60 secs = ~33  min (finality time)
-NoNewFinalizedHeadsThreshold = '10m'
+# ( batching_size_finalization_percentage * finality_depth) * block_time / 60 secs = ~10  min (finality time)
+# After running soak tests using 10m threw issues as there are batchs that take 35m, so we are bumping it to 45m to be sure 
+NoNewFinalizedHeadsThreshold = '45m'
 
 FinalityTagEnabled = true
 
@@ -17,8 +18,9 @@ EIP1559DynamicFees = true
 Mode = 'FeeHistory'
 
 [GasEstimator.FeeHistory]
-# block_time was: 1s
-CacheTimeout = '1s'
+# block_time was: 1s, per recommendation skip 1-2 blocks
+CacheTimeout = '2s'
 
 [GasEstimator.BlockHistory]
+# As we see blocks containing between ~[8-12]tx, to get about ~1000 tx to check we would need to rougly go 100 tx back
 BlockHistorySize = 100

core/chains/evm/config/toml/defaults/Worldchain_Mainnet.toml

@@ -3,8 +3,8 @@ ChainID = '480'
 ChainType = 'optimismBedrock'
 # finality_depth was: ~2400
 FinalityDepth = 2500
-# block_time was:  2s 
-LogPollInterval = '5s'
+# block_time was:  2s, adding 1 second buffer   
+LogPollInterval = '3s'
 
 # finality_depth * block_time / 60 secs = ~83  min (finality time)
 NoNewFinalizedHeadsThreshold = '90m'
@@ -16,8 +16,8 @@ EIP1559DynamicFees = true
 Mode = 'FeeHistory'
 
 [GasEstimator.FeeHistory]
-# block_time was: 2s
-CacheTimeout = '2s'
+# block_time was: 2s, per recommendation skip 1-2 blocks
+CacheTimeout = '4s'
 
 [GasEstimator.BlockHistory]
 BlockHistorySize = 100

core/chains/evm/config/toml/defaults/Worldchain_Testnet.toml

@@ -3,8 +3,8 @@ ChainID = '4801'
 ChainType = 'optimismBedrock'
 # finality_depth was: ~2400
 FinalityDepth = 2500
-# block_time was:  2s 
-LogPollInterval = '5s'
+# block_time was:  2s, adding 1 second buffer  
+LogPollInterval = '3s'
 
 # finality_depth * block_time / 60 secs = ~83  min (finality time)
 NoNewFinalizedHeadsThreshold = '90m'
@@ -16,8 +16,8 @@ EIP1559DynamicFees = true
 Mode = 'FeeHistory'
 
 [GasEstimator.FeeHistory]
-# block_time was: 2s
-CacheTimeout = '2s'
+# block_time was: 2s, per recommendation skip 1-2 blocks
+CacheTimeout = '4s'
 
 [GasEstimator.BlockHistory]
 BlockHistorySize = 100

core/scripts/ccip/manual-execution/go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/ethereum/go-ethereum v1.11.3
 	github.com/pkg/errors v0.9.1
-	github.com/smartcontractkit/chain-selectors v1.0.28
+	github.com/smartcontractkit/chain-selectors v1.0.30
 	go.uber.org/multierr v1.1.0
 	golang.org/x/crypto v0.1.0
 )

core/scripts/ccip/manual-execution/go.sum

@@ -72,8 +72,8 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
-github.com/smartcontractkit/chain-selectors v1.0.28 h1:oY4/X4MJTUptVo0mG21xYYfni/78iqnqyS7LddQGeVo=
-github.com/smartcontractkit/chain-selectors v1.0.28/go.mod h1:xsKM0aN3YGcQKTPRPDDtPx2l4mlTN1Djmg0VVXV40b8=
+github.com/smartcontractkit/chain-selectors v1.0.30 h1:jk+xVRocgQ/uvKLSd1JzeHJ/v+EKu1BjrreaSGqKzjo=
+github.com/smartcontractkit/chain-selectors v1.0.30/go.mod h1:xsKM0aN3YGcQKTPRPDDtPx2l4mlTN1Djmg0VVXV40b8=
 github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=

core/scripts/go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.20.0
 	github.com/shopspring/decimal v1.4.0
-	github.com/smartcontractkit/chain-selectors v1.0.28
+	github.com/smartcontractkit/chain-selectors v1.0.30
 	github.com/smartcontractkit/chainlink-automation v1.0.4
 	github.com/smartcontractkit/chainlink-common v0.2.3-0.20240925085218-aded1b263ecc
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000

core/scripts/go.sum

@@ -1073,8 +1073,8 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/smartcontractkit/chain-selectors v1.0.28 h1:oY4/X4MJTUptVo0mG21xYYfni/78iqnqyS7LddQGeVo=
-github.com/smartcontractkit/chain-selectors v1.0.28/go.mod h1:xsKM0aN3YGcQKTPRPDDtPx2l4mlTN1Djmg0VVXV40b8=
+github.com/smartcontractkit/chain-selectors v1.0.30 h1:jk+xVRocgQ/uvKLSd1JzeHJ/v+EKu1BjrreaSGqKzjo=
+github.com/smartcontractkit/chain-selectors v1.0.30/go.mod h1:xsKM0aN3YGcQKTPRPDDtPx2l4mlTN1Djmg0VVXV40b8=
 github.com/smartcontractkit/chainlink-automation v1.0.4 h1:iyW181JjKHLNMnDleI8umfIfVVlwC7+n5izbLSFgjw8=
 github.com/smartcontractkit/chainlink-automation v1.0.4/go.mod h1:u4NbPZKJ5XiayfKHD/v3z3iflQWqvtdhj13jVZXj/cM=
 github.com/smartcontractkit/chainlink-common v0.2.3-0.20240925085218-aded1b263ecc h1:ALbyaoRzUSXQ2NhGFKVOyJqO22IB5yQjhjKWbIZGbrI=

core/services/ocr2/plugins/promwrapper/plugin.go

@@ -7,15 +7,23 @@ import (
 	"context"
 	"fmt"
 	"math/big"
-	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/patrickmn/go-cache"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/smartcontractkit/libocr/offchainreporting2plus/types"
 )
 
+const (
+	// defaultExpiration is the default expiration time for cache items.
+	defaultExpiration = 30 * time.Minute
+
+	// defaultCleanupInterval is the default interval for cache cleanup.
+	defaultCleanupInterval = 5 * time.Minute
+)
+
 // Type assertions, buckets and labels.
 var (
 	_       types.ReportingPlugin = &promPlugin{}
@@ -160,10 +168,10 @@ type (
 		chainID                       *big.Int
 		oracleID                      string
 		configDigest                  string
-		queryEndTimes                 sync.Map
-		observationEndTimes           sync.Map
-		reportEndTimes                sync.Map
-		acceptFinalizedReportEndTimes sync.Map
+		queryEndTimes                 *cache.Cache
+		observationEndTimes           *cache.Cache
+		reportEndTimes                *cache.Cache
+		acceptFinalizedReportEndTimes *cache.Cache
 		prometheusBackend             PrometheusBackend
 	}
 )
@@ -223,13 +231,17 @@ func New(
 	}
 
 	return &promPlugin{
-		wrapped:           plugin,
-		name:              name,
-		chainType:         chainType,
-		chainID:           chainID,
-		oracleID:          fmt.Sprintf("%d", config.OracleID),
-		configDigest:      common.Bytes2Hex(config.ConfigDigest[:]),
-		prometheusBackend: prometheusBackend,
+		wrapped:                       plugin,
+		name:                          name,
+		chainType:                     chainType,
+		chainID:                       chainID,
+		oracleID:                      fmt.Sprintf("%d", config.OracleID),
+		configDigest:                  common.Bytes2Hex(config.ConfigDigest[:]),
+		prometheusBackend:             prometheusBackend,
+		queryEndTimes:                 cache.New(defaultExpiration, defaultCleanupInterval),
+		observationEndTimes:           cache.New(defaultExpiration, defaultCleanupInterval),
+		reportEndTimes:                cache.New(defaultExpiration, defaultCleanupInterval),
+		acceptFinalizedReportEndTimes: cache.New(defaultExpiration, defaultCleanupInterval),
 	}
 }
 
@@ -238,7 +250,7 @@ func (p *promPlugin) Query(ctx context.Context, timestamp types.ReportTimestamp)
 	defer func() {
 		duration := float64(time.Now().UTC().Sub(start))
 		p.prometheusBackend.SetQueryDuration(getLabelsValues(p, timestamp), duration)
-		p.queryEndTimes.Store(timestamp, time.Now().UTC()) // note time at end of Query()
+		p.setEndTime(timestamp, p.queryEndTimes) // note time at end of Query()
 	}()
 
 	return p.wrapped.Query(ctx, timestamp)
@@ -249,17 +261,16 @@ func (p *promPlugin) Observation(ctx context.Context, timestamp types.ReportTime
 
 	// Report latency between Query() and Observation().
 	labelValues := getLabelsValues(p, timestamp)
-	if queryEndTime, ok := p.queryEndTimes.Load(timestamp); ok {
+	if queryEndTime, ok := p.queryEndTimes.Get(timestampToKey(timestamp)); ok {
 		latency := float64(start.Sub(queryEndTime.(time.Time)))
 		p.prometheusBackend.SetQueryToObservationLatency(labelValues, latency)
-		p.queryEndTimes.Delete(timestamp)
 	}
 
 	// Report latency for Observation() at end of call.
 	defer func() {
 		duration := float64(time.Now().UTC().Sub(start))
 		p.prometheusBackend.SetObservationDuration(labelValues, duration)
-		p.observationEndTimes.Store(timestamp, time.Now().UTC()) // note time at end of Observe()
+		p.setEndTime(timestamp, p.observationEndTimes) // note time at end of Observe()
 	}()
 
 	return p.wrapped.Observation(ctx, timestamp, query)
@@ -270,17 +281,16 @@ func (p *promPlugin) Report(ctx context.Context, timestamp types.ReportTimestamp
 
 	// Report latency between Observation() and Report().
 	labelValues := getLabelsValues(p, timestamp)
-	if observationEndTime, ok := p.observationEndTimes.Load(timestamp); ok {
+	if observationEndTime, ok := p.observationEndTimes.Get(timestampToKey(timestamp)); ok {
 		latency := float64(start.Sub(observationEndTime.(time.Time)))
 		p.prometheusBackend.SetObservationToReportLatency(labelValues, latency)
-		p.observationEndTimes.Delete(timestamp)
 	}
 
 	// Report latency for Report() at end of call.
 	defer func() {
 		duration := float64(time.Now().UTC().Sub(start))
 		p.prometheusBackend.SetReportDuration(labelValues, duration)
-		p.reportEndTimes.Store(timestamp, time.Now().UTC()) // note time at end of Report()
+		p.setEndTime(timestamp, p.reportEndTimes) // note time at end of Report()
 	}()
 
 	return p.wrapped.Report(ctx, timestamp, query, observations)
@@ -291,17 +301,16 @@ func (p *promPlugin) ShouldAcceptFinalizedReport(ctx context.Context, timestamp
 
 	// Report latency between Report() and ShouldAcceptFinalizedReport().
 	labelValues := getLabelsValues(p, timestamp)
-	if reportEndTime, ok := p.reportEndTimes.Load(timestamp); ok {
+	if reportEndTime, ok := p.reportEndTimes.Get(timestampToKey(timestamp)); ok {
 		latency := float64(start.Sub(reportEndTime.(time.Time)))
 		p.prometheusBackend.SetReportToAcceptFinalizedReportLatency(labelValues, latency)
-		p.reportEndTimes.Delete(timestamp)
 	}
 
 	// Report latency for ShouldAcceptFinalizedReport() at end of call.
 	defer func() {
 		duration := float64(time.Now().UTC().Sub(start))
 		p.prometheusBackend.SetShouldAcceptFinalizedReportDuration(labelValues, duration)
-		p.acceptFinalizedReportEndTimes.Store(timestamp, time.Now().UTC()) // note time at end of ShouldAcceptFinalizedReport()
+		p.setEndTime(timestamp, p.acceptFinalizedReportEndTimes) // note time at end of ShouldAcceptFinalizedReport()
 	}()
 
 	return p.wrapped.ShouldAcceptFinalizedReport(ctx, timestamp, report)
@@ -312,10 +321,9 @@ func (p *promPlugin) ShouldTransmitAcceptedReport(ctx context.Context, timestamp
 
 	// Report latency between ShouldAcceptFinalizedReport() and ShouldTransmitAcceptedReport().
 	labelValues := getLabelsValues(p, timestamp)
-	if acceptFinalizedReportEndTime, ok := p.acceptFinalizedReportEndTimes.Load(timestamp); ok {
+	if acceptFinalizedReportEndTime, ok := p.acceptFinalizedReportEndTimes.Get(timestampToKey(timestamp)); ok {
 		latency := float64(start.Sub(acceptFinalizedReportEndTime.(time.Time)))
 		p.prometheusBackend.SetAcceptFinalizedReportToTransmitAcceptedReportLatency(labelValues, latency)
-		p.acceptFinalizedReportEndTimes.Delete(timestamp)
 	}
 
 	defer func() {
@@ -343,3 +351,11 @@ func (p *promPlugin) Close() error {
 
 	return p.wrapped.Close()
 }
+
+func (p *promPlugin) setEndTime(timestamp types.ReportTimestamp, cache *cache.Cache) {
+	cache.SetDefault(timestampToKey(timestamp), time.Now().UTC())
+}
+
+func timestampToKey(timestamp types.ReportTimestamp) string {
+	return fmt.Sprintf("%x_%d_%d", timestamp.ConfigDigest[:], timestamp.Epoch, timestamp.Round)
+}