Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix race condition in homechain test #5

Merged
merged 6 commits into from
Jun 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions internal/reader/home_chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"time"

mapset "github.com/deckarep/golang-set/v2"

"github.com/smartcontractkit/chainlink-common/pkg/logger"
"github.com/smartcontractkit/chainlink-common/pkg/services"
"github.com/smartcontractkit/chainlink-common/pkg/types"
Expand Down Expand Up @@ -40,6 +41,7 @@ type state struct {
}

type homeChainPoller struct {
wg sync.WaitGroup
stopCh services.StopChan
sync services.StateMachine
homeChainReader types.ContractReader
Expand Down Expand Up @@ -70,21 +72,25 @@ func NewHomeChainConfigPoller(
}

func (r *homeChainPoller) Start(ctx context.Context) error {
err := r.fetchAndSetConfigs(ctx)
if err != nil {
// Just log, don't return error as we want to keep polling
r.lggr.Errorw("Initial fetch of on-chain configs failed", "err", err)
}
r.lggr.Infow("Start Polling ChainConfig")
return r.sync.StartOnce(r.Name(), func() error {
r.wg.Add(1)
go r.poll()
return nil
})
}

func (r *homeChainPoller) poll() {

defer r.wg.Done()
ctx, cancel := r.stopCh.NewCtx()
defer cancel()
// Initial fetch once poll is called before any ticks
if err := r.fetchAndSetConfigs(ctx); err != nil {
// Just log, don't return error as we want to keep polling
r.lggr.Errorw("Initial fetch of on-chain configs failed", "err", err)
}

ticker := time.NewTicker(r.pollingDuration)
defer ticker.Stop()
for {
Expand All @@ -99,7 +105,6 @@ func (r *homeChainPoller) poll() {
r.mutex.Lock()
r.failedPolls++
r.mutex.Unlock()
r.lggr.Errorw("fetching and setting configs failed", "failedPolls", r.failedPolls, "err", err)
}
}
}
Expand All @@ -111,7 +116,6 @@ func (r *homeChainPoller) fetchAndSetConfigs(ctx context.Context) error {
ctx, "CCIPCapabilityConfiguration", "getAllChainConfigs", nil, &chainConfigInfos,
)
if err != nil {
r.lggr.Errorw("fetching on-chain configs failed", "err", err)
return err
}
if len(chainConfigInfos) == 0 {
Expand All @@ -124,7 +128,6 @@ func (r *homeChainPoller) fetchAndSetConfigs(ctx context.Context) error {
r.lggr.Errorw("error converting OnChainConfigs to ChainConfig", "err", err)
return err
}
r.lggr.Infow("Setting ChainConfig")
r.setState(homeChainConfigs)
return nil
}
Expand Down Expand Up @@ -202,6 +205,7 @@ func (r *homeChainPoller) GetOCRConfigs(

func (r *homeChainPoller) Close() error {
return r.sync.StopOnce(r.Name(), func() error {
defer r.wg.Wait()
close(r.stopCh)
return nil
})
Expand Down
55 changes: 27 additions & 28 deletions internal/reader/home_chain_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"time"

mapset "github.com/deckarep/golang-set/v2"

libocrtypes "github.com/smartcontractkit/libocr/ragep2p/types"

"github.com/smartcontractkit/chainlink-ccip/internal/mocks"
Expand All @@ -15,7 +16,6 @@ import (
cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3"
"github.com/smartcontractkit/libocr/commontypes"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -43,31 +43,24 @@ func TestHomeChainConfigPoller_HealthReport(t *testing.T) {
mock.Anything).Return(fmt.Errorf("error"))

var (
tickTime = 10 * time.Millisecond
totalSleepTime = 11 * tickTime
tickTime = 1 * time.Millisecond
totalSleepTime = 20 * tickTime // to allow it to fail 10 times at least
)

configPoller := NewHomeChainConfigPoller(
homeChainReader,
logger.Test(t),
tickTime,
)
_ = configPoller.Start(context.Background())

require.NoError(t, configPoller.Start(context.Background()))
// Initially it's healthy
healthy := configPoller.HealthReport()
assert.Equal(t, map[string]error{configPoller.Name(): error(nil)}, healthy)

require.Equal(t, map[string]error{configPoller.Name(): error(nil)}, healthy)
// After one second it will try polling 10 times and fail
time.Sleep(totalSleepTime)

errors := configPoller.HealthReport()

err := configPoller.Close()
time.Sleep(tickTime)
assert.NoError(t, err)
assert.Equal(t, 1, len(errors))
assert.Errorf(t, errors[configPoller.Name()], "polling failed %d times in a row", MaxFailedPolls)
require.Equal(t, 1, len(errors))
require.Errorf(t, errors[configPoller.Name()], "polling failed %d times in a row", MaxFailedPolls)
require.NoError(t, configPoller.Close())
}

func Test_PollingWorking(t *testing.T) {
Expand Down Expand Up @@ -130,10 +123,11 @@ func Test_PollingWorking(t *testing.T) {
*arg = onChainConfigs
}).Return(nil)

defer homeChainReader.AssertExpectations(t)

var (
tickTime = 20 * time.Millisecond
totalSleepTime = (tickTime * 2) + (10 * time.Millisecond)
expNumCalls = int(totalSleepTime/tickTime) + 1 // +1 for the initial call
tickTime = 2 * time.Millisecond
totalSleepTime = tickTime * 4
)

configPoller := NewHomeChainConfigPoller(
Expand All @@ -142,19 +136,24 @@ func Test_PollingWorking(t *testing.T) {
tickTime,
)

ctx := context.Background()
err := configPoller.Start(ctx)
assert.NoError(t, err)
require.NoError(t, configPoller.Start(context.Background()))
// sleep to allow polling to happen
time.Sleep(totalSleepTime)
err = configPoller.Close()
assert.NoError(t, err)

// called 3 times, once when it's started, and 2 times when it's polling
homeChainReader.AssertNumberOfCalls(t, "GetLatestValue", expNumCalls)
require.NoError(t, configPoller.Close())

calls := homeChainReader.Calls
callCount := 0
for _, call := range calls {
if call.Method == "GetLatestValue" {
callCount++
}
}
//called at least 2 times, one for start and one for the first tick
require.GreaterOrEqual(t, callCount, 2)

configs, err := configPoller.GetAllChainConfigs()
assert.NoError(t, err)
assert.Equal(t, homeChainConfig, configs)
require.NoError(t, err)
require.Equal(t, homeChainConfig, configs)
}

func Test_HomeChainPoller_GetOCRConfig(t *testing.T) {
Expand Down