From c138da1c9bf291ec60b8655edda85bf2efb6134e Mon Sep 17 00:00:00 2001 From: krish-z <122767080+krish-nr@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:09:57 +0800 Subject: [PATCH] feat: auto recover from pbss geth unclean shutdown (#214) Co-authored-by: Owen <103096885+owen-reorg@users.noreply.github.com> --- op-node/rollup/derive/engine_controller.go | 80 ++++++++++++++++++++++ op-node/rollup/derive/engine_queue.go | 5 ++ op-service/eth/types.go | 2 + 3 files changed, 87 insertions(+) diff --git a/op-node/rollup/derive/engine_controller.go b/op-node/rollup/derive/engine_controller.go index f73ce21891..1a7c4601b2 100644 --- a/op-node/rollup/derive/engine_controller.go +++ b/op-node/rollup/derive/engine_controller.go @@ -271,6 +271,9 @@ func (e *EngineController) resetBuildingState() { // It returns true if the status is acceptable. func (e *EngineController) checkNewPayloadStatus(status eth.ExecutePayloadStatus) bool { if e.syncMode == sync.ELSync { + if status == eth.ExecutionInconsistent { + return true + } if status == eth.ExecutionValid && e.syncStatus == syncStatusStartedEL { e.syncStatus = syncStatusFinishedELButNotFinalized } @@ -348,6 +351,41 @@ func (e *EngineController) InsertUnsafePayload(ctx context.Context, envelope *et if err != nil { return NewTemporaryError(fmt.Errorf("failed to update insert payload: %w", err)) } + + //process inconsistent state + if status.Status == eth.ExecutionInconsistent { + currentL2Info, err := e.getCurrentL2Info(ctx) + if err != nil { + return NewTemporaryError(fmt.Errorf("failed to process inconsistent state: %w", err)) + } else { + log.Info("engine has inconsistent state", "unsafe", currentL2Info.Unsafe.Number, "safe", currentL2Info.Safe.Number, "final", currentL2Info.Finalized.Number) + e.SetUnsafeHead(currentL2Info.Unsafe) + if currentL2Info.Safe.Number > currentL2Info.Unsafe.Number { + log.Info("current safe is higher than unsafe block, reset it", "set safe after", currentL2Info.Unsafe.Number, "set safe before", e.safeHead.Number) + e.SetSafeHead(currentL2Info.Unsafe) + } + if currentL2Info.Finalized.Number > currentL2Info.Unsafe.Number { + log.Info("current finalized is higher than unsafe block, reset it", "set Finalized after", currentL2Info.Unsafe.Number, "set Finalized before", e.safeHead.Number) + e.SetFinalizedHead(currentL2Info.Unsafe) + } + } + + fcuReq := eth.ForkchoiceState{ + HeadBlockHash: e.unsafeHead.Hash, + SafeBlockHash: e.safeHead.Hash, + FinalizedBlockHash: e.finalizedHead.Hash, + } + + fcuRes, err := e.engine.ForkchoiceUpdate(ctx, &fcuReq, nil) + if fcuRes.PayloadStatus.Status == eth.ExecutionValid { + log.Info("engine processed data successfully") + e.needFCUCall = false + return nil + } else { + return NewTemporaryError(fmt.Errorf("engine failed to process inconsistent data: %w", err)) + } + } + if !e.checkNewPayloadStatus(status.Status) { payload := envelope.ExecutionPayload return NewTemporaryError(fmt.Errorf("cannot process unsafe payload: new - %v; parent: %v; err: %w", @@ -360,6 +398,22 @@ func (e *EngineController) InsertUnsafePayload(ctx context.Context, envelope *et SafeBlockHash: e.safeHead.Hash, FinalizedBlockHash: e.finalizedHead.Hash, } + + //update unsafe,safe,finalize and send fcu for sync + if status.Status == eth.ExecutionInconsistent { + log.Info("engine meet inconsistent here") + currentUnsafe, _ := e.engine.L2BlockRefByLabel(ctx, eth.Unsafe) + //reset unsafe + e.SetUnsafeHead(currentUnsafe) + //force reset safe,finalize + e.SetSafeHead(currentUnsafe) + e.SetFinalizedHead(currentUnsafe) + + fc.HeadBlockHash = currentUnsafe.Hash + fc.SafeBlockHash = currentUnsafe.Hash + fc.FinalizedBlockHash = currentUnsafe.Hash + } + if e.syncStatus == syncStatusFinishedELButNotFinalized { fc.SafeBlockHash = envelope.ExecutionPayload.BlockHash fc.FinalizedBlockHash = envelope.ExecutionPayload.BlockHash @@ -468,3 +522,29 @@ func (e *EngineController) TryBackupUnsafeReorg(ctx context.Context) (bool, erro func (e *EngineController) ResetBuildingState() { e.resetBuildingState() } + +// getCurrentL2Info returns the current finalized, safe and unsafe heads of the execution engine. +func (e *EngineController) getCurrentL2Info(ctx context.Context) (*sync.FindHeadsResult, error) { + finalized, err := e.engine.L2BlockRefByLabel(ctx, eth.Finalized) + if err != nil { + log.Error("err get finalized", "err", err) + return nil, fmt.Errorf("failed to find the finalized L2 block: %w", err) + } + + safe, err := e.engine.L2BlockRefByLabel(ctx, eth.Safe) + if errors.Is(err, ethereum.NotFound) { + safe = finalized + } else if err != nil { + return nil, fmt.Errorf("failed to find the safe L2 block: %w", err) + } + + unsafe, err := e.engine.L2BlockRefByLabel(ctx, eth.Unsafe) + if err != nil { + return nil, fmt.Errorf("failed to find the L2 head block: %w", err) + } + return &sync.FindHeadsResult{ + Unsafe: unsafe, + Safe: safe, + Finalized: finalized, + }, nil +} diff --git a/op-node/rollup/derive/engine_queue.go b/op-node/rollup/derive/engine_queue.go index dee9f46d0b..a394ab148a 100644 --- a/op-node/rollup/derive/engine_queue.go +++ b/op-node/rollup/derive/engine_queue.go @@ -695,6 +695,11 @@ func (eq *EngineQueue) Reset(ctx context.Context, _ eth.L1BlockRef, _ eth.System return NewTemporaryError(fmt.Errorf("failed to find the L2 Heads to start from: %w", err)) } finalized, safe, unsafe := result.Finalized, result.Safe, result.Unsafe + + if finalized.Number > safe.Number { + finalized = safe + } + l1Origin, err := eq.l1Fetcher.L1BlockRefByHash(ctx, safe.L1Origin.Hash) if err != nil { return NewTemporaryError(fmt.Errorf("failed to fetch the new L1 progress: origin: %v; err: %w", safe.L1Origin, err)) diff --git a/op-service/eth/types.go b/op-service/eth/types.go index 58d16c3cdd..18ae2cf3af 100644 --- a/op-service/eth/types.go +++ b/op-service/eth/types.go @@ -338,6 +338,8 @@ const ( ExecutionInvalidBlockHash ExecutePayloadStatus = "INVALID_BLOCK_HASH" // proof-of-stake transition only, not used in rollup ExecutionInvalidTerminalBlock ExecutePayloadStatus = "INVALID_TERMINAL_BLOCK" + // given payload is invalid + ExecutionInconsistent ExecutePayloadStatus = "INCONSISTENT" ) type PayloadStatusV1 struct {