refactor(zetacore): delete ballots after maturity

changelog.md

@@ -27,6 +27,7 @@
 * [2725](https://github.com/zeta-chain/node/pull/2725) - refactor SetCctxAndNonceToCctxAndInboundHashToCctx to receive tsspubkey as an argument
 * [2802](https://github.com/zeta-chain/node/pull/2802) - set default liquidity cap for new ZRC20s
 * [2826](https://github.com/zeta-chain/node/pull/2826) - remove unused code from emissions module and add new parameter for fixed block reward amount
+* [2863](https://github.com/zeta-chain/node/pull/2863) - delete ballots after they mature
 
 ### Tests
 

proto/zetachain/zetacore/observer/ballot.proto

@@ -41,3 +41,8 @@ message BallotListForHeight {
   int64 height = 1;
   repeated string ballots_index_list = 2;
 }
+
+message VoterList {
+  string voter_address = 1;
+  VoteType vote_type = 2;
+}

proto/zetachain/zetacore/observer/events.proto

@@ -4,6 +4,7 @@ package zetachain.zetacore.observer;
 import "gogoproto/gogo.proto";
 import "zetachain/zetacore/observer/crosschain_flags.proto";
 import "zetachain/zetacore/observer/observer.proto";
+import "zetachain/zetacore/observer/ballot.proto";
 
 option go_package = "github.com/zeta-chain/node/x/observer/types";
 
@@ -15,6 +16,13 @@ message EventBallotCreated {
   string ballot_type = 5;
 }
 
+message EventBallotDeleted {
+  string msg_type_url = 1;
+  string ballot_identifier = 2;
+  string ballot_type = 3;
+  repeated VoterList voters = 4 [ (gogoproto.nullable) = false ];
+}
+
 message EventKeygenBlockUpdated {
   string msg_type_url = 1;
   string keygen_block = 2;

proto/zetachain/zetacore/observer/query.proto

@@ -246,14 +246,10 @@ message QueryHasVotedRequest {
 message QueryHasVotedResponse { bool has_voted = 1; }
 
 message QueryBallotByIdentifierRequest { string ballot_identifier = 1; }
-message VoterList {
-  string voter_address = 1;
-  VoteType vote_type = 2;
-}
 
 message QueryBallotByIdentifierResponse {
   string ballot_identifier = 1;
-  repeated VoterList voters = 2;
+  repeated VoterList voters = 2 [ (gogoproto.nullable) = false ];
   ObservationType observation_type = 3;
   BallotStatus ballot_status = 4;
 }

testutil/testdata/testdata.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/stretchr/testify/require"
+	observertypes "github.com/zeta-chain/node/x/observer/types"
 )
 
 const (
@@ -28,6 +29,28 @@ var ethFiles embed.FS
 //go:embed *
 var testDataFiles embed.FS
 
+//go:embed * observer/*
+var ObserverFiles embed.FS
+
+type Ballots struct {
+	Ballots []*observertypes.Ballot `json:"Ballots"`
+}
+
+// ReadMainnetBallots reads the mainnet ballots from a file. This was last updated on 09-30-2024.
+func ReadMainnetBallots(t *testing.T) []*observertypes.Ballot {
+
+	file, err := ObserverFiles.Open("observer/ballots.json")
+	require.NoError(t, err)
+	defer file.Close()
+
+	// Decode the JSON into the data struct
+	var b Ballots
+	err = json.NewDecoder(file).Decode(&b)
+	require.NoError(t, err)
+
+	return b.Ballots
+}
+
 // ReadEthHeader reads a header from a file.
 // TODO: centralize test data
 // https://github.com/zeta-chain/node/issues/1874

typescript/zetachain/zetacore/observer/ballot_pb.d.ts

@@ -142,3 +142,32 @@ export declare class BallotListForHeight extends Message<BallotListForHeight> {
   static equals(a: BallotListForHeight | PlainMessage<BallotListForHeight> | undefined, b: BallotListForHeight | PlainMessage<BallotListForHeight> | undefined): boolean;
 }
 
+/**
+ * @generated from message zetachain.zetacore.observer.VoterList
+ */
+export declare class VoterList extends Message<VoterList> {
+  /**
+   * @generated from field: string voter_address = 1;
+   */
+  voterAddress: string;
+
+  /**
+   * @generated from field: zetachain.zetacore.observer.VoteType vote_type = 2;
+   */
+  voteType: VoteType;
+
+  constructor(data?: PartialMessage<VoterList>);
+
+  static readonly runtime: typeof proto3;
+  static readonly typeName = "zetachain.zetacore.observer.VoterList";
+  static readonly fields: FieldList;
+
+  static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): VoterList;
+
+  static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): VoterList;
+
+  static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): VoterList;
+
+  static equals(a: VoterList | PlainMessage<VoterList> | undefined, b: VoterList | PlainMessage<VoterList> | undefined): boolean;
+}
+

typescript/zetachain/zetacore/observer/events_pb.d.ts

@@ -5,6 +5,7 @@
 
 import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf";
 import { Message, proto3 } from "@bufbuild/protobuf";
+import type { VoterList } from "./ballot_pb.js";
 import type { GasPriceIncreaseFlags } from "./crosschain_flags_pb.js";
 
 /**
@@ -51,6 +52,45 @@ export declare class EventBallotCreated extends Message<EventBallotCreated> {
   static equals(a: EventBallotCreated | PlainMessage<EventBallotCreated> | undefined, b: EventBallotCreated | PlainMessage<EventBallotCreated> | undefined): boolean;
 }
 
+/**
+ * @generated from message zetachain.zetacore.observer.EventBallotDeleted
+ */
+export declare class EventBallotDeleted extends Message<EventBallotDeleted> {
+  /**
+   * @generated from field: string msg_type_url = 1;
+   */
+  msgTypeUrl: string;
+
+  /**
+   * @generated from field: string ballot_identifier = 2;
+   */
+  ballotIdentifier: string;
+
+  /**
+   * @generated from field: string ballot_type = 3;
+   */
+  ballotType: string;
+
+  /**
+   * @generated from field: repeated zetachain.zetacore.observer.VoterList voters = 4;
+   */
+  voters: VoterList[];
+
+  constructor(data?: PartialMessage<EventBallotDeleted>);
+
+  static readonly runtime: typeof proto3;
+  static readonly typeName = "zetachain.zetacore.observer.EventBallotDeleted";
+  static readonly fields: FieldList;
+
+  static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): EventBallotDeleted;
+
+  static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): EventBallotDeleted;
+
+  static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): EventBallotDeleted;
+
+  static equals(a: EventBallotDeleted | PlainMessage<EventBallotDeleted> | undefined, b: EventBallotDeleted | PlainMessage<EventBallotDeleted> | undefined): boolean;
+}
+
 /**
  * @generated from message zetachain.zetacore.observer.EventKeygenBlockUpdated
  */

typescript/zetachain/zetacore/observer/query_pb.d.ts

@@ -10,7 +10,7 @@ import type { ChainNonces } from "./chain_nonces_pb.js";
 import type { PageRequest, PageResponse } from "../../../cosmos/base/query/v1beta1/pagination_pb.js";
 import type { PendingNonces } from "./pending_nonces_pb.js";
 import type { TSS } from "./tss_pb.js";
-import type { BallotStatus, VoteType } from "./ballot_pb.js";
+import type { BallotStatus, VoterList } from "./ballot_pb.js";
 import type { LastObserverCount, ObservationType } from "./observer_pb.js";
 import type { Chain } from "../pkg/chains/chains_pb.js";
 import type { ChainParams, ChainParamsList } from "./params_pb.js";
@@ -596,35 +596,6 @@ export declare class QueryBallotByIdentifierRequest extends Message<QueryBallotB
   static equals(a: QueryBallotByIdentifierRequest | PlainMessage<QueryBallotByIdentifierRequest> | undefined, b: QueryBallotByIdentifierRequest | PlainMessage<QueryBallotByIdentifierRequest> | undefined): boolean;
 }
 
-/**
- * @generated from message zetachain.zetacore.observer.VoterList
- */
-export declare class VoterList extends Message<VoterList> {
-  /**
-   * @generated from field: string voter_address = 1;
-   */
-  voterAddress: string;
-
-  /**
-   * @generated from field: zetachain.zetacore.observer.VoteType vote_type = 2;
-   */
-  voteType: VoteType;
-
-  constructor(data?: PartialMessage<VoterList>);
-
-  static readonly runtime: typeof proto3;
-  static readonly typeName = "zetachain.zetacore.observer.VoterList";
-  static readonly fields: FieldList;
-
-  static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): VoterList;
-
-  static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): VoterList;
-
-  static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): VoterList;
-
-  static equals(a: VoterList | PlainMessage<VoterList> | undefined, b: VoterList | PlainMessage<VoterList> | undefined): boolean;
-}
-
 /**
  * @generated from message zetachain.zetacore.observer.QueryBallotByIdentifierResponse
  */

x/emissions/abci.go

@@ -10,6 +10,7 @@ import (
 	"github.com/zeta-chain/node/cmd/zetacored/config"
 	"github.com/zeta-chain/node/x/emissions/keeper"
 	"github.com/zeta-chain/node/x/emissions/types"
+	observertypes "github.com/zeta-chain/node/x/observer/types"
 )
 
 func BeginBlocker(ctx sdk.Context, keeper keeper.Keeper) {
@@ -102,11 +103,13 @@ func DistributeObserverRewards(
 	if len(ballotIdentifiers) == 0 {
 		return nil
 	}
+	ballots := make([]observertypes.Ballot, 0, len(ballotIdentifiers))
 	for _, ballotIdentifier := range ballotIdentifiers {
 		ballot, found := keeper.GetObserverKeeper().GetBallot(ctx, ballotIdentifier)
 		if !found {
 			continue
 		}
+		ballots = append(ballots, ballot)
 		totalRewardsUnits += ballot.BuildRewardsDistribution(rewardsDistributer)
 	}
 	rewardPerUnit := sdkmath.ZeroInt()
@@ -161,8 +164,9 @@ func DistributeObserverRewards(
 		}
 	}
 	types.EmitObserverEmissions(ctx, finalDistributionList)
-	// TODO : Delete Ballots after distribution
-	// https://github.com/zeta-chain/node/issues/942
+
+	// Clear the matured ballots
+	keeper.GetObserverKeeper().ClearMaturedBallotsAndBallotList(ctx, ballots, params.BallotMaturityBlocks)
 	return nil
 }
 

x/emissions/abci_test.go

@@ -203,7 +203,8 @@ func TestBeginBlocker(t *testing.T) {
 	})
 
 	t.Run("successfully distribute rewards", func(t *testing.T) {
-		numberOfTestBlocks := 100
+		//Arrange
+		numberOfTestBlocks := 10
 		k, ctx, sk, zk := keepertest.EmissionsKeeper(t)
 		observerSet := sample.ObserverSet(10)
 		zk.ObserverKeeper.SetObserverSet(ctx, observerSet)
@@ -238,6 +239,9 @@ func TestBeginBlocker(t *testing.T) {
 
 		params, found := k.GetParams(ctx)
 		require.True(t, found)
+		// Set the ballot maturity blocks to numberOfTestBlocks so that the ballot mature at the end of the for loop which produces blocks
+		params.BallotMaturityBlocks = int64(numberOfTestBlocks)
+		err = k.SetParams(ctx, params)
 
 		// Get the rewards distribution, this is a fixed amount based on total block rewards and distribution percentages
 		validatorRewardsForABlock, observerRewardsForABlock, tssSignerRewardsForABlock := emissionstypes.GetRewardsDistributions(
@@ -247,6 +251,11 @@ func TestBeginBlocker(t *testing.T) {
 		distributedRewards := observerRewardsForABlock.Add(validatorRewardsForABlock).Add(tssSignerRewardsForABlock)
 		require.True(t, blockRewards.TruncateInt().GT(distributedRewards))
 
+		require.Len(t, zk.ObserverKeeper.GetAllBallots(ctx), len(ballotList))
+		_, found = zk.ObserverKeeper.GetBallotListForHeight(ctx, 0)
+		require.True(t, found)
+
+		// Act
 		for i := 0; i < numberOfTestBlocks; i++ {
 			emissionPoolBeforeBlockDistribution := sk.BankKeeper.GetBalance(ctx, emissionPool, config.BaseDenom).Amount
 			// produce a block
@@ -277,12 +286,23 @@ func TestBeginBlocker(t *testing.T) {
 			ctx = ctx.WithBlockHeight(ctx.BlockHeight() + 1)
 		}
 
+		// Assert
+
+		// 1. Assert Observer rewards, these are distributed at the block in which the ballots mature.
+		// numberOfTestBlocks is the same maturity blocks for the ballots
+
 		// We can simplify the calculation as the rewards are distributed equally among all the observers
 		rewardPerUnit := observerRewardsForABlock.Quo(
 			sdk.NewInt(int64(len(ballotList) * len(observerSet.ObserverList))),
 		)
 		emissionAmount := rewardPerUnit.Mul(sdk.NewInt(int64(len(ballotList))))
 
+		// 2 . Assert ballots and ballot list are deleted on maturity
+		require.Len(t, zk.ObserverKeeper.GetAllBallots(ctx), 0)
+		_, found = zk.ObserverKeeper.GetBallotListForHeight(ctx, 0)
+		require.False(t, found)
+
+		//3. Assert amounts in undistributed pools
 		// Check if the rewards are distributed equally among all the observers
 		for _, observer := range observerSet.ObserverList {
 			observerEmission, found := k.GetWithdrawableEmission(ctx, observer)

x/emissions/types/expected_keepers.go

@@ -16,6 +16,7 @@ type AccountKeeper interface {
 type ObserverKeeper interface {
 	GetBallot(ctx sdk.Context, index string) (val observertypes.Ballot, found bool)
 	GetMaturedBallots(ctx sdk.Context, maturityBlocks int64) (val observertypes.BallotListForHeight, found bool)
+	ClearMaturedBallotsAndBallotList(ctx sdk.Context, ballots []observertypes.Ballot, maturityBlocks int64)
 }
 
 // BankKeeper defines the expected interface needed to retrieve account balances.