diff --git a/.env b/.env new file mode 100644 index 00000000..bd9c31dc --- /dev/null +++ b/.env @@ -0,0 +1,10 @@ +STATE_ANALYZER_CMD="rewards" +STATE_ANALYZER_LOG_LEVEL="debug" +STATE_ANALYZER_BN_ENDPOINT="http://beacon_node_ip:api-port" +STATE_ANALYZER_OUTFOLDER="results" +STATE_ANALYZER_INIT_SLOT="10000" +STATE_ANALYZER_FINAL_SLOT="10050" +STATE_ANALYZER_VALIDATOR_INDEXES="test_validators.json" +STATE_ANALYZER_DB_URL="postgresql://db_user:db_password@db_ip:db_port/db_name" +STATE_ANALYZER_WORKERS_NUM="50" +STATE_ANALYZER_DB_WORKERS_NUM="4" \ No newline at end of file diff --git a/.gitignore b/.gitignore index fbca2253..36680fb6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ results/ +eth2-state-analyzer +.vscode/** +test*.json +.ipynb_checkpoints/** +metrika/** +.config \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..206038a4 --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +#!make + +GOCC=go +MKDIR_P=mkdir -p + +BIN_PATH=./build +BIN="./build/eth2-state-analyzer" + +include .env + +.PHONY: check build install run clean + +build: + $(GOCC) build -o $(BIN) + +install: + $(GOCC) install + +run: + $(BIN) $(STATE_ANALYZER_CMD) \ + --log-level=${STATE_ANALYZER_LOG_LEVEL} \ + --bn-endpoint=${STATE_ANALYZER_BN_ENDPOINT} \ + --outfolder=${STATE_ANALYZER_OUTFOLDER} \ + --init-slot=${STATE_ANALYZER_INIT_SLOT} \ + --final-slot=${STATE_ANALYZER_FINAL_SLOT} \ + --validator-indexes=${STATE_ANALYZER_VALIDATOR_INDEXES} \ + --db-url=${STATE_ANALYZER_DB_URL} \ + --workers-num=${STATE_ANALYZER_WORKERS_NUM} \ + --db-workers-num=${STATE_ANALYZER_DB_WORKERS_NUM} + +clean: + rm -r $(BIN_PATH) + diff --git a/README.md b/README.md index aee8da8a..74e4bba3 100644 --- a/README.md +++ b/README.md @@ -1,2 +1 @@ -# api-benchmark -Little client that measures API's response times +# Eth2 State Analyzer diff --git a/cmd/reward_cmd.go b/cmd/reward_cmd.go index da91d4a1..c25f48cb 100644 --- a/cmd/reward_cmd.go +++ b/cmd/reward_cmd.go @@ -1,6 +1,9 @@ package cmd import ( + "os" + "os/signal" + "syscall" "time" "github.com/pkg/errors" @@ -40,6 +43,18 @@ var RewardsCommand = &cli.Command{ &cli.StringFlag{ Name: "log-level", Usage: "log level: debug, warn, info, error", + }, + &cli.StringFlag{ + Name: "db-url", + Usage: "example: postgresql://beaconchain:beaconchain@localhost:5432/beacon_states_kiln", + }, + &cli.IntFlag{ + Name: "workers-num", + Usage: "example: 50", + }, + &cli.IntFlag{ + Name: "db-workers-num", + Usage: "example: 50", }}, } @@ -47,10 +62,12 @@ var logRewardsRewards = logrus.WithField( "module", "RewardsCommand", ) -var QueryTimeout = 30 * time.Second +var QueryTimeout = 90 * time.Second // CrawlAction is the function that is called when running `eth2`. func LaunchRewardsCalculator(c *cli.Context) error { + coworkers := 1 + dbWorkers := 1 logRewardsRewards.Info("parsing flags") // check if a config file is set if !c.IsSet("bn-endpoint") { @@ -71,10 +88,23 @@ func LaunchRewardsCalculator(c *cli.Context) error { if c.IsSet("log-level") { logrus.SetLevel(utils.ParseLogLevel(c.String("log-level"))) } + if !c.IsSet("db-url") { + return errors.New("db-url not provided") + } + if !c.IsSet("workers-num") { + logRewardsRewards.Infof("workers-num flag not provided, default: 1") + } else { + coworkers = c.Int("workers-num") + } + if !c.IsSet("db-workers-num") { + logRewardsRewards.Infof("db-workers-num flag not provided, default: 1") + } else { + dbWorkers = c.Int("db-workers-num") + } bnEndpoint := c.String("bn-endpoint") - outputFile := c.String("outfolder") initSlot := uint64(c.Int("init-slot")) finalSlot := uint64(c.Int("final-slot")) + dbUrl := c.String("db-url") validatorIndexes, err := utils.GetValIndexesFromJson(c.String("validator-indexes")) if err != nil { @@ -86,16 +116,33 @@ func LaunchRewardsCalculator(c *cli.Context) error { if err != nil { return err } + // generate the state analyzer - stateAnalyzer, err := analyzer.NewStateAnalyzer(c.Context, cli, initSlot, finalSlot, validatorIndexes) + stateAnalyzer, err := analyzer.NewStateAnalyzer(c.Context, cli, initSlot, finalSlot, validatorIndexes, dbUrl, coworkers, dbWorkers) if err != nil { return err } - stateAnalyzer.Run() + procDoneC := make(chan struct{}) + sigtermC := make(chan os.Signal) + + signal.Notify(sigtermC, syscall.SIGINT, syscall.SIGTERM, os.Interrupt, syscall.SIGTERM) - // TODO: Compose Results - stateAnalyzer.ExportToCsv(outputFile) + go func() { + stateAnalyzer.Run() + procDoneC <- struct{}{} + }() + + select { + case <-sigtermC: + logRewardsRewards.Info("Sudden shutdown detected, controlled shutdown of the cli triggered") + stateAnalyzer.Close() + + case <-procDoneC: + logRewardsRewards.Info("Process successfully finish!") + } + close(sigtermC) + close(procDoneC) return nil } diff --git a/go.mod b/go.mod index ba1fc951..5fe9cf31 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.17 require ( github.com/attestantio/go-eth2-client v0.11.3 + github.com/jackc/pgx/v4 v4.16.1 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.26.1 github.com/sirupsen/logrus v1.6.0 @@ -15,8 +16,17 @@ require ( github.com/fatih/color v1.13.0 // indirect github.com/ferranbt/fastssz v0.0.0-20220103083642-bc5fefefa28b // indirect github.com/goccy/go-yaml v1.9.5 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.12.1 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.11.0 // indirect + github.com/jackc/puddle v1.2.1 // indirect github.com/klauspost/cpuid/v2 v2.0.11 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/minio/sha256-simd v1.0.0 // indirect @@ -24,8 +34,10 @@ require ( github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect github.com/r3labs/sse/v2 v2.7.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e // indirect golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 79dcb8e4..2eaaef4e 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -51,9 +52,13 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -82,6 +87,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0= github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -139,6 +145,55 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8= +github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= +github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= +github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y= +github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -153,18 +208,30 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.11 h1:i2lw1Pm7Yi/4O6XCSyJWqEHI2MDw2FzUK6o/D21xn2A= github.com/klauspost/cpuid/v2 v2.0.11/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -210,20 +277,29 @@ github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h github.com/r3labs/sse/v2 v2.7.4 h1:pvCMswPDlXd/ZUFx1dry0LbXJNHXwWPulLcUGYwClc0= github.com/r3labs/sse/v2 v2.7.4/go.mod h1:hUrYMKfu9WquG9MyI0r6TKiNH+6Sw/QPKm2YbNbU5g8= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/urfave/cli/v2 v2.4.4 h1:IvwT3XfI6RytTmIzC35UAu9oyK+bHgUPXDDZNqribkI= github.com/urfave/cli/v2 v2.4.4/go.mod h1:oDzoM7pVwz6wHn5ogWgFUU1s4VJayeQS+aEZDqXIEJs= @@ -231,17 +307,35 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -288,6 +382,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -328,7 +423,9 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -336,7 +433,9 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -368,6 +467,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -375,7 +475,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -387,14 +489,18 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -402,6 +508,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -421,6 +528,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -508,6 +617,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/launcher.sh b/launcher.sh index d1b890cd..171e31c8 100755 --- a/launcher.sh +++ b/launcher.sh @@ -5,11 +5,10 @@ CLI_NAME="state-analyzer" echo "launching State-Analyzer" -#BN_ENDPOINT="https://20PdJoS82pnejJJ9joDMnbjsQ32:0c9b868d8621332ea91c7fc24c5fc34f@eth2-beacon-mainnet.infura.io" BN_ENDPOINT="http://localhost:5052" OUT_FOLDER="results" -INIT_SLOT="25" -FINAL_SLOT="63" +INIT_SLOT="300000" +FINAL_SLOT="300063" VALIDATOR_LIST_FILE="test_validators.json" go get diff --git a/main.go b/main.go index bb303a67..579846fb 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,7 @@ var ( ) func main() { - fmt.Println(CliName, Version, "\n") + fmt.Println(CliName, Version) //ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/analyzer/analyzer.go b/pkg/analyzer/analyzer.go index a19c7fb6..1be7a324 100644 --- a/pkg/analyzer/analyzer.go +++ b/pkg/analyzer/analyzer.go @@ -2,19 +2,16 @@ package analyzer import ( "context" - "fmt" - "os" "sync" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" - api "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/phase0" - "github.com/cortze/eth2-state-analyzer/pkg/clientapi" + "github.com/cortze/eth2-state-analyzer/pkg/db/postgresql" + "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics" + "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics/fork_state" "github.com/cortze/eth2-state-analyzer/pkg/utils" ) @@ -23,290 +20,190 @@ var ( log = logrus.WithField( "module", modName, ) - maxWorkers = 10 - minReqTime = 10 * time.Second + maxWorkers = 50 + minReqTime = 10 * time.Second + MAX_VAL_BATCH_SIZE = 20000 + VAL_LEN = 400000 + SLOT_SECONDS = 12 + EPOCH_SLOTS = 32 ) type StateAnalyzer struct { - ctx context.Context - InitSlot uint64 - FinalSlot uint64 - ValidatorIndexes []uint64 - // map of [validatorIndexes]RewardMetrics - Metrics sync.Map - SlotRanges []uint64 - // - ValidatorTaskChan chan *ValidatorTask - // http CLient - cli *clientapi.APIClient - - // + ctx context.Context + cancel context.CancelFunc + InitSlot uint64 + FinalSlot uint64 + ValidatorIndexes []uint64 + SlotRanges []uint64 + MonitorSlotProcess map[uint64]uint64 + EpochTaskChan chan *EpochTask + ValTaskChan chan *ValTask + validatorWorkerNum int + + cli *clientapi.APIClient + dbClient *postgresql.PostgresDBService + + // Control Variables + finishDownload bool + routineClosed chan struct{} + initTime time.Time } -func NewStateAnalyzer(ctx context.Context, httpCli *clientapi.APIClient, initSlot uint64, finalSlot uint64, valIdxs []uint64) (*StateAnalyzer, error) { +func NewStateAnalyzer( + pCtx context.Context, + httpCli *clientapi.APIClient, + initSlot uint64, + finalSlot uint64, + valIdxs []uint64, + idbUrl string, + workerNum int, + dbWorkerNum int) (*StateAnalyzer, error) { log.Infof("generating new State Analzyer from slots %d:%d, for validators %v", initSlot, finalSlot, valIdxs) + // gen new ctx from parent + ctx, cancel := context.WithCancel(pCtx) + // Check if the range of slots is valid if !utils.IsValidRangeuint64(initSlot, finalSlot) { return nil, errors.New("provided slot range isn't valid") } + valLength := len(valIdxs) // check if valIdx where given if len(valIdxs) < 1 { - return nil, errors.New("no validator indexes where provided") + log.Infof("No validator indexes provided: running all validators") + valLength = VAL_LEN // estimation to declare channels } // calculate the list of slots that we will analyze slotRanges := make([]uint64, 0) epochRange := uint64(0) - for i := initSlot; i < (finalSlot + utils.SlotBase); i += utils.SlotBase { + + // minimum slot is 31 + // force to be in the previous epoch than select by user + initEpoch := uint64(initSlot) / 32 + finalEpoch := uint64(finalSlot / 32) + + initSlot = (initEpoch+1)*fork_state.SLOTS_PER_EPOCH - 1 // take last slot of init Epoch + finalSlot = (finalEpoch+1)*fork_state.SLOTS_PER_EPOCH - 1 // take last slot of final Epoch + + // start two epochs before and end two epochs after + for i := initSlot - (fork_state.SLOTS_PER_EPOCH * 2); i <= (finalSlot + fork_state.SLOTS_PER_EPOCH*2); i += utils.SlotBase { slotRanges = append(slotRanges, i) epochRange++ } log.Debug("slotRanges are:", slotRanges) - var metrics sync.Map - // Compose the metrics array with each of the RewardMetrics - for _, val := range valIdxs { - mets, err := NewRewardMetrics(initSlot, epochRange, val) - if err != nil { - return nil, errors.Wrap(err, "unable to generate RewarMetrics.") - } - metrics.Store(val, mets) + i_dbClient, err := postgresql.ConnectToDB(ctx, idbUrl, valLength*maxWorkers, dbWorkerNum) + if err != nil { + return nil, errors.Wrap(err, "unable to generate DB Client.") } return &StateAnalyzer{ - ctx: ctx, - InitSlot: initSlot, - FinalSlot: finalSlot, - ValidatorIndexes: valIdxs, - SlotRanges: slotRanges, - Metrics: metrics, - ValidatorTaskChan: make(chan *ValidatorTask, len(valIdxs)), - cli: httpCli, + ctx: ctx, + cancel: cancel, + InitSlot: initSlot, + FinalSlot: finalSlot, + ValidatorIndexes: valIdxs, + SlotRanges: slotRanges, + EpochTaskChan: make(chan *EpochTask, 10), + ValTaskChan: make(chan *ValTask, valLength*maxWorkers), + MonitorSlotProcess: make(map[uint64]uint64), + cli: httpCli, + dbClient: i_dbClient, + validatorWorkerNum: workerNum, + routineClosed: make(chan struct{}), }, nil } func (s *StateAnalyzer) Run() { + defer s.cancel() + // Get init time + s.initTime = time.Now() + + log.Info("State Analyzer initialized at ", s.initTime) + // State requester - var wg sync.WaitGroup + var wgDownload sync.WaitGroup + downloadFinishedFlag := false + + // Rewards per process + var wgProcess sync.WaitGroup + processFinishedFlag := false + + // Workers to process each validator rewards + var wgWorkers sync.WaitGroup + + totalTime := int64(0) + start := time.Now() - wg.Add(1) // State requester + Task generator - go func() { - defer wg.Done() - log.Info("Launching Beacon State Requester") - // loop over the list of slots that we need to analyze - for _, slot := range s.SlotRanges { - ticker := time.NewTicker(minReqTime) - select { - case <-s.ctx.Done(): - log.Info("context has died, closing state requester routine") - close(s.ValidatorTaskChan) - return - - default: - // make the state query - log.Debug("requesting Beacon State from endpoint") - bstate, err := s.cli.Api.BeaconState(s.ctx, fmt.Sprintf("%d", slot)) - if err != nil { - // close the channel (to tell other routines to stop processing and end) - log.Errorf("Unable to retrieve Beacon State from the beacon node, closing requester routine. %s", err.Error()) - close(s.ValidatorTaskChan) - return - } - - log.Debug("requesting Validator list from endpoint") - validatorFilter := make([]phase0.ValidatorIndex, 0) - activeValidators, err := s.cli.Api.Validators(s.ctx, fmt.Sprintf("%d", slot), validatorFilter) - if err != nil { - // close the channel (to tell other routines to stop processing and end) - log.Errorf("Unable to retrieve Validators from the beacon node, closing requester routine. %s", err.Error()) - close(s.ValidatorTaskChan) - return - } - - var totalActiveBalance uint64 = 0 - var totalEffectiveBalance uint64 = 0 - - for _, val := range activeValidators { - // only count active validators - if !val.Status.IsActive() { - continue - } - // since it's active - totalActiveBalance += uint64(val.Balance) - totalEffectiveBalance += uint64(val.Validator.EffectiveBalance) - - } - - // Once the state has been downloaded, loop over the validator - for _, val := range s.ValidatorIndexes { - // compose the next task - valTask := &ValidatorTask{ - ValIdx: val, - Slot: slot, - State: bstate, - TotalValidatorStatus: &activeValidators, - TotalEffectiveBalance: totalEffectiveBalance, - TotalActiveBalance: totalActiveBalance, - } - - log.Debugf("sending task for slot %d and validator %d", slot, val) - s.ValidatorTaskChan <- valTask - } - } - // check if the min Request time has been completed (to avoid spaming the API) - <-ticker.C - } - log.Infof("All states for the slot ranges has been successfully retrieved, clossing go routine") - close(s.ValidatorTaskChan) - }() - - // generate workers, validator tasks consumers - coworkers := len(s.ValidatorIndexes) - if coworkers > maxWorkers { - coworkers = maxWorkers - } - for i := 0; i < coworkers; i++ { + wgDownload.Add(1) + go s.runDownloadStates(&wgDownload) + + // State requester in finalized slots, not used for now + // wgDownload.Add(1) + // go s.runDownloadStatesFinalized(&wgDownload) + + wgProcess.Add(1) + go s.runProcessState(&wgProcess, &downloadFinishedFlag) + + for i := 0; i < s.validatorWorkerNum; i++ { // state workers, receiving State and valIdx to measure performance wlog := logrus.WithField( "worker", i, ) - wlog.Info("Launching Task Worker") - wg.Add(1) - go func() { - defer wg.Done() - - // keep iterrating until the channel is closed due to finishing - for { - // cehck if the channel has been closed - task, ok := <-s.ValidatorTaskChan - if !ok { - wlog.Warn("the task channel has been closed, finishing worker routine") - return - } - wlog.Debugf("task received for slot %d and val %d", task.Slot, task.ValIdx) - // Proccess State - wlog.Debug("analyzing the receved state") - - // TODO: Analyze rewards for the given Validator - - // check if there is a metrics already - metInterface, ok := s.Metrics.Load(task.ValIdx) - if !ok { - log.Errorf("Validator %d not found in list of tracked validators", task.ValIdx) - } - // met is already the pointer to the metrics, we don't need to store it again - met := metInterface.(*RewardMetrics) - log.Debug("Calculating the performance of the validator") - err := met.CalculateEpochPerformance(task.State, task.TotalValidatorStatus, task.TotalEffectiveBalance) - if err != nil { - log.Errorf("unable to calculate the performance for validator %d on slot %d. %s", task.ValIdx, task.Slot, err.Error()) - } - // save the calculated rewards on the the list of items - fmt.Println(met) - s.Metrics.Store(task.ValIdx, met) - } - - }() + wlog.Tracef("Launching Task Worker") + wgWorkers.Add(1) + go s.runWorker(wlog, &wgWorkers, &processFinishedFlag) } - // Get init time - s.initTime = time.Now() + wgDownload.Wait() + downloadFinishedFlag = true + log.Info("Beacon State Downloads finished") - log.Info("State Analyzer initialized at", s.initTime) - wg.Wait() + wgProcess.Wait() + processFinishedFlag = true + log.Info("Beacon State Processing finished") - analysisDuration := time.Since(s.initTime) + wgWorkers.Wait() + log.Info("All validator workers finished") + s.dbClient.DoneTasks() + <-s.dbClient.FinishSignalChan + + close(s.ValTaskChan) + totalTime += int64(time.Since(start).Seconds()) + analysisDuration := time.Since(s.initTime).Seconds() log.Info("State Analyzer finished in ", analysisDuration) + if s.finishDownload { + s.routineClosed <- struct{}{} + } +} +func (s *StateAnalyzer) Close() { + log.Info("Sudden closed detected, closing StateAnalyzer") + s.finishDownload = true + <-s.routineClosed + s.cancel() } // -type ValidatorTask struct { - ValIdx uint64 - Slot uint64 - State *spec.VersionedBeaconState - TotalValidatorStatus *map[phase0.ValidatorIndex]*api.Validator - TotalEffectiveBalance uint64 - TotalActiveBalance uint64 +type EpochTask struct { + ValIdxs []uint64 + NextState fork_state.ForkStateContentBase + State fork_state.ForkStateContentBase + PrevState fork_state.ForkStateContentBase + OnlyPrevAtt bool } -// Exporter Functions - -func (s *StateAnalyzer) ExportToCsv(outputFolder string) error { - // check if the folder exists - csvRewardsFile, err := os.Create(outputFolder + "/validator_rewards.csv") - if err != nil { - return err - } - csvMaxRewardFile, err := os.Create(outputFolder + "/validator_max_rewards.csv") - if err != nil { - return err - } - csvPercentageFile, err := os.Create(outputFolder + "/validator_rewards_percentage.csv") - if err != nil { - return err - } - // write headers on the csvs - headers := "slot,total" - for _, val := range s.ValidatorIndexes { - headers += "," + fmt.Sprintf("%d", val) - } - csvRewardsFile.WriteString(headers + "\n") - csvMaxRewardFile.WriteString(headers + "\n") - csvPercentageFile.WriteString(headers + "\n") - - for _, slot := range s.SlotRanges { - rowRewards := fmt.Sprintf("%d", slot) - rowMaxRewards := fmt.Sprintf("%d", slot) - rowRewardsPerc := fmt.Sprintf("%d", slot) - - auxRowRewards := "" - auxRowMaxRewards := "" - auxRowRewardsPerc := "" - - var totRewards uint64 - var totMaxRewards uint64 - var totPerc float64 - - // iter through the validator results - for _, val := range s.ValidatorIndexes { - - m, ok := s.Metrics.Load(val) - if !ok { - log.Errorf("validator %d has no metrics", val) - } - met := m.(*RewardMetrics) - valMetrics, err := met.GetEpochMetrics(slot) - if err != nil { - return err - } - totRewards += valMetrics.Reward - totMaxRewards += valMetrics.MaxReward - totPerc += valMetrics.RewardPercentage - - auxRowRewards += "," + fmt.Sprintf("%d", valMetrics.Reward) - auxRowMaxRewards += "," + fmt.Sprintf("%d", valMetrics.MaxReward) - auxRowRewardsPerc += "," + fmt.Sprintf("%.3f", valMetrics.RewardPercentage) - - } - - rowRewards += fmt.Sprintf(",%d", totRewards) + auxRowRewards - rowMaxRewards += fmt.Sprintf(",%d", totMaxRewards) + auxRowMaxRewards - rowRewardsPerc += fmt.Sprintf(",%.3f", totPerc/float64(len(s.ValidatorIndexes))) + auxRowRewardsPerc - - // end up with the line - csvRewardsFile.WriteString(rowRewards + "\n") - csvMaxRewardFile.WriteString(rowMaxRewards + "\n") - csvPercentageFile.WriteString(rowRewardsPerc + "\n") - } - - csvRewardsFile.Close() - csvMaxRewardFile.Close() - csvPercentageFile.Close() +type ValTask struct { + ValIdxs []uint64 + StateMetricsObj fork_metrics.StateMetrics + OnlyPrevAtt bool +} - return nil +type MonitorTasks struct { + ValIdxs []uint64 + Slot uint64 } diff --git a/pkg/analyzer/download_state.go b/pkg/analyzer/download_state.go new file mode 100644 index 00000000..87dfefab --- /dev/null +++ b/pkg/analyzer/download_state.go @@ -0,0 +1,191 @@ +package analyzer + +import ( + "fmt" + "sync" + "time" + + "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics/fork_state" +) + +func (s *StateAnalyzer) runDownloadStates(wgDownload *sync.WaitGroup) { + defer wgDownload.Done() + log.Info("Launching Beacon State Requester") + // loop over the list of slots that we need to analyze + + // We need three consecutive states to compute max rewards easier + prevBState := fork_state.ForkStateContentBase{} + bstate := fork_state.ForkStateContentBase{} + nextBstate := fork_state.ForkStateContentBase{} + ticker := time.NewTicker(minReqTime) + for _, slot := range s.SlotRanges { + + select { + case <-s.ctx.Done(): + log.Info("context has died, closing state requester routine") + close(s.EpochTaskChan) + return + + default: + if s.finishDownload { + log.Info("sudden shutdown detected, state downloader routine") + close(s.EpochTaskChan) + return + } + ticker.Reset(minReqTime) + firstIteration := true + secondIteration := true + // make the state query + log.Infof("requesting Beacon State from endpoint: slot %d", slot) + + // We need three states to calculate both, rewards and maxRewards + + if bstate.AttestingBalance != nil { // in case we already had a bstate (only false the first time) + prevBState = bstate + firstIteration = false + } + if nextBstate.AttestingBalance != nil { // in case we already had a nextBstate (only false the first and second time) + bstate = nextBstate + secondIteration = false + } + newState, err := s.cli.Api.BeaconState(s.ctx, fmt.Sprintf("%d", slot)) + if newState == nil { + log.Errorf("Unable to retrieve Beacon State from the beacon node, closing requester routine. Nil State") + return + } + if err != nil { + // close the channel (to tell other routines to stop processing and end) + log.Errorf("Unable to retrieve Beacon State from the beacon node, closing requester routine. %s", err.Error()) + return + } + if firstIteration { + bstate, err = fork_state.GetCustomState(*newState, s.cli.Api) + if err != nil { + // close the channel (to tell other routines to stop processing and end) + log.Errorf("Unable to retrieve Beacon State from the beacon node, closing requester routine. %s", err.Error()) + return + } + } else { + nextBstate, err = fork_state.GetCustomState(*newState, s.cli.Api) + if err != nil { + // close the channel (to tell other routines to stop processing and end) + log.Errorf("Unable to retrieve Beacon State from the beacon node, closing requester routine. %s", err.Error()) + return + } + } + + if !firstIteration && !secondIteration { + // only execute tasks if it is not the first or second iteration iteration ==> we have three states + + epochTask := &EpochTask{ + ValIdxs: s.ValidatorIndexes, + NextState: nextBstate, + State: bstate, + PrevState: prevBState, + } + + log.Debugf("sending task for slot: %d", epochTask.State.Slot) + s.EpochTaskChan <- epochTask + } + // check if the min Request time has been completed (to avoid spaming the API) + <-ticker.C + + } + + } + + log.Infof("All states for the slot ranges has been successfully retrieved, clossing go routine") +} + +func (s *StateAnalyzer) runDownloadStatesFinalized(wgDownload *sync.WaitGroup) { + defer wgDownload.Done() + log.Info("Launching Beacon State Finalized Requester") + // loop over the list of slots that we need to analyze + prevBState := fork_state.ForkStateContentBase{} + bstate := fork_state.ForkStateContentBase{} + nextBstate := fork_state.ForkStateContentBase{} + finalizedSlot := 0 + timerCh := time.NewTicker(time.Second * 384) // epoch seconds = 384 + ticker := time.NewTicker(minReqTime) + for { + + select { + case <-s.ctx.Done(): + log.Info("context has died, closing state requester routine") + close(s.EpochTaskChan) + return + + case <-timerCh.C: + ticker.Reset(minReqTime) + firstIteration := true + secondIteration := true + // make the state query + log.Infof("requesting Beacon State from endpoint: finalized") + if bstate.AttestingBalance != nil { // in case we already had a bstate (only false the first time) + prevBState = bstate + firstIteration = false + } + if nextBstate.AttestingBalance != nil { // in case we already had a nextBstate (only false the first time) + bstate = nextBstate + secondIteration = false + } + header, err := s.cli.Api.BeaconBlockHeader(s.ctx, "finalized") + if err != nil { + log.Errorf("Unable to retrieve Beacon State from the beacon node, closing finalized requester routine. %s", err.Error()) + return + } + if int(header.Header.Message.Slot) == finalizedSlot { + log.Infof("No new finalized state yet") + continue + } + + finalizedSlot = int(header.Header.Message.Slot) - 1 + log.Infof("New finalized state at slot: %d", finalizedSlot) + newState, err := s.cli.Api.BeaconState(s.ctx, fmt.Sprintf("%d", finalizedSlot)) + if newState == nil { + log.Errorf("Unable to retrieve Finalized Beacon State from the beacon node, closing requester routine. Nil State") + return + } + if err != nil { + // close the channel (to tell other routines to stop processing and end) + log.Errorf("Unable to retrieve Finalized Beacon State from the beacon node, closing requester routine. %s", err.Error()) + return + } + if firstIteration { + + bstate, err = fork_state.GetCustomState(*newState, s.cli.Api) + if err != nil { + // close the channel (to tell other routines to stop processing and end) + log.Errorf("Unable to retrieve Beacon State from the beacon node, closing requester routine. %s", err.Error()) + return + } + } else { + nextBstate, err = fork_state.GetCustomState(*newState, s.cli.Api) + if err != nil { + // close the channel (to tell other routines to stop processing and end) + log.Errorf("Unable to retrieve Beacon State from the beacon node, closing requester routine. %s", err.Error()) + return + } + } + if !firstIteration && !secondIteration { + // only execute tasks if it is not the first iteration or second iteration + + // we now only compose one single task that contains a list of validator indexes + epochTask := &EpochTask{ + ValIdxs: s.ValidatorIndexes, + NextState: nextBstate, + State: bstate, + PrevState: prevBState, + } + + log.Debugf("sending task for slot: %d", epochTask.State.Slot) + s.EpochTaskChan <- epochTask + } + <-ticker.C + // check if the min Request time has been completed (to avoid spaming the API) + default: + + } + + } +} diff --git a/pkg/analyzer/process_state.go b/pkg/analyzer/process_state.go new file mode 100644 index 00000000..5d749014 --- /dev/null +++ b/pkg/analyzer/process_state.go @@ -0,0 +1,149 @@ +package analyzer + +import ( + "math" + "sync" + + "github.com/attestantio/go-eth2-client/spec/altair" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/cortze/eth2-state-analyzer/pkg/db/postgresql" + "github.com/cortze/eth2-state-analyzer/pkg/db/postgresql/model" + "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics" + "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics/fork_state" + "github.com/jackc/pgx/v4" +) + +func (s *StateAnalyzer) runProcessState(wgProcess *sync.WaitGroup, downloadFinishedFlag *bool) { + defer wgProcess.Done() + + epochBatch := pgx.Batch{} + log.Info("Launching Beacon State Pre-Processer") +loop: + for { + // in case the downloads have finished, and there are no more tasks to execute + if *downloadFinishedFlag && len(s.EpochTaskChan) == 0 { + log.Warn("the task channel has been closed, finishing epoch routine") + if epochBatch.Len() == 0 { + log.Debugf("Sending last epoch batch to be stored...") + s.dbClient.WriteChan <- epochBatch + epochBatch = pgx.Batch{} + } + + break loop + } + + select { + case <-s.ctx.Done(): + log.Info("context has died, closing state processer routine") + return + + case task, ok := <-s.EpochTaskChan: + + // check if the channel has been closed + if !ok { + log.Warn("the task channel has been closed, finishing epoch routine") + return + } + log.Infof("epoch task received for slot %d, analyzing...", task.State.Slot) + + // returns the state in a custom struct for Phase0, Altair of Bellatrix + stateMetrics, err := fork_metrics.StateMetricsByForkVersion(task.NextState, task.State, task.PrevState, s.cli.Api) + + if err != nil { + log.Errorf(err.Error()) + continue + } + + log.Debugf("Creating validator batches for slot %d...", task.State.Slot) + + if len(task.ValIdxs) == 0 { + // in case no validators provided, do all the active ones in the next epoch, take into account proposer and sync committee rewards + task.ValIdxs = stateMetrics.GetMetricsBase().NextState.GetAllVals() + } else { + finalValidxs := make([]uint64, 0) + for _, item := range task.ValIdxs { + // check that validator number does not exceed the number of validators + if int(item) >= len(stateMetrics.GetMetricsBase().NextState.Validators) { + continue + } + // in case no validators provided, do all the active ones in the next epoch, take into account proposer and sync committee rewards + if fork_state.IsActive(*stateMetrics.GetMetricsBase().NextState.Validators[item], phase0.Epoch(stateMetrics.GetMetricsBase().PrevState.Epoch)) { + finalValidxs = append(finalValidxs, item) + } + } + task.ValIdxs = finalValidxs + } + if task.NextState.Slot <= s.FinalSlot { + + stepSize := int(math.Min(float64(MAX_VAL_BATCH_SIZE), float64(len(task.ValIdxs)/s.validatorWorkerNum))) + stepSize = int(math.Max(float64(1), float64(stepSize))) // in case it is 0, at least set to 1 + for i := 0; i < len(task.ValIdxs); i += stepSize { + endIndex := int(math.Min(float64(len(task.ValIdxs)), float64(i+stepSize))) + // subslice does not include the endIndex + valTask := &ValTask{ + ValIdxs: task.ValIdxs[i:endIndex], + StateMetricsObj: stateMetrics, + } + s.ValTaskChan <- valTask + } + } + if task.PrevState.Slot >= s.InitSlot { // only write epoch metrics inside the defined range + + log.Debugf("Writing epoch metrics to DB for slot %d...", task.State.Slot) + // create a model to be inserted into the db, we only insert previous epoch metrics + + missedBlocks := stateMetrics.GetMetricsBase().PrevState.MissedBlocks + // take into accoutn epoch transition + nextMissedBlock := stateMetrics.GetMetricsBase().CurrentState.TrackPrevMissingBlock() + if nextMissedBlock != 0 { + missedBlocks = append(missedBlocks, nextMissedBlock) + } + epochDBRow := model.NewEpochMetrics( + stateMetrics.GetMetricsBase().PrevState.Epoch, + stateMetrics.GetMetricsBase().PrevState.Slot, + uint64(len(stateMetrics.GetMetricsBase().CurrentState.PrevAttestations)), + uint64(stateMetrics.GetMetricsBase().CurrentState.NumAttestingVals), + uint64(stateMetrics.GetMetricsBase().PrevState.NumActiveVals), + uint64(stateMetrics.GetMetricsBase().PrevState.TotalActiveRealBalance), + uint64(stateMetrics.GetMetricsBase().CurrentState.AttestingBalance[altair.TimelyTargetFlagIndex]), // as per BEaconcha.in + uint64(stateMetrics.GetMetricsBase().PrevState.TotalActiveBalance), + uint64(stateMetrics.GetMetricsBase().CurrentState.GetMissingFlagCount(int(altair.TimelySourceFlagIndex))), + uint64(stateMetrics.GetMetricsBase().CurrentState.GetMissingFlagCount(int(altair.TimelyTargetFlagIndex))), + uint64(stateMetrics.GetMetricsBase().CurrentState.GetMissingFlagCount(int(altair.TimelyHeadFlagIndex))), + missedBlocks) + + epochBatch.Queue(model.UpsertEpoch, + epochDBRow.Epoch, + epochDBRow.Slot, + epochDBRow.PrevNumAttestations, + epochDBRow.PrevNumAttValidators, + epochDBRow.PrevNumValidators, + epochDBRow.TotalBalance, + epochDBRow.AttEffectiveBalance, + epochDBRow.TotalEffectiveBalance, + epochDBRow.MissingSource, + epochDBRow.MissingTarget, + epochDBRow.MissingHead, + epochDBRow.MissedBlocks) + + // Proposer Duties + + for _, item := range stateMetrics.GetMetricsBase().PrevState.EpochStructs.ProposerDuties { + newDuty := model.NewProposerDuties(uint64(item.ValidatorIndex), uint64(item.Slot)) + epochBatch.Queue(model.InsertProposerDuty, + newDuty.ValIdx, + newDuty.ProposerSlot) + } + } + + // Flush the database batches + if epochBatch.Len() >= postgresql.MAX_EPOCH_BATCH_QUEUE { + s.dbClient.WriteChan <- epochBatch + epochBatch = pgx.Batch{} + } + default: + } + + } + log.Infof("Pre process routine finished...") +} diff --git a/pkg/analyzer/reward_metrics.go b/pkg/analyzer/reward_metrics.go deleted file mode 100644 index eb3a7d70..00000000 --- a/pkg/analyzer/reward_metrics.go +++ /dev/null @@ -1,152 +0,0 @@ -package analyzer - -import ( - "errors" - "sync" - - api "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/phase0" - - "github.com/cortze/eth2-state-analyzer/pkg/utils" -) - -type RewardMetrics struct { - m sync.Mutex - baseslot uint64 - innerCnt uint64 - validatorIdx uint64 - ValidatorBalances []uint64 // Gwei ?¿ - MaxRewards []uint64 // Gwei ?¿ - Rewards []uint64 // Gweis ?¿ - RewardPercentage []float64 // % - - MissingSources []uint64 - MissingHeads []uint64 - MissingTargets []uint64 -} - -func NewRewardMetrics(initslot uint64, epochRange uint64, validatorIdx uint64) (*RewardMetrics, error) { - log.Debugf("len of the epoch range %d", epochRange) - return &RewardMetrics{ - baseslot: initslot, - validatorIdx: validatorIdx, - ValidatorBalances: make([]uint64, epochRange), - MaxRewards: make([]uint64, epochRange), - Rewards: make([]uint64, epochRange), - RewardPercentage: make([]float64, epochRange), - MissingSources: make([]uint64, epochRange), - MissingHeads: make([]uint64, epochRange), - MissingTargets: make([]uint64, epochRange), - }, nil -} - -// Supposed to be -func (m *RewardMetrics) CalculateEpochPerformance(bState *spec.VersionedBeaconState, validators *map[phase0.ValidatorIndex]*api.Validator, totalActiveBalance uint64) error { - - validatorBalance, err := GetValidatorBalance(bState, m.validatorIdx) - if err != nil { - return err - } - // add always the balance to the array - log.Debugf("starting performance calculation of validator %d", m.validatorIdx) - m.m.Lock() - defer func() { - log.Debugf("finishing the performance calculation of validator %d", m.validatorIdx) - m.m.Unlock() - }() - m.ValidatorBalances[m.innerCnt] = uint64(validatorBalance) - - if m.innerCnt != 0 { - // calculate Reward from the previous Balance - reward := validatorBalance - m.ValidatorBalances[m.innerCnt-1] - log.Debugf("reward for validator %d = %d", m.validatorIdx, reward) - - // Add Reward - m.Rewards[m.innerCnt] = reward - - // Proccess Max-Rewards - maxReward, err := GetMaxReward(m.validatorIdx, validators, totalActiveBalance) - if err != nil { - return err - } - log.Debugf("max reward for validator %d = %d", m.validatorIdx, maxReward) - m.MaxRewards[m.innerCnt] = maxReward - - // Proccess Reward-Performance-Ratio - rewardPerf := (float64(reward) * 100) / float64(maxReward) - m.RewardPercentage[m.innerCnt] = rewardPerf - log.Debugf("reward performance for %d = %f%", m.validatorIdx, rewardPerf) - - // TODO: - // Add number of missing sources - - // Add number of missing head - - // Add number of missing targets - - } - // do not forget to increment the internal counter - m.innerCnt++ - log.Debugf("done with validator %d", m.validatorIdx) - - return nil -} - -func (m *RewardMetrics) GetEpochMetrics(slot uint64) (SingleEpochMetrics, error) { - var epochMetrics SingleEpochMetrics - - // calculate the index - idx := m.GetIndexFromslot(slot) - if idx < 0 { - log.Errorf("requested metrics for slot: %d couldn't be found. Max slot is %d", slot, m.baseslot+(32*uint64(len(m.Rewards)))) - return epochMetrics, errors.New("requested slot can not be found on the analyzed data") - } - - m.m.Lock() - // if the index is okey, compose the Singe epoch metrics - epochMetrics.ValidatorIdx = m.validatorIdx - epochMetrics.Slot = m.validatorIdx - epochMetrics.Epoch = utils.GetEpochFromSlot(slot) - epochMetrics.ValidatorBalance = m.ValidatorBalances[idx] - epochMetrics.MaxReward = m.MaxRewards[idx] - epochMetrics.Reward = m.Rewards[idx] - epochMetrics.RewardPercentage = m.RewardPercentage[idx] - epochMetrics.MissingSource = m.MissingSources[idx] - epochMetrics.MissingHead = m.MissingHeads[idx] - epochMetrics.MissingTarget = m.MissingTargets[idx] - epochMetrics.ValidatorIdx = m.validatorIdx - m.m.Unlock() - return epochMetrics, nil -} - -func (m *RewardMetrics) GetIndexFromslot(slot uint64) int { - idx := -1 - m.m.Lock() - defer m.m.Unlock() - idx = int(slot/m.baseslot) - 1 - if idx >= len(m.Rewards) { - return -1 - } - - return idx -} - -func (m *RewardMetrics) AddslotPerformance() error { - - return nil -} - -type SingleEpochMetrics struct { - ValidatorIdx uint64 - Slot uint64 - Epoch uint64 - ValidatorBalance uint64 // Gwei ?¿ - MaxReward uint64 // Gwei ?¿ - Reward uint64 // Gweis ?¿ - RewardPercentage float64 // % - - MissingSource uint64 - MissingHead uint64 - MissingTarget uint64 -} diff --git a/pkg/analyzer/rewards.go b/pkg/analyzer/rewards.go deleted file mode 100644 index 900f822a..00000000 --- a/pkg/analyzer/rewards.go +++ /dev/null @@ -1,80 +0,0 @@ -package analyzer - -import ( - "fmt" - "math" - - "github.com/pkg/errors" - - api "github.com/attestantio/go-eth2-client/api/v1" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/phase0" -) - -const ( - participationRate = 0.945 - baseRewardFactor = 64 - baseRewardsPerEpoch = 4 -) - -func GetValidatorBalance(bstate *spec.VersionedBeaconState, valIdx uint64) (uint64, error) { - var balance uint64 - var err error - switch bstate.Version { - case spec.DataVersionPhase0: - if uint64(len(bstate.Phase0.Balances)) < valIdx { - err = fmt.Errorf("phase0 - validator index %d wasn't activated in slot %d", valIdx, bstate.Phase0.Slot) - } - balance = bstate.Phase0.Balances[valIdx] - - case spec.DataVersionAltair: - if uint64(len(bstate.Altair.Balances)) < valIdx { - err = fmt.Errorf("altair - validator index %d wasn't activated in slot %d", valIdx, bstate.Phase0.Slot) - } - balance = bstate.Altair.Balances[valIdx] - - case spec.DataVersionBellatrix: - if uint64(len(bstate.Bellatrix.Balances)) < valIdx { - err = fmt.Errorf("bellatrix - validator index %d wasn't activated in slot %d", valIdx, bstate.Phase0.Slot) - } - balance = bstate.Bellatrix.Balances[valIdx] - default: - - } - return balance, err -} - -// https://kb.beaconcha.in/rewards-and-penalties -// https://consensys.net/blog/codefi/rewards-and-penalties-on-ethereum-20-phase-0/ -// TODO: -would be nice to incorporate top the max value wheather there were 2-3 consecutive missed blocks afterwards -func GetMaxReward(valIdx uint64, totValStatus *map[phase0.ValidatorIndex]*api.Validator, totalActiveBalance uint64) (uint64, error) { - // First iteration just taking 31/8*BaseReward as Max value - // BaseReward = ( effectiveBalance * (BaseRewardFactor)/(BaseRewardsPerEpoch * sqrt(activeBalance)) ) - - idx := phase0.ValidatorIndex(valIdx) - - valStatus, ok := (*totValStatus)[idx] - if !ok { - return 0, errors.New("") - } - // apply formula - //baseReward := GetBaseReward(valStatus.Validator.EffectiveBalance, totalActiveBalance) - maxReward := ((31.0 / 8.0) * participationRate * (float64(uint64(valStatus.Validator.EffectiveBalance)) * baseRewardFactor)) - maxReward = maxReward / (baseRewardsPerEpoch * math.Sqrt(float64(totalActiveBalance))) - return uint64(maxReward), nil -} - -// directly calculated on the MaxReward fucntion -func GetBaseReward(valEffectiveBalance phase0.Gwei, totalActiveBalance uint64) uint64 { - // BaseReward = ( effectiveBalance * (BaseRewardFactor)/(BaseRewardsPerEpoch * sqrt(activeBalance)) ) - var baseReward uint64 - - sqrt := math.Sqrt(float64(totalActiveBalance)) - - denom := baseRewardsPerEpoch * sqrt - - bsRewrd := (float64(uint64(valEffectiveBalance)) * baseRewardFactor) / denom - - baseReward = uint64(bsRewrd) - return baseReward -} diff --git a/pkg/analyzer/rewards_test.go b/pkg/analyzer/rewards_test.go new file mode 100644 index 00000000..ccc9564b --- /dev/null +++ b/pkg/analyzer/rewards_test.go @@ -0,0 +1,118 @@ +package analyzer + +import ( + "testing" +) + +func TestParticipationRatePhase0(t *testing.T) { + + // ctx := context.Background() + // log = logrus.WithField( + // "cli", "CliName", + // ) + // bnEndpoint := "https://20PdJoS82pnejJJ9joDMnbjsQ32:0c9b868d8621332ea91c7fc24c5fc34f@eth2-beacon-mainnet.infura.io" + // queryEndpoint := time.Duration(time.Second * 90) + // cli, err := clientapi.NewAPIClient(ctx, bnEndpoint, queryEndpoint) + // if err != nil { + // fmt.Println("could not create API client", err) + // } + + // for i := 95; i < 200; i = i + 32 { + // bstate, err := cli.Api.BeaconState(ctx, fmt.Sprintf("%d", i)) + + // phase0State := custom_spec.NewPhase0Spec(bstate) + + // if err != nil { + // fmt.Println("could not parse State Fork Version", err) + // } + + // valsVotes := phase0State.PreviousEpochAttestations() // max one vote per validator + // doubleVotes := phase0State.GetDoubleVotes() + + // previousEpochAggregations := phase0State.PreviousEpochAggregations() + + // totalVotesPreviousEpoch := 0 + // for _, aggregation := range previousEpochAggregations { + // totalVotesPreviousEpoch += int(aggregation.AggregationBits.Count()) + // } + + // fmt.Println(valsVotes, doubleVotes, totalVotesPreviousEpoch) + + // assert.Equal(t, totalVotesPreviousEpoch, int(valsVotes+doubleVotes)) + // } + +} + +func TestParticipationRateAltair(t *testing.T) { + + // ctx := context.Background() + // log = logrus.WithField( + // "cli", "CliName", + // ) + + // bnEndpoint := "http://localhost:5052" + // queryEndpoint := time.Duration(time.Second * 90) + // cli, err := clientapi.NewAPIClient(ctx, bnEndpoint, queryEndpoint) + // if err != nil { + // fmt.Println("could not create API client", err) + // } + + // for i := 2000; i < 2033; i = i + 32 { + // bstate, err := cli.Api.BeaconState(ctx, fmt.Sprintf("%d", i)) + + // altairState := custom_spec.NewAltairSpec(bstate) + + // if err != nil { + // fmt.Println("could not parse State Fork Version", err) + // } + + // valsVotes := altairState.PreviousEpochAttestations() // max one vote per validator + // missedVotes := altairState.PreviousEpochMissedAttestations() + // totalVals := altairState.PreviousEpochValNum() + + // fmt.Println(valsVotes, missedVotes, totalVals) + + // assert.Equal(t, totalVals, uint64(valsVotes+missedVotes)) + // } + +} + +func TestParticipationRateBellatrix(t *testing.T) { + + // ctx := context.Background() + // log = logrus.WithField( + // "cli", "CliName", + // ) + + // bnEndpoint := "http://localhost:5052" + // queryEndpoint := time.Duration(time.Second * 2000) + // cli, err := clientapi.NewAPIClient(ctx, bnEndpoint, queryEndpoint) + // if err != nil { + // fmt.Println("could not create API client", err) + // } + + // for i := 30000; i < 30032; i = i + 32 { + // bstate, err := cli.Api.BeaconState(ctx, fmt.Sprintf("%d", i)) + + // if err != nil { + // fmt.Println(err) + // t.Fail() + // return + // } + + // bellatrixState := custom_spec.NewBellatrixSpec(bstate) + + // if err != nil { + // fmt.Println("could not parse State Fork Version", err) + // } + + // valsVotes := bellatrixState.PreviousEpochAttestations() // max one vote per validator + // missedVotes := bellatrixState.PreviousEpochMissedAttestations() + // totalVals := bellatrixState.PreviousEpochValNum() + + // fmt.Println(valsVotes, missedVotes, totalVals) + + // assert.Equal(t, totalVals, uint64(valsVotes+missedVotes)) + // } + +} diff --git a/pkg/analyzer/validatorWorker.go b/pkg/analyzer/validatorWorker.go new file mode 100644 index 00000000..76fc8eb7 --- /dev/null +++ b/pkg/analyzer/validatorWorker.go @@ -0,0 +1,128 @@ +package analyzer + +import ( + "sync" + "time" + + "github.com/attestantio/go-eth2-client/spec/altair" + "github.com/cortze/eth2-state-analyzer/pkg/db/postgresql" + "github.com/cortze/eth2-state-analyzer/pkg/db/postgresql/model" + "github.com/jackc/pgx/v4" + "github.com/sirupsen/logrus" +) + +func (s *StateAnalyzer) runWorker(wlog *logrus.Entry, wgWorkers *sync.WaitGroup, processFinishedFlag *bool) { + defer wgWorkers.Done() + batch := pgx.Batch{} + // keep iterating until the channel is closed due to finishing +loop: + for { + + if *processFinishedFlag && len(s.ValTaskChan) == 0 { + wlog.Warn("the task channel has been closed, finishing worker routine") + if batch.Len() > 0 { + + wlog.Debugf("Sending last validator batch to be stored...") + s.dbClient.WriteChan <- batch + batch = pgx.Batch{} + + } + break loop + } + + select { + case valTask, ok := <-s.ValTaskChan: + // check if the channel has been closed + if !ok { + wlog.Warn("the task channel has been closed, finishing worker routine") + return + } + + stateMetrics := valTask.StateMetricsObj + wlog.Debugf("task received for val %d - %d in slot %d", valTask.ValIdxs[0], valTask.ValIdxs[len(valTask.ValIdxs)-1], stateMetrics.GetMetricsBase().CurrentState.Slot) + // Proccess State + snapshot := time.Now() + for _, valIdx := range valTask.ValIdxs { + + // get max reward at given epoch using the formulas + maxRewards, err := stateMetrics.GetMaxReward(valIdx) + + if err != nil { + log.Errorf("Error obtaining max reward: ", err.Error()) + continue + } + + // calculate the current balance of validator + balance := stateMetrics.GetMetricsBase().NextState.Balances[valIdx] + + if err != nil { + log.Errorf("Error obtaining validator balance: ", err.Error()) + continue + } + + // keep in mind that att rewards for epoch 10 can be seen at beginning of epoch 12, + // after state_transition + // https://notes.ethereum.org/@vbuterin/Sys3GLJbD#Epoch-processing + + flags := stateMetrics.GetMetricsBase().CurrentState.MissingFlags(valIdx) + + // create a model to be inserted into the db in the next epoch + validatorDBRow := model.NewValidatorRewards( + valIdx, + stateMetrics.GetMetricsBase().NextState.Slot, + stateMetrics.GetMetricsBase().NextState.Epoch, + balance, + stateMetrics.GetMetricsBase().EpochReward(valIdx), // reward is written after state transition + maxRewards.MaxReward, + maxRewards.Attestation, + maxRewards.InclusionDelay, + maxRewards.FlagIndex, + maxRewards.SyncCommittee, + stateMetrics.GetMetricsBase().GetAttSlot(valIdx), + stateMetrics.GetMetricsBase().GetAttInclusionSlot(valIdx), + maxRewards.BaseReward, + maxRewards.InSyncCommittee, + maxRewards.ProposerSlot, // TODO: there can be several proposer slots, deprecate + flags[altair.TimelySourceFlagIndex], + flags[altair.TimelyTargetFlagIndex], + flags[altair.TimelyHeadFlagIndex], + stateMetrics.GetMetricsBase().NextState.GetValStatus(valIdx)) + + batch.Queue(model.UpsertValidator, + validatorDBRow.ValidatorIndex, + validatorDBRow.Slot, + validatorDBRow.Epoch, + validatorDBRow.ValidatorBalance, + validatorDBRow.Reward, + validatorDBRow.MaxReward, + validatorDBRow.FlagIndexReward, + validatorDBRow.SyncCommitteeReward, + validatorDBRow.AttSlot, + validatorDBRow.InclusionDelay, + validatorDBRow.BaseReward, + validatorDBRow.InSyncCommittee, + validatorDBRow.ProposerSlot, + validatorDBRow.MissingSource, + validatorDBRow.MissingTarget, + validatorDBRow.MissingHead, + validatorDBRow.Status) + + if batch.Len() > postgresql.MAX_BATCH_QUEUE { + wlog.Debugf("Sending batch to be stored...") + s.dbClient.WriteChan <- batch + batch = pgx.Batch{} + } + + } + + wlog.Debugf("Validator group processed, worker freed for next group. Took %f seconds", time.Since(snapshot).Seconds()) + + case <-s.ctx.Done(): + log.Info("context has died, closing state worker routine") + return + default: + } + + } + wlog.Infof("Validator worker finished, no more tasks to process") +} diff --git a/pkg/db/postgresql/epoch_metrics.go b/pkg/db/postgresql/epoch_metrics.go new file mode 100644 index 00000000..c188e67d --- /dev/null +++ b/pkg/db/postgresql/epoch_metrics.go @@ -0,0 +1,25 @@ +package postgresql + +/* + +This file together with the model, has all the needed methods to interact with the epoch_metrics table of the database + +*/ + +import ( + "context" + + "github.com/cortze/eth2-state-analyzer/pkg/db/postgresql/model" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" +) + +// in case the table did not exist +func (p *PostgresDBService) createEpochMetricsTable(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + _, err := pool.Exec(ctx, model.CreateEpochMetricsTable) + if err != nil { + return errors.Wrap(err, "error creating epoch metrics table") + } + return nil +} diff --git a/pkg/db/postgresql/model/epoch_metrics.go b/pkg/db/postgresql/model/epoch_metrics.go new file mode 100644 index 00000000..f77c049f --- /dev/null +++ b/pkg/db/postgresql/model/epoch_metrics.go @@ -0,0 +1,111 @@ +package model + +import ( + "fmt" + + "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics/fork_state" +) + +// Postgres intregration variables +var ( + CreateEpochMetricsTable = ` + CREATE TABLE IF NOT EXISTS t_epoch_metrics_summary( + f_epoch INT, + f_slot INT, + f_num_att INT, + f_num_att_vals INT, + f_num_vals INT, + f_total_balance_eth REAL, + f_att_effective_balance_eth REAL, + f_total_effective_balance_eth REAL, + f_missing_source INT, + f_missing_target INT, + f_missing_head INT, + f_missed_blocks TEXT, + CONSTRAINT PK_Epoch PRIMARY KEY (f_slot));` + + UpsertEpoch = ` + INSERT INTO t_epoch_metrics_summary ( + f_epoch, + f_slot, + f_num_att, + f_num_att_vals, + f_num_vals, + f_total_balance_eth, + f_att_effective_balance_eth, + f_total_effective_balance_eth, + f_missing_source, + f_missing_target, + f_missing_head, + f_missed_blocks) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + ON CONFLICT ON CONSTRAINT PK_Epoch + DO + UPDATE SET + f_num_att = excluded.f_num_att, + f_num_att_vals = excluded.f_num_att_vals, + f_num_vals = excluded.f_num_vals, + f_total_balance_eth = excluded.f_total_balance_eth, + f_att_effective_balance_eth = excluded.f_att_effective_balance_eth, + f_total_effective_balance_eth = excluded.f_total_effective_balance_eth, + f_missing_source = excluded.f_missing_source, + f_missing_target = excluded.f_missing_target, + f_missing_head = excluded.f_missing_head, + f_missed_blocks = excluded.f_missed_blocks; + ` +) + +type EpochMetrics struct { + Epoch uint64 + Slot uint64 + PrevNumAttestations int + PrevNumAttValidators int + PrevNumValidators int + TotalBalance float32 + AttEffectiveBalance float32 + TotalEffectiveBalance float32 + + MissingSource int + MissingTarget int + MissingHead int + + MissedBlocks string +} + +func NewEpochMetrics(iEpoch uint64, + iSlot uint64, + iNumAtt uint64, + iNumAttVals uint64, + iNumVals uint64, + iTotBal uint64, + iAttEfBal uint64, + iTotEfBal uint64, + iMissingSource uint64, + iMissingTarget uint64, + iMissingHead uint64, + iMissedBlocks []uint64) EpochMetrics { + + missedBlocks := "[" + for _, item := range iMissedBlocks { + missedBlocks += fmt.Sprintf("%d", item) + "," + } + missedBlocks += "]" + return EpochMetrics{ + Epoch: iEpoch, + Slot: iSlot, + PrevNumAttestations: int(iNumAtt), + PrevNumAttValidators: int(iNumAttVals), + PrevNumValidators: int(iNumVals), + TotalBalance: float32(iTotBal) / float32(fork_state.EFFECTIVE_BALANCE_INCREMENT), + AttEffectiveBalance: float32(iAttEfBal) / float32(fork_state.EFFECTIVE_BALANCE_INCREMENT), + TotalEffectiveBalance: float32(iTotEfBal) / float32(fork_state.EFFECTIVE_BALANCE_INCREMENT), + MissingSource: int(iMissingSource), + MissingTarget: int(iMissingTarget), + MissingHead: int(iMissingHead), + MissedBlocks: missedBlocks, + } +} + +func NewEmptyEpochMetrics() EpochMetrics { + return EpochMetrics{} +} diff --git a/pkg/db/postgresql/model/proposer_duties.go b/pkg/db/postgresql/model/proposer_duties.go new file mode 100644 index 00000000..9584bff3 --- /dev/null +++ b/pkg/db/postgresql/model/proposer_duties.go @@ -0,0 +1,38 @@ +package model + +// Postgres intregration variables +var ( + CreateProposerDutiesTable = ` + CREATE TABLE IF NOT EXISTS t_proposer_duties( + f_val_idx INT, + f_proposer_slot INT, + CONSTRAINT PK_Val_Slot PRIMARY KEY (f_val_idx, f_proposer_slot));` + + InsertProposerDuty = ` + INSERT INTO t_proposer_duties ( + f_val_idx, + f_proposer_slot) + VALUES ($1, $2) + ON CONFLICT DO NOTHING; + ` + // if there is a confilct the line already exists +) + +type ProposerDuties struct { + ValIdx uint64 + ProposerSlot uint64 +} + +func NewProposerDuties( + iValIdx uint64, + iProposerSlot uint64) ProposerDuties { + + return ProposerDuties{ + ValIdx: iValIdx, + ProposerSlot: iProposerSlot, + } +} + +func NewEmptyProposerDuties() ProposerDuties { + return ProposerDuties{} +} diff --git a/pkg/db/postgresql/model/validator_rewards.go b/pkg/db/postgresql/model/validator_rewards.go new file mode 100644 index 00000000..ccc0ee64 --- /dev/null +++ b/pkg/db/postgresql/model/validator_rewards.go @@ -0,0 +1,136 @@ +package model + +import "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics/fork_state" + +// Postgres intregration variables +var ( + CreateValidatorRewardsTable = ` + CREATE TABLE IF NOT EXISTS t_validator_rewards_summary( + f_val_idx INT, + f_slot INT, + f_epoch INT, + f_balance_eth REAL, + f_reward INT, + f_max_reward INT, + f_max_att_reward INT, + f_max_sync_reward INT, + f_att_slot INT, + f_att_inclusion_slot INT, + f_base_reward INT, + f_in_sync_committee BOOL, + f_proposer_slot INT, + f_missing_source BOOL, + f_missing_target BOOL, + f_missing_head BOOL, + f_status TEXT, + CONSTRAINT PK_ValidatorSlot PRIMARY KEY (f_val_idx,f_slot));` + + UpsertValidator = ` + INSERT INTO t_validator_rewards_summary ( + f_val_idx, + f_slot, + f_epoch, + f_balance_eth, + f_reward, + f_max_reward, + f_max_att_reward, + f_max_sync_reward, + f_att_slot, + f_att_inclusion_slot, + f_base_reward, + f_in_sync_committee, + f_proposer_slot, + f_missing_source, + f_missing_target, + f_missing_head, + f_status) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) + ON CONFLICT ON CONSTRAINT PK_ValidatorSlot + DO + UPDATE SET + f_epoch = excluded.f_epoch, + f_balance_eth = excluded.f_balance_eth, + f_reward = excluded.f_reward, + f_max_reward = excluded.f_max_reward, + f_max_att_reward = excluded.f_max_att_reward, + f_max_sync_reward = excluded.f_max_sync_reward, + f_att_slot = excluded.f_att_slot, + f_att_inclusion_slot = excluded.f_att_inclusion_slot, + f_base_reward = excluded.f_base_reward, + f_in_sync_committee = excluded.f_in_sync_committee, + f_proposer_slot = excluded.f_proposer_slot, + f_missing_source = excluded.f_missing_source, + f_missing_target = excluded.f_missing_target, + f_missing_head = excluded.f_missing_head, + f_status = excluded.f_status; + ` +) + +type ValidatorRewards struct { + ValidatorIndex uint64 + Slot int + Epoch int + ValidatorBalance float32 + Reward int64 + MaxReward uint64 + AttestationReward uint64 + InclusionDelayReward uint64 + FlagIndexReward uint64 + SyncCommitteeReward uint64 + BaseReward uint64 + AttSlot uint64 + InclusionDelay int64 + InSyncCommittee bool + ProposerSlot int64 + MissingSource bool + MissingTarget bool + MissingHead bool + Status string +} + +func NewValidatorRewards( + iValIdx uint64, + iSlot uint64, + iEpoch uint64, + iValBal uint64, + iReward int64, + iMaxReward uint64, + iMaxAttReward uint64, + iMaxInDelayReward uint64, + iMaxFlagReward uint64, + iMaxSyncComReward uint64, + iAttSlot uint64, + iInclusionDelay int64, + iBaseReward uint64, + iSyncCommittee bool, + iProposerSlot int64, + iMissingSource bool, + iMissingTarget bool, + iMissingHead bool, + iStatus string) ValidatorRewards { + return ValidatorRewards{ + ValidatorIndex: iValIdx, + Slot: int(iSlot), + Epoch: int(iEpoch), + ValidatorBalance: float32(iValBal) / float32(fork_state.EFFECTIVE_BALANCE_INCREMENT), + Reward: iReward, + MaxReward: iMaxReward, + AttestationReward: iMaxAttReward, + InclusionDelayReward: iMaxInDelayReward, + FlagIndexReward: iMaxFlagReward, + SyncCommitteeReward: iMaxSyncComReward, + AttSlot: iAttSlot, + InclusionDelay: iInclusionDelay, + BaseReward: iBaseReward, + InSyncCommittee: iSyncCommittee, + ProposerSlot: iProposerSlot, + MissingSource: iMissingSource, + MissingTarget: iMissingTarget, + MissingHead: iMissingHead, + Status: iStatus, + } +} + +func NewEmptyValidatorRewards() ValidatorRewards { + return ValidatorRewards{} +} diff --git a/pkg/db/postgresql/proposer_duties.go b/pkg/db/postgresql/proposer_duties.go new file mode 100644 index 00000000..1a88f8fa --- /dev/null +++ b/pkg/db/postgresql/proposer_duties.go @@ -0,0 +1,25 @@ +package postgresql + +/* + +This file together with the model, has all the needed methods to interact with the epoch_metrics table of the database + +*/ + +import ( + "context" + + "github.com/cortze/eth2-state-analyzer/pkg/db/postgresql/model" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" +) + +// in case the table did not exist +func (p *PostgresDBService) createProposerDutiesTable(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + _, err := pool.Exec(ctx, model.CreateProposerDutiesTable) + if err != nil { + return errors.Wrap(err, "error creating proposer duties table") + } + return nil +} diff --git a/pkg/db/postgresql/service.go b/pkg/db/postgresql/service.go new file mode 100644 index 00000000..10bff660 --- /dev/null +++ b/pkg/db/postgresql/service.go @@ -0,0 +1,184 @@ +package postgresql + +import ( + "context" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// Static postgres queries, for each modification in the tables, the table needs to be reseted +var ( + // wlogrus associated with the postgres db + PsqlType = "postgres-db" + wlog = logrus.WithField( + "module", PsqlType, + ) + MAX_BATCH_QUEUE = 300 + MAX_EPOCH_BATCH_QUEUE = 1 +) + +type PostgresDBService struct { + // Control Variables + ctx context.Context + cancel context.CancelFunc + connectionUrl string // the url might not be necessary (better to remove it?¿) + psqlPool *pgxpool.Pool + + WriteChan chan pgx.Batch + doneTasks chan interface{} + endProcess int32 + FinishSignalChan chan struct{} + workerNum int + // Network DB Model +} + +// Connect to the PostgreSQL Database and get the multithread-proof connection +// from the given url-composed credentials +func ConnectToDB(ctx context.Context, url string, chanLength int, workerNum int) (*PostgresDBService, error) { + mainCtx, cancel := context.WithCancel(ctx) + // spliting the url to don't share any confidential information on wlogs + wlog.Infof("Conneting to postgres DB %s", url) + if strings.Contains(url, "@") { + wlog.Debugf("Connecting to PostgresDB at %s", strings.Split(url, "@")[1]) + } + psqlPool, err := pgxpool.Connect(mainCtx, url) + if err != nil { + return nil, err + } + if strings.Contains(url, "@") { + wlog.Infof("PostgresDB %s succesfully connected", strings.Split(url, "@")[1]) + } + // filter the type of network that we are filtering + + psqlDB := &PostgresDBService{ + ctx: mainCtx, + cancel: cancel, + connectionUrl: url, + psqlPool: psqlPool, + WriteChan: make(chan pgx.Batch, chanLength), + doneTasks: make(chan interface{}, 1), + endProcess: 0, + FinishSignalChan: make(chan struct{}, 1), + workerNum: workerNum, + } + // init the psql db + err = psqlDB.init(ctx, psqlDB.psqlPool) + if err != nil { + return psqlDB, errors.Wrap(err, "error initializing the tables of the psqldb") + } + go psqlDB.runWriters() + return psqlDB, err +} + +// Close the connection with the PostgreSQL +func (p *PostgresDBService) Close() { +} + +func (p *PostgresDBService) init(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + err := p.createRewardsTable(ctx, pool) + if err != nil { + return err + } + + err = p.createEpochMetricsTable(ctx, pool) + if err != nil { + return err + } + + err = p.createProposerDutiesTable(ctx, pool) + if err != nil { + return err + } + + return nil +} + +func (p *PostgresDBService) DoneTasks() { + atomic.AddInt32(&p.endProcess, int32(1)) + wlog.Infof("Received finish signal") +} + +func (p *PostgresDBService) runWriters() { + var wgDBWriters sync.WaitGroup + wlog.Info("Launching Beacon State Writers") + wlog.Infof("Launching %d Beacon State Writers", p.workerNum) + for i := 0; i < p.workerNum; i++ { + wgDBWriters.Add(1) + go func(dbWriterID int) { + defer wgDBWriters.Done() + wlogWriter := wlog.WithField("DBWriter", dbWriterID) + loop: + for { + + if p.endProcess >= 1 && len(p.WriteChan) == 0 { + wlogWriter.Info("finish detected, closing persister") + break loop + } + + select { + case task := <-p.WriteChan: + wlogWriter.Debugf("Received new write task") + err := p.ExecuteBatch(task) + if err != nil { + wlogWriter.Errorf("Error processing batch", err.Error()) + } + + case <-p.ctx.Done(): + wlogWriter.Info("shutdown detected, closing persister") + break loop + default: + } + + } + wlogWriter.Debugf("DB Writer finished...") + + }(i) + } + + wgDBWriters.Wait() + p.FinishSignalChan <- struct{}{} + +} + +type WriteTask struct { + QueryID int + ModelObj interface{} +} + +func (p PostgresDBService) ExecuteBatch(batch pgx.Batch) error { + + // for i := 0; i < batch.Len(); i++ { + // wlog.Tracef("Executing SQL: %s", batch.items[i]) + // } + snapshot := time.Now() + tx, err := p.psqlPool.Begin(p.ctx) + if err != nil { + panic(err) + } + + batchResults := tx.SendBatch(p.ctx, &batch) + + var qerr error + var rows pgx.Rows + for qerr == nil { + rows, qerr = batchResults.Query() + rows.Close() + } + if qerr.Error() != "no result" { + wlog.Errorf(qerr.Error()) + } + + // p.MonitorStruct.AddDBWrite(time.Since(snapshot).Seconds()) + wlog.Debugf("Batch process time: %f, batch size: %d", time.Since(snapshot).Seconds(), batch.Len()) + + return tx.Commit(p.ctx) + +} diff --git a/pkg/db/postgresql/validator_rewards.go b/pkg/db/postgresql/validator_rewards.go new file mode 100644 index 00000000..37118b02 --- /dev/null +++ b/pkg/db/postgresql/validator_rewards.go @@ -0,0 +1,18 @@ +package postgresql + +import ( + "context" + + "github.com/cortze/eth2-state-analyzer/pkg/db/postgresql/model" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" +) + +func (p *PostgresDBService) createRewardsTable(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + _, err := pool.Exec(ctx, model.CreateValidatorRewardsTable) + if err != nil { + return errors.Wrap(err, "error creating rewards table") + } + return nil +} diff --git a/pkg/fork_metrics/altair.go b/pkg/fork_metrics/altair.go new file mode 100644 index 00000000..26e87837 --- /dev/null +++ b/pkg/fork_metrics/altair.go @@ -0,0 +1,165 @@ +package fork_metrics + +import ( + "math" + + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics/fork_state" +) + +type AltairMetrics struct { + StateMetricsBase +} + +func NewAltairMetrics(nextBstate fork_state.ForkStateContentBase, bstate fork_state.ForkStateContentBase, prevBstate fork_state.ForkStateContentBase) AltairMetrics { + + altairObj := AltairMetrics{} + altairObj.CurrentState = bstate + altairObj.PrevState = prevBstate + altairObj.NextState = nextBstate + + return altairObj +} + +func (p AltairMetrics) GetMetricsBase() StateMetricsBase { + return p.StateMetricsBase +} + +// TODO: to be implemented once we can process each block +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#modified-process_attestation +func (p AltairMetrics) GetMaxProposerAttReward(valIdx uint64) (uint64, int64) { + + proposerSlot := -1 + reward := 0 + duties := p.NextState.EpochStructs.ProposerDuties + // validator will only have duties it is active at this point + for _, duty := range duties { + if duty.ValidatorIndex == phase0.ValidatorIndex(valIdx) { + proposerSlot = int(duty.Slot) + break + } + } + + return uint64(reward), int64(proposerSlot) + +} + +// TODO: to be implemented once we can process each block +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#sync-aggregate-processing +func (p AltairMetrics) GetMaxProposerSyncReward(valIdx uint64, valPubKey phase0.BLSPubKey, valEffectiveBalance uint64, totalEffectiveBalance uint64) int64 { + + return 0 + +} + +// So far we have computed the max sync committee proposer reward for a slot. Since the validator remains in the sync committee for the full epoch, we multiply the reward for the 32 slots in the epoch. +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#sync-aggregate-processing +func (p AltairMetrics) GetMaxSyncComReward(valIdx uint64) uint64 { + + inCommittee := false + + valPubKey := p.NextState.Validators[valIdx].PublicKey + + syncCommitteePubKeys := p.NextState.SyncCommittee + + for _, item := range syncCommitteePubKeys.Pubkeys { + if valPubKey == item { + inCommittee = true + } + } + + if !inCommittee { + return 0 + } + + // at this point we know the validator was inside the sync committee and, therefore, active at that point + + totalActiveInc := p.NextState.TotalActiveBalance / fork_state.EFFECTIVE_BALANCE_INCREMENT + totalBaseRewards := p.GetBaseRewardPerInc(p.NextState.TotalActiveBalance) * uint64(totalActiveInc) + maxParticipantRewards := totalBaseRewards * uint64(fork_state.SYNC_REWARD_WEIGHT) / uint64(fork_state.WEIGHT_DENOMINATOR) / fork_state.SLOTS_PER_EPOCH + participantReward := maxParticipantRewards / uint64(fork_state.SYNC_COMMITTEE_SIZE) // this is the participantReward for a single slot + + return participantReward * uint64(fork_state.SLOTS_PER_EPOCH-len(p.NextState.MissedBlocks)) // max reward would be 32 perfect slots + +} + +// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#get_flag_index_deltas +func (p AltairMetrics) GetMaxAttestationReward(valIdx uint64) uint64 { + + maxFlagsReward := uint64(0) + // the maxReward would be each flag_index_weight * base_reward * (attesting_balance_inc / total_active_balance_inc) / WEIGHT_DENOMINATOR + + if fork_state.IsActive(*p.NextState.Validators[valIdx], phase0.Epoch(p.PrevState.Epoch)) { + baseReward := p.GetBaseReward(valIdx, uint64(p.CurrentState.Validators[valIdx].EffectiveBalance), p.CurrentState.TotalActiveBalance) + // only consider flag Index rewards if the validator was active in the previous epoch + + for i := range p.CurrentState.AttestingBalance { + + // apply formula + attestingBalanceInc := p.CurrentState.AttestingBalance[i] / fork_state.EFFECTIVE_BALANCE_INCREMENT + + flagReward := uint64(fork_state.PARTICIPATING_FLAGS_WEIGHT[i]) * baseReward * uint64(attestingBalanceInc) + flagReward = flagReward / ((uint64(p.CurrentState.TotalActiveBalance / fork_state.EFFECTIVE_BALANCE_INCREMENT)) * uint64(fork_state.WEIGHT_DENOMINATOR)) + maxFlagsReward += flagReward + } + } + + return maxFlagsReward +} + +// This method returns the Max Reward the validator could gain +// Keep in mind we are calculating rewards at the last slot of the current epoch +// The max reward we calculate now, will be seen in the next epoch, but we will do this at the last slot of it. +// Therefore we consider: +// Attestations from last epoch (we see them in this epoch), balance change will take effect in the first slot of next epoch +// Sync Committee attestations from next epoch: balance change is added on the fly +// Proposer Rewards from next epoch: balance change is added on the fly + +func (p AltairMetrics) GetMaxReward(valIdx uint64) (ValidatorSepRewards, error) { + + baseReward := p.GetBaseReward(valIdx, uint64(p.NextState.Validators[valIdx].EffectiveBalance), p.NextState.TotalActiveBalance) + + flagIndexMaxReward := p.GetMaxAttestationReward(valIdx) + + syncComMaxReward := p.GetMaxSyncComReward(valIdx) + + inSyncCommitte := false + if syncComMaxReward > 0 { + inSyncCommitte = true + } + + _, proposerSlot := p.GetMaxProposerAttReward( + valIdx) + + maxReward := flagIndexMaxReward + syncComMaxReward + + result := ValidatorSepRewards{ + Attestation: 0, + InclusionDelay: 0, + FlagIndex: flagIndexMaxReward, + SyncCommittee: syncComMaxReward, + MaxReward: maxReward, + BaseReward: baseReward, + ProposerSlot: proposerSlot, + InSyncCommittee: inSyncCommitte, + } + return result, nil + +} + +func (p AltairMetrics) GetBaseReward(valIdx uint64, effectiveBalance uint64, totalEffectiveBalance uint64) uint64 { + effectiveBalanceInc := effectiveBalance / fork_state.EFFECTIVE_BALANCE_INCREMENT + return p.GetBaseRewardPerInc(totalEffectiveBalance) * uint64(effectiveBalanceInc) +} + +func (p AltairMetrics) GetBaseRewardPerInc(totalEffectiveBalance uint64) uint64 { + + var baseReward uint64 + + sqrt := uint64(math.Sqrt(float64(totalEffectiveBalance))) + + num := fork_state.EFFECTIVE_BALANCE_INCREMENT * fork_state.BASE_REWARD_FACTOR + baseReward = uint64(num) / uint64(sqrt) + + return baseReward +} diff --git a/pkg/fork_metrics/bellatrix.go b/pkg/fork_metrics/bellatrix.go new file mode 100644 index 00000000..479675d3 --- /dev/null +++ b/pkg/fork_metrics/bellatrix.go @@ -0,0 +1 @@ +package fork_metrics diff --git a/pkg/fork_metrics/fork_state/additional_structs.go b/pkg/fork_metrics/fork_state/additional_structs.go new file mode 100644 index 00000000..1487f60c --- /dev/null +++ b/pkg/fork_metrics/fork_state/additional_structs.go @@ -0,0 +1,103 @@ +package fork_state + +import ( + "context" + "math" + "strconv" + + api "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/http" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/cortze/eth2-state-analyzer/pkg/utils" +) + +type EpochData struct { + ProposerDuties []*api.ProposerDuty + BeaconCommittees []*api.BeaconCommittee // Beacon Committees organized by slot for the whole epoch + ValidatorAttSlot map[uint64]uint64 // for each validator we have which slot it had to attest to + ValidatorsPerSlot map[uint64][]uint64 // each Slot, which validators had to attest +} + +func NewEpochData(iApi *http.Service, slot uint64) EpochData { + + epochCommittees, err := iApi.BeaconCommittees(context.Background(), strconv.Itoa(int(slot))) + + if err != nil { + log.Errorf(err.Error()) + } + + validatorsAttSlot := make(map[uint64]uint64) // each validator, when it had to attest + validatorsPerSlot := make(map[uint64][]uint64) + + for _, committee := range epochCommittees { + for _, valID := range committee.Validators { + validatorsAttSlot[uint64(valID)] = uint64(committee.Slot) + + if val, ok := validatorsPerSlot[uint64(committee.Slot)]; ok { + // the slot exists in the map + validatorsPerSlot[uint64(committee.Slot)] = append(val, uint64(valID)) + } else { + // the slot does not exist, create + validatorsPerSlot[uint64(committee.Slot)] = []uint64{uint64(valID)} + } + } + } + + proposerDuties, err := iApi.ProposerDuties(context.Background(), phase0.Epoch(utils.GetEpochFromSlot(uint64(slot))), nil) + + if err != nil { + log.Errorf(err.Error()) + } + + return EpochData{ + ProposerDuties: proposerDuties, + BeaconCommittees: epochCommittees, + ValidatorAttSlot: validatorsAttSlot, + ValidatorsPerSlot: validatorsPerSlot, + } +} + +func (p EpochData) GetValList(slot uint64, committeeIndex uint64) []phase0.ValidatorIndex { + for _, committee := range p.BeaconCommittees { + if (uint64(committee.Slot) == slot) && (uint64(committee.Index) == committeeIndex) { + return committee.Validators + } + } + + return nil +} + +func GetEffectiveBalance(balance float64) float64 { + return math.Min(MAX_EFFECTIVE_INCREMENTS*EFFECTIVE_BALANCE_INCREMENT, balance) +} + +type ValVote struct { + ValId uint64 + AttestedSlot []uint64 + InclusionSlot []uint64 +} + +func (p *ValVote) AddNewAtt(attestedSlot uint64, inclusionSlot uint64) { + + if p.AttestedSlot == nil { + p.AttestedSlot = make([]uint64, 0) + } + + if p.InclusionSlot == nil { + p.InclusionSlot = make([]uint64, 0) + } + + // keep in mind that for the proposer, the vote only counts if it is the first to include this attestation + for i, item := range p.AttestedSlot { + if item == attestedSlot { + if inclusionSlot < p.InclusionSlot[i] { + p.InclusionSlot[i] = inclusionSlot + } + return + } + } + + p.AttestedSlot = append(p.AttestedSlot, attestedSlot) + p.InclusionSlot = append(p.InclusionSlot, inclusionSlot) + +} diff --git a/pkg/fork_metrics/fork_state/altair.go b/pkg/fork_metrics/fork_state/altair.go new file mode 100644 index 00000000..39516b7d --- /dev/null +++ b/pkg/fork_metrics/fork_state/altair.go @@ -0,0 +1,76 @@ +package fork_state + +import ( + "math" + + "github.com/attestantio/go-eth2-client/http" + "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/altair" + "github.com/cortze/eth2-state-analyzer/pkg/utils" +) + +var ( // spec weight constants + TIMELY_SOURCE_WEIGHT = 14 + TIMELY_TARGET_WEIGHT = 26 + TIMELY_HEAD_WEIGHT = 14 + PARTICIPATING_FLAGS_WEIGHT = []int{TIMELY_SOURCE_WEIGHT, TIMELY_TARGET_WEIGHT, TIMELY_HEAD_WEIGHT} + SYNC_REWARD_WEIGHT = 2 + PROPOSER_WEIGHT = 8 + WEIGHT_DENOMINATOR = 64 + SYNC_COMMITTEE_SIZE = 512 +) + +// This Wrapper is meant to include all necessary data from the Altair Fork +func NewAltairState(bstate spec.VersionedBeaconState, iApi *http.Service) ForkStateContentBase { + + altairObj := ForkStateContentBase{ + Version: bstate.Version, + Balances: bstate.Altair.Balances, + Validators: bstate.Altair.Validators, + EpochStructs: NewEpochData(iApi, bstate.Altair.Slot), + Epoch: utils.GetEpochFromSlot(bstate.Altair.Slot), + Slot: bstate.Altair.Slot, + BlockRoots: bstate.Altair.BlockRoots, + SyncCommittee: *bstate.Altair.CurrentSyncCommittee, + } + + altairObj.Setup() + + ProcessAttestations(&altairObj, bstate.Altair.PreviousEpochParticipation) + + return altairObj +} + +func ProcessAttestations(customState *ForkStateContentBase, participation []altair.ParticipationFlags) { + // calculate attesting vals only once + flags := []altair.ParticipationFlag{ + altair.TimelySourceFlagIndex, + altair.TimelyTargetFlagIndex, + altair.TimelyHeadFlagIndex} + + for participatingFlag := range flags { + + flag := altair.ParticipationFlags(math.Pow(2, float64(participatingFlag))) + + for valIndex, item := range participation { + // Here we have one item per validator + // Item is a 3-bit string + // each bit represents a flag + + if (item & flag) == flag { + // The attestation has a timely flag, therefore we consider it correct flag + customState.CorrectFlags[participatingFlag][valIndex] += uint64(1) + + // we sum the attesting balance in the corresponding flag index + customState.AttestingBalance[participatingFlag] += uint64(customState.Validators[valIndex].EffectiveBalance) + + // if this validator was not counted as attesting before, count it now + if !customState.AttestingVals[valIndex] { + customState.NumAttestingVals++ + customState.MaxAttestingBalance = uint64(customState.Validators[valIndex].EffectiveBalance) + } + customState.AttestingVals[valIndex] = true + } + } + } +} diff --git a/pkg/fork_metrics/fork_state/bellatrix.go b/pkg/fork_metrics/fork_state/bellatrix.go new file mode 100644 index 00000000..c32a5379 --- /dev/null +++ b/pkg/fork_metrics/fork_state/bellatrix.go @@ -0,0 +1,28 @@ +package fork_state + +import ( + "github.com/attestantio/go-eth2-client/http" + "github.com/attestantio/go-eth2-client/spec" + "github.com/cortze/eth2-state-analyzer/pkg/utils" +) + +// This Wrapper is meant to include all necessary data from the Bellatrix Fork +func NewBellatrixState(bstate spec.VersionedBeaconState, iApi *http.Service) ForkStateContentBase { + + bellatrixObj := ForkStateContentBase{ + Version: bstate.Version, + Balances: bstate.Bellatrix.Balances, + Validators: bstate.Bellatrix.Validators, + EpochStructs: NewEpochData(iApi, bstate.Bellatrix.Slot), + Epoch: utils.GetEpochFromSlot(bstate.Bellatrix.Slot), + Slot: bstate.Bellatrix.Slot, + BlockRoots: bstate.Bellatrix.BlockRoots, + SyncCommittee: *bstate.Bellatrix.CurrentSyncCommittee, + } + + bellatrixObj.Setup() + + ProcessAttestations(&bellatrixObj, bstate.Bellatrix.PreviousEpochParticipation) + + return bellatrixObj +} diff --git a/pkg/fork_metrics/fork_state/phase0.go b/pkg/fork_metrics/fork_state/phase0.go new file mode 100644 index 00000000..87d94fab --- /dev/null +++ b/pkg/fork_metrics/fork_state/phase0.go @@ -0,0 +1,39 @@ +package fork_state + +import ( + "github.com/attestantio/go-eth2-client/http" + "github.com/attestantio/go-eth2-client/spec" + "github.com/cortze/eth2-state-analyzer/pkg/utils" +) + +const ( + MAX_EFFECTIVE_INCREMENTS = 32 + BASE_REWARD_FACTOR = 64 + BASE_REWARD_PER_EPOCH = 4 + EFFECTIVE_BALANCE_INCREMENT = 1000000000 + SLOTS_PER_EPOCH = 32 + SHUFFLE_ROUND_COUNT = uint64(90) + PROPOSER_REWARD_QUOTIENT = 8 + GENESIS_EPOCH = 0 + SLOTS_PER_HISTORICAL_ROOT = 8192 +) + +// This Wrapper is meant to include all necessary data from the Phase0 Fork +func NewPhase0State(bstate spec.VersionedBeaconState, iApi *http.Service) ForkStateContentBase { + + phase0Obj := ForkStateContentBase{ + Version: bstate.Version, + Balances: bstate.Phase0.Balances, + Validators: bstate.Phase0.Validators, + EpochStructs: NewEpochData(iApi, bstate.Phase0.Slot), + Epoch: utils.GetEpochFromSlot(bstate.Phase0.Slot), + Slot: bstate.Phase0.Slot, + BlockRoots: bstate.Phase0.BlockRoots, + PrevAttestations: bstate.Phase0.PreviousEpochAttestations, + } + + phase0Obj.Setup() + + return phase0Obj + +} diff --git a/pkg/fork_metrics/fork_state/standard.go b/pkg/fork_metrics/fork_state/standard.go new file mode 100644 index 00000000..68fe639b --- /dev/null +++ b/pkg/fork_metrics/fork_state/standard.go @@ -0,0 +1,265 @@ +package fork_state + +import ( + "bytes" + "fmt" + + "github.com/attestantio/go-eth2-client/http" + "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/altair" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/sirupsen/logrus" +) + +var ( + log = logrus.WithField( + "module", "FoskStateContent", + ) +) + +// This Wrapper is meant to include all common objects across Ethereum Hard Fork Specs +type ForkStateContentBase struct { + Version spec.DataVersion + Balances []uint64 // balance of each validator + Validators []*phase0.Validator // list of validators + TotalActiveBalance uint64 // effective balance + TotalActiveRealBalance uint64 // real balance + AttestingBalance []uint64 // one attesting balance per flag + MaxAttestingBalance uint64 // the effective balance of validators that did attest in any manner + EpochStructs EpochData // structs about beacon committees, proposers and attestation + CorrectFlags [][]uint64 // one aray per flag + AttestingVals []bool // the number of validators that did attest in the last epoch + PrevAttestations []*phase0.PendingAttestation // array of attestations (currently only for Phase0) + NumAttestingVals uint64 // number of validators that attested in the last epoch + NumActiveVals uint64 // number of active validators in the epoch + ValAttestationInclusion map[uint64]ValVote // one map per validator including which slots it had to attest and when it was included + AttestedValsPerSlot map[uint64][]uint64 // for each slot in the epoch, how many vals attested + Epoch uint64 // Epoch of the state + Slot uint64 // Slot of the state + BlockRoots [][]byte // array of block roots at this point (8192) + MissedBlocks []uint64 // blocks missed in the epoch until this point + SyncCommittee altair.SyncCommittee // list of pubkeys in the current sync committe +} + +func GetCustomState(bstate spec.VersionedBeaconState, iApi *http.Service) (ForkStateContentBase, error) { + switch bstate.Version { + + case spec.DataVersionPhase0: + return NewPhase0State(bstate, iApi), nil + + case spec.DataVersionAltair: + return NewAltairState(bstate, iApi), nil + + case spec.DataVersionBellatrix: + return NewBellatrixState(bstate, iApi), nil + default: + return ForkStateContentBase{}, fmt.Errorf("could not figure out the Beacon State Fork Version: %s", bstate.Version) + } +} + +// Initialize all necessary arrays and process anything standard +func (p *ForkStateContentBase) Setup() error { + if p.Validators == nil { + return fmt.Errorf("validator list not provided, cannot create") + } + arrayLen := len(p.Validators) + if p.PrevAttestations == nil { + p.PrevAttestations = make([]*phase0.PendingAttestation, 0) + } + + p.AttestingBalance = make([]uint64, 3) + p.AttestingVals = make([]bool, arrayLen) + p.CorrectFlags = make([][]uint64, 3) + p.MissedBlocks = make([]uint64, 0) + p.ValAttestationInclusion = make(map[uint64]ValVote) + p.AttestedValsPerSlot = make(map[uint64][]uint64) + + for i := range p.CorrectFlags { + p.CorrectFlags[i] = make([]uint64, arrayLen) + } + + p.TotalActiveBalance = p.GetTotalActiveEffBalance() + p.TotalActiveRealBalance = p.GetTotalActiveRealBalance() + p.TrackMissingBlocks() + return nil +} + +// the length of the valList = number of validators +// each position represents a valIdx +// if the item has a number > 0, count it +func (p ForkStateContentBase) ValsEffectiveBalance(valList []uint64) uint64 { + + resultBalance := uint64(0) + + for valIdx, item := range valList { // loop over validators + if item > 0 && valIdx < len(p.Validators) { + resultBalance += uint64(p.Validators[valIdx].EffectiveBalance) + } + } + + return uint64(resultBalance) +} + +func (p ForkStateContentBase) Balance(valIdx uint64) (uint64, error) { + if uint64(len(p.Balances)) < valIdx { + err := fmt.Errorf("phase0 - validator index %d wasn't activated in slot %d", valIdx, p.Slot) + return 0, err + } + balance := p.Balances[valIdx] + + return balance, nil +} + +// Edit NumActiveVals +func (p *ForkStateContentBase) GetTotalActiveEffBalance() uint64 { + + val_array := make([]uint64, len(p.Validators)) + p.NumActiveVals = 0 // any time we calculate total effective balance, the number of active vals is refreshed and recalculated + for idx := range val_array { + if IsActive(*p.Validators[idx], phase0.Epoch(p.Epoch)) { + val_array[idx] += 1 + p.NumActiveVals++ + } + + } + + return p.ValsEffectiveBalance(val_array) +} + +// Not effective balance, but balance +func (p ForkStateContentBase) GetTotalActiveRealBalance() uint64 { + totalBalance := uint64(0) + + for idx := range p.Validators { + if IsActive(*p.Validators[idx], phase0.Epoch(p.Epoch)) { + totalBalance += p.Balances[idx] + } + + } + return totalBalance +} + +func IsActive(validator phase0.Validator, epoch phase0.Epoch) bool { + if validator.ActivationEpoch <= epoch && + epoch < validator.ExitEpoch { + return true + } + return false +} + +// check if there was a missed block at last slot of previous epoch +func (p ForkStateContentBase) TrackPrevMissingBlock() uint64 { + firstIndex := (p.Slot - SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT + + lastItem := p.BlockRoots[firstIndex-1] + item := p.BlockRoots[firstIndex] + res := bytes.Compare(lastItem, item) + + if res == 0 { + // both consecutive roots were the same ==> missed block + slot := p.Slot - SLOTS_PER_EPOCH + return slot + } + + return 0 + +} + +// We use blockroots to track missed blocks. When there is a missed block, the block root is repeated +func (p *ForkStateContentBase) TrackMissingBlocks() { + firstIndex := (p.Slot - SLOTS_PER_EPOCH + 1) % SLOTS_PER_HISTORICAL_ROOT + lastIndex := (p.Slot) % SLOTS_PER_HISTORICAL_ROOT + + for i := firstIndex; i <= lastIndex; i++ { + if i == 0 { + continue + } + lastItem := p.BlockRoots[i-1] + item := p.BlockRoots[i] + res := bytes.Compare(lastItem, item) + + if res == 0 { + // both consecutive roots were the same ==> missed block + slot := i - firstIndex + p.Slot - SLOTS_PER_EPOCH + 1 + p.MissedBlocks = append(p.MissedBlocks, uint64(slot)) + } + } +} + +// List of validators that were active in the epoch of the state +// Lenght of the list is variable, each position containing the valIdx +func (p ForkStateContentBase) GetActiveVals() []uint64 { + result := make([]uint64, 0) + + for i, val := range p.Validators { + if IsActive(*val, phase0.Epoch(p.Epoch)) { + result = append(result, uint64(i)) + } + + } + return result +} + +// List of validators that were in the epoch of the state +// Length of the list is variable, each position containing the valIdx +func (p ForkStateContentBase) GetAllVals() []uint64 { + result := make([]uint64, 0) + + for i := range p.Validators { + result = append(result, uint64(i)) + + } + return result +} + +// Returns a list of missing flags for the corresponding valIdx +func (p ForkStateContentBase) MissingFlags(valIdx uint64) []bool { + result := []bool{false, false, false} + + if int(valIdx) >= len(p.CorrectFlags[0]) { + return result + } + + for i, item := range p.CorrectFlags { + if IsActive(*p.Validators[valIdx], phase0.Epoch(p.Epoch-1)) && item[valIdx] == 0 { + if item[valIdx] == 0 { + // no missing flag + result[i] = true + } + } + + } + return result +} + +// Argument: 0 for source, 1 for target and 2 for head +// Return the count of missing flag in the previous epoch participation / attestations +func (p ForkStateContentBase) GetMissingFlagCount(flagIndex int) uint64 { + result := uint64(0) + for idx, item := range p.CorrectFlags[flagIndex] { + // if validator was active and no correct flag + if IsActive(*p.Validators[idx], phase0.Epoch(p.Epoch-1)) && item == 0 { + result += 1 + } + } + + return result +} + +func (p ForkStateContentBase) GetValStatus(valIdx uint64) string { + + if p.Validators[valIdx].ExitEpoch <= phase0.Epoch(p.Epoch) { + return "exit" + } + + if p.Validators[valIdx].Slashed { + return "slashed" + } + + if p.Validators[valIdx].ActivationEpoch <= phase0.Epoch(p.Epoch) { + return "active" + } + + return "in queue to activation" + +} diff --git a/pkg/fork_metrics/phase0.go b/pkg/fork_metrics/phase0.go new file mode 100644 index 00000000..56d0f8a6 --- /dev/null +++ b/pkg/fork_metrics/phase0.go @@ -0,0 +1,222 @@ +package fork_metrics + +import ( + "bytes" + "math" + + "github.com/attestantio/go-eth2-client/spec/altair" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics/fork_state" +) + +// var ( +// log = logrus.WithField( +// "module", "custom_spec", +// ) +// ) + +type Phase0Metrics struct { + StateMetricsBase +} + +func NewPhase0Metrics(nextBstate fork_state.ForkStateContentBase, currentState fork_state.ForkStateContentBase, prevState fork_state.ForkStateContentBase) Phase0Metrics { + + phase0Obj := Phase0Metrics{} + phase0Obj.NextState = nextBstate + phase0Obj.CurrentState = currentState + phase0Obj.PrevState = prevState + + phase0Obj.CalculateAttestingVals() + return phase0Obj + +} + +func (p Phase0Metrics) GetMetricsBase() StateMetricsBase { + return p.StateMetricsBase +} + +// Processes attestations and fills several structs +func (p *Phase0Metrics) CalculateAttestingVals() { + + for _, item := range p.CurrentState.PrevAttestations { + + slot := item.Data.Slot // Block that is being attested, not included + committeeIndex := item.Data.Index // committee in the attested slot + inclusionSlot := slot + item.InclusionDelay + + validatorIDs := p.PrevState.EpochStructs.GetValList(uint64(slot), uint64(committeeIndex)) // Beacon Committee + + attestingIndices := item.AggregationBits.BitIndices() // we only get the 1s, meaning the validator voted + + for _, index := range attestingIndices { + attestingValIdx := validatorIDs[index] + + p.CurrentState.AttestingVals[attestingValIdx] = true + + // add correct flags and balances + if p.IsCorrectSource() && p.CurrentState.CorrectFlags[altair.TimelySourceFlagIndex][attestingValIdx] == 0 { + p.CurrentState.CorrectFlags[altair.TimelySourceFlagIndex][attestingValIdx] += 1 + p.CurrentState.AttestingBalance[altair.TimelySourceFlagIndex] += uint64(p.CurrentState.Validators[attestingValIdx].EffectiveBalance) + } + + if p.IsCorrectTarget(*item) && p.CurrentState.CorrectFlags[altair.TimelyTargetFlagIndex][attestingValIdx] == 0 { + p.CurrentState.CorrectFlags[altair.TimelyTargetFlagIndex][attestingValIdx] += 1 + p.CurrentState.AttestingBalance[altair.TimelyTargetFlagIndex] += uint64(p.CurrentState.Validators[attestingValIdx].EffectiveBalance) + } + + if p.IsCorrectHead(*item) && p.CurrentState.CorrectFlags[altair.TimelyHeadFlagIndex][attestingValIdx] == 0 { + p.CurrentState.CorrectFlags[altair.TimelyHeadFlagIndex][attestingValIdx] += 1 + p.CurrentState.AttestingBalance[altair.TimelyHeadFlagIndex] += uint64(p.CurrentState.Validators[attestingValIdx].EffectiveBalance) + } + + // we also organize which validator attested when, and when was the attestation included + if val, ok := p.CurrentState.ValAttestationInclusion[uint64(attestingValIdx)]; ok { + // it already existed + val.AddNewAtt(uint64(slot), uint64(inclusionSlot)) + p.CurrentState.ValAttestationInclusion[uint64(attestingValIdx)] = val + } else { + + // it did not exist + newAtt := fork_state.ValVote{ + ValId: uint64(attestingValIdx), + } + newAtt.AddNewAtt(uint64(slot), uint64(inclusionSlot)) + p.CurrentState.ValAttestationInclusion[uint64(attestingValIdx)] = newAtt + + } + } + + } +} + +// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#components-of-attestation-deltas +func (p Phase0Metrics) GetMaxProposerReward(valIdx uint64, baseReward uint64) (uint64, int64) { + + isProposer := false + proposerSlot := 0 + duties := append(p.CurrentState.EpochStructs.ProposerDuties, p.PrevState.EpochStructs.ProposerDuties...) + // there will be no duties if the validator is not active + for _, duty := range duties { + if duty.ValidatorIndex == phase0.ValidatorIndex(valIdx) { + isProposer = true + proposerSlot = int(duty.Slot) + break + } + } + + if isProposer { + votesIncluded := 0 + for _, valAttestation := range p.CurrentState.ValAttestationInclusion { + for _, item := range valAttestation.InclusionSlot { + if item == uint64(proposerSlot) { + // the block the attestation was included is the same as the slot the val proposed a block + // therefore, proposer included this attestation + votesIncluded += 1 + } + } + } + if votesIncluded > 0 { + return (baseReward / fork_state.PROPOSER_REWARD_QUOTIENT) * uint64(votesIncluded), int64(proposerSlot) + } + + } + + return 0, -1 +} + +// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#rewards-and-penalties-1 +// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#components-of-attestation-deltas +func (p Phase0Metrics) GetMaxReward(valIdx uint64) (ValidatorSepRewards, error) { + + if p.CurrentState.Epoch == fork_state.GENESIS_EPOCH { // No rewards are applied at genesis + return ValidatorSepRewards{}, nil + } + baseReward := p.GetBaseReward(uint64(p.CurrentState.Validators[valIdx].EffectiveBalance)) + voteReward := uint64(0) + proposerReward := uint64(0) + proposerSlot := int64(-1) + maxReward := uint64(0) + inclusionDelayReward := uint64(0) + + if fork_state.IsActive(*p.NextState.Validators[valIdx], phase0.Epoch(p.PrevState.Epoch)) { + // only consider attestations rewards in case the validator was active in the previous epoch + for i := range p.CurrentState.CorrectFlags { + + previousAttestedBalance := p.CurrentState.AttestingBalance[i] + + // participationRate per flag ==> previousAttestBalance / TotalActiveBalance + singleReward := baseReward * (uint64(previousAttestedBalance) / fork_state.EFFECTIVE_BALANCE_INCREMENT) + + // for each flag, we add baseReward * participationRate + voteReward += singleReward / (uint64(p.CurrentState.TotalActiveBalance) / fork_state.EFFECTIVE_BALANCE_INCREMENT) + } + + proposerReward := baseReward / fork_state.PROPOSER_REWARD_QUOTIENT + // only add it when there was an attestation (correct source) + inclusionDelayReward = baseReward - proposerReward + } + + _, proposerSlot = p.GetMaxProposerReward(valIdx, baseReward) + maxReward = voteReward + inclusionDelayReward + proposerReward + + result := ValidatorSepRewards{ + Attestation: voteReward, + InclusionDelay: inclusionDelayReward, + FlagIndex: 0, + SyncCommittee: 0, + MaxReward: maxReward, + BaseReward: baseReward, + ProposerSlot: proposerSlot, + InSyncCommittee: false, + } + return result, nil +} + +// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#helper-functions-1 +func (p Phase0Metrics) IsCorrectSource() bool { + epoch := phase0.Epoch(p.CurrentState.Slot / fork_state.SLOTS_PER_EPOCH) + if epoch == phase0.Epoch(p.CurrentState.Epoch) || epoch == phase0.Epoch(p.PrevState.Epoch) { + return true + } + return false +} + +// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#helper-functions-1 +func (p Phase0Metrics) IsCorrectTarget(attestation phase0.PendingAttestation) bool { + target := attestation.Data.Target.Root + + slot := int(p.PrevState.Slot / fork_state.SLOTS_PER_EPOCH) + slot = slot * fork_state.SLOTS_PER_EPOCH + expected := p.PrevState.BlockRoots[slot%fork_state.SLOTS_PER_HISTORICAL_ROOT] + + res := bytes.Compare(target[:], expected) + + return res == 0 // if 0, then block roots are the same +} + +// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#helper-functions-1 +func (p Phase0Metrics) IsCorrectHead(attestation phase0.PendingAttestation) bool { + head := attestation.Data.BeaconBlockRoot + + index := attestation.Data.Slot % fork_state.SLOTS_PER_HISTORICAL_ROOT + expected := p.CurrentState.BlockRoots[index] + + res := bytes.Compare(head[:], expected) + return res == 0 // if 0, then block roots are the same +} + +// BaseReward = ( effectiveBalance * (BaseRewardFactor)/(BaseRewardsPerEpoch * sqrt(activeBalance)) ) +// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#helpers +func (p Phase0Metrics) GetBaseReward(valEffectiveBalance uint64) uint64 { + + var baseReward uint64 + + sqrt := uint64(math.Sqrt(float64(p.CurrentState.TotalActiveBalance))) + + denom := uint64(fork_state.BASE_REWARD_PER_EPOCH * sqrt) + + num := (uint64(valEffectiveBalance) * fork_state.BASE_REWARD_FACTOR) + baseReward = num / denom + + return baseReward +} diff --git a/pkg/fork_metrics/standard.go b/pkg/fork_metrics/standard.go new file mode 100644 index 00000000..d49dcce8 --- /dev/null +++ b/pkg/fork_metrics/standard.go @@ -0,0 +1,73 @@ +package fork_metrics + +import ( + "fmt" + + "github.com/attestantio/go-eth2-client/http" + "github.com/attestantio/go-eth2-client/spec" + "github.com/cortze/eth2-state-analyzer/pkg/fork_metrics/fork_state" +) + +type StateMetricsBase struct { + CurrentState fork_state.ForkStateContentBase + PrevState fork_state.ForkStateContentBase + NextState fork_state.ForkStateContentBase +} + +func (p StateMetricsBase) EpochReward(valIdx uint64) int64 { + if valIdx < uint64(len(p.CurrentState.Balances)) && valIdx < uint64(len(p.NextState.Balances)) { + return int64(p.NextState.Balances[valIdx]) - int64(p.CurrentState.Balances[valIdx]) + } + + return 0 + +} + +func (p StateMetricsBase) GetAttSlot(valIdx uint64) uint64 { + + return p.PrevState.EpochStructs.ValidatorAttSlot[valIdx] +} + +func (p StateMetricsBase) GetAttInclusionSlot(valIdx uint64) int64 { + + for i, item := range p.CurrentState.ValAttestationInclusion[valIdx].AttestedSlot { + // we are looking for a vote to the previous epoch + if item >= p.PrevState.Slot+1-fork_state.SLOTS_PER_EPOCH && + item <= p.PrevState.Slot { + return int64(p.CurrentState.ValAttestationInclusion[valIdx].InclusionSlot[i]) + } + } + return int64(-1) +} + +type StateMetrics interface { + GetMetricsBase() StateMetricsBase + GetMaxReward(valIdx uint64) (ValidatorSepRewards, error) +} + +func StateMetricsByForkVersion(nextBstate fork_state.ForkStateContentBase, bstate fork_state.ForkStateContentBase, prevBstate fork_state.ForkStateContentBase, iApi *http.Service) (StateMetrics, error) { + switch bstate.Version { + + case spec.DataVersionPhase0: + return NewPhase0Metrics(nextBstate, bstate, prevBstate), nil + + case spec.DataVersionAltair: + return NewAltairMetrics(nextBstate, bstate, prevBstate), nil + + case spec.DataVersionBellatrix: + return NewAltairMetrics(nextBstate, bstate, prevBstate), nil // We use Altair as Rewards system is the same + default: + return nil, fmt.Errorf("could not figure out the Beacon State Fork Version: %s", bstate.Version) + } +} + +type ValidatorSepRewards struct { + Attestation uint64 + InclusionDelay uint64 + FlagIndex uint64 + SyncCommittee uint64 + MaxReward uint64 + BaseReward uint64 + InSyncCommittee bool + ProposerSlot int64 +} diff --git a/pkg/metrics/monitor.go b/pkg/metrics/monitor.go new file mode 100644 index 00000000..6f4b8522 --- /dev/null +++ b/pkg/metrics/monitor.go @@ -0,0 +1,52 @@ +package metrics + +import ( + "sync" +) + +type Monitor struct { + m sync.Mutex + DBWriteTime float64 + DownloadTime float64 + PreprocessTime float64 + BatchingTime float64 + ValidatorLength int +} + +func NewMonitorMetrics(valLength int) Monitor { + return Monitor{ + DBWriteTime: 0, + DownloadTime: 0, + PreprocessTime: 0, + BatchingTime: 0, + ValidatorLength: valLength, + } +} + +func (p *Monitor) AddDBWrite(executionTime float64) { + p.m.Lock() + p.DBWriteTime += executionTime + + p.m.Unlock() +} + +func (p *Monitor) AddDownload(executionTime float64) { + p.m.Lock() + p.DownloadTime += executionTime + + p.m.Unlock() +} + +func (p *Monitor) AddPreprocessTime(executionTime float64) { + p.m.Lock() + p.PreprocessTime += executionTime + + p.m.Unlock() +} + +func (p *Monitor) AddBatchingTime(executionTime float64) { + p.m.Lock() + p.BatchingTime += executionTime + + p.m.Unlock() +} diff --git a/pkg/utils/bitset.go b/pkg/utils/bitset.go new file mode 100644 index 00000000..6bb6529f --- /dev/null +++ b/pkg/utils/bitset.go @@ -0,0 +1,6 @@ +package utils + +// Check if bit n (0..7) is set where 0 is the LSB in little endian +func IsBitSet(input uint8, n int) bool { + return (input & (1 << n)) > uint8(0) +} diff --git a/pkg/utils/epoch.go b/pkg/utils/epoch.go index c0439a00..b63c705f 100644 --- a/pkg/utils/epoch.go +++ b/pkg/utils/epoch.go @@ -4,9 +4,9 @@ var SlotBase uint64 = 32 func GetEpochFromSlot(slot uint64) uint64 { ent := slot / SlotBase - rest := slot % uint64(SlotBase) - if rest > 0 { - ent += 1 - } + // rest := slot % uint64(SlotBase) + // if rest > 0 { + // ent += 1 + // } return ent } diff --git a/pkg/utils/validator_indexes.go b/pkg/utils/validator_indexes.go index 1373bab3..2ca6b8f9 100644 --- a/pkg/utils/validator_indexes.go +++ b/pkg/utils/validator_indexes.go @@ -15,8 +15,23 @@ func GetValIndexesFromJson(filePath string) ([]uint64, error) { } err = json.Unmarshal(fbytes, &validatorIndex) + if err != nil { + log.Errorf("Error unmarshalling val list", err.Error()) + } + log.Infof("Readed %d validators", len(validatorIndex)) return validatorIndex, nil } + +func BoolToUint(input []bool) []uint64 { + result := make([]uint64, len(input)) + + for i, item := range input { + if item { + result[i] += 1 + } + } + return result +} diff --git a/state-analyzer b/state-analyzer index 9fb83a71..337dfc58 100755 Binary files a/state-analyzer and b/state-analyzer differ diff --git a/test_validators.json b/test_validators.json index 8e28a67b..0637a088 100644 --- a/test_validators.json +++ b/test_validators.json @@ -1 +1 @@ -[15000, 16000, 17000, 18000] \ No newline at end of file +[] \ No newline at end of file