quaistats: Reworked the stats package in the client to send new stats

Makefile

@@ -65,6 +65,9 @@ endif
 ifeq ($(QUAI_STATS),true)
 	BASE_CMD += --quaistats ${STATS_NAME}:${STATS_PASS}@${STATS_HOST}
 endif
+ifeq ($(TRUSTED_NODE),true)
+	BASE_CMD += --trustednode
+endif
 
 ifeq ($(SHOW_COLORS),true)
 	BASE_CMD += --showcolors

cmd/go-quai/config.go

@@ -156,10 +156,11 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, quaiConfig) {
 func makeFullNode(ctx *cli.Context) (*node.Node, quaiapi.Backend) {
 	stack, cfg := makeConfigNode(ctx)
 	backend, _ := utils.RegisterEthService(stack, &cfg.Eth)
+	trusted := ctx.Bool(utils.TrustedNodeFlag.Name)
 
 	// Add the Quai Stats daemon if requested.
 	if cfg.Ethstats.URL != "" {
-		utils.RegisterQuaiStatsService(stack, backend, cfg.Ethstats.URL)
+		utils.RegisterQuaiStatsService(stack, backend, cfg.Ethstats.URL, trusted)
 	}
 	return stack, backend
 }

cmd/go-quai/main.go

@@ -105,6 +105,7 @@ var (
 		utils.OrchardFlag,
 		utils.PasswordFileFlag,
 		utils.QuaiStatsURLFlag,
+		utils.TrustedNodeFlag,
 		utils.RegionFlag,
 		utils.ShowColorsFlag,
 		utils.SlicesRunningFlag,

cmd/go-quai/usage.go

@@ -51,6 +51,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.GCModeFlag,
 			utils.TxLookupLimitFlag,
 			utils.QuaiStatsURLFlag,
+			utils.TrustedNodeFlag,
 			utils.IdentityFlag,
 			utils.LightKDFFlag,
 			utils.WhitelistFlag,

cmd/utils/flags.go

@@ -399,6 +399,10 @@ var (
 		Name:  "quaistats",
 		Usage: "Reporting URL of a quaistats service (nodename:secret@host:port)",
 	}
+	TrustedNodeFlag = cli.BoolFlag{
+		Name:  "trustednode",
+		Usage: "Truted node boolean flag for use by team",
+	}
 	FakePoWFlag = cli.BoolFlag{
 		Name:  "fakepow",
 		Usage: "Disables proof-of-work verification",
@@ -1633,8 +1637,8 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (quaiapi.Backen
 
 // RegisterQuaiStatsService configures the Quai Stats daemon and adds it to
 // the given node.
-func RegisterQuaiStatsService(stack *node.Node, backend quaiapi.Backend, url string) {
-	if err := quaistats.New(stack, backend, backend.Engine(), url); err != nil {
+func RegisterQuaiStatsService(stack *node.Node, backend quaiapi.Backend, url string, trusted bool) {
+	if err := quaistats.New(stack, backend, backend.Engine(), url, trusted); err != nil {
 		Fatalf("Failed to register the Quai Stats service: %v", err)
 	}
 }

common/address.go

@@ -48,6 +48,19 @@ func (a Address) InternalAddress() (InternalAddress, error) {
 	return *internal, nil
 }
 
+func (a Address) InSameSliceAs(cmp Location) bool {
+	loc := *a.Location()
+	// Figure out which location is shorter
+	shorter := loc
+	longer := cmp
+	if len(loc) > len(cmp) {
+		longer = loc
+		shorter = cmp
+	}
+	// Compare bytes up to the shorter depth
+	return shorter.Equal(longer[:len(shorter)])
+}
+
 func (a Address) Equal(b Address) bool {
 	if a.inner == nil && b.inner == nil {
 		return true

core/headerchain.go

@@ -28,6 +28,7 @@ import (
 const (
 	headerCacheLimit      = 512
 	numberCacheLimit      = 2048
+	c_subRollupCacheSize  = 50
 	primeHorizonThreshold = 20
 )
 
@@ -62,6 +63,7 @@ type HeaderChain struct {
 	pendingEtxsRollup *lru.Cache
 	pendingEtxs       *lru.Cache
 	blooms            *lru.Cache
+	subRollupCache    *lru.Cache
 
 	wg            sync.WaitGroup // chain processing wait group for shutting down
 	running       int32          // 0 if chain is running, 1 when stopped
@@ -77,6 +79,7 @@ type HeaderChain struct {
 func NewHeaderChain(db ethdb.Database, engine consensus.Engine, pEtxsRollupFetcher getPendingEtxsRollup, pEtxsFetcher getPendingEtxs, chainConfig *params.ChainConfig, cacheConfig *CacheConfig, txLookupLimit *uint64, vmConfig vm.Config, slicesRunning []common.Location) (*HeaderChain, error) {
 	headerCache, _ := lru.New(headerCacheLimit)
 	numberCache, _ := lru.New(numberCacheLimit)
+	nodeCtx := common.NodeLocation.Context()
 
 	hc := &HeaderChain{
 		config:          chainConfig,
@@ -92,12 +95,20 @@ func NewHeaderChain(db ethdb.Database, engine consensus.Engine, pEtxsRollupFetch
 	pendingEtxsRollup, _ := lru.New(c_maxPendingEtxsRollup)
 	hc.pendingEtxsRollup = pendingEtxsRollup
 
-	pendingEtxs, _ := lru.New(c_maxPendingEtxBatches)
-	hc.pendingEtxs = pendingEtxs
+	if nodeCtx == common.PRIME_CTX {
+		pendingEtxs, _ := lru.New(c_maxPendingEtxBatchesPrime)
+		hc.pendingEtxs = pendingEtxs
+	} else {
+		pendingEtxs, _ := lru.New(c_maxPendingEtxBatchesRegion)
+		hc.pendingEtxs = pendingEtxs
+	}
 
 	blooms, _ := lru.New(c_maxBloomFilters)
 	hc.blooms = blooms
 
+	subRollupCache, _ := lru.New(c_subRollupCacheSize)
+	hc.subRollupCache = subRollupCache
+
 	hc.genesisHeader = hc.GetHeaderByNumber(0)
 	if hc.genesisHeader.Hash() != chainConfig.GenesisHash {
 		return nil, fmt.Errorf("genesis block mismatch: have %x, want %x", hc.genesisHeader.Hash(), chainConfig.GenesisHash)

core/slice.go

@@ -26,7 +26,8 @@ import (
 )
 
 const (
-	c_maxPendingEtxBatches            = 1024
+	c_maxPendingEtxBatchesPrime       = 30000
+	c_maxPendingEtxBatchesRegion      = 10000
 	c_maxPendingEtxsRollup            = 256
 	c_maxBloomFilters                 = 1024
 	c_pendingHeaderChacheBufferFactor = 2
@@ -216,22 +217,26 @@ func (sl *Slice) Append(header *types.Header, domPendingHeader *types.Header, do
 	// list of confirmed ETXs using the subordinate manifest In either case, if
 	// we are a dominant node, we need to collect the ETX rollup from our sub.
 	if !domOrigin && nodeCtx != common.ZONE_CTX {
-		newInboundEtxs, _, err = sl.CollectNewlyConfirmedEtxs(block, block.Location())
-		if err != nil {
-			log.Trace("Error collecting newly confirmed etxs: ", "err", err)
-			// Keeping track of the number of times pending etx fails and if it crossed the retry threshold
-			// ask the sub for the pending etx/rollup data
-			val, exist := sl.pEtxRetryCache.Get(block.Hash())
-			var retry uint64
-			if exist {
-				pEtxCurrent, ok := val.(pEtxRetry)
-				if ok {
-					retry = pEtxCurrent.retries + 1
+		newInboundEtxs = rawdb.ReadInboundEtxs(sl.sliceDb, block.Hash())
+		if newInboundEtxs == nil {
+			newInboundEtxs, _, err = sl.CollectNewlyConfirmedEtxs(block, block.Location())
+			if err != nil {
+				log.Trace("Error collecting newly confirmed etxs: ", "err", err)
+				// Keeping track of the number of times pending etx fails and if it crossed the retry threshold
+				// ask the sub for the pending etx/rollup data
+				val, exist := sl.pEtxRetryCache.Get(block.Hash())
+				var retry uint64
+				if exist {
+					pEtxCurrent, ok := val.(pEtxRetry)
+					if ok {
+						retry = pEtxCurrent.retries + 1
+					}
 				}
+				pEtxNew := pEtxRetry{hash: block.Hash(), retries: retry}
+				sl.pEtxRetryCache.Add(block.Hash(), pEtxNew)
+				return nil, false, false, ErrSubNotSyncedToDom
 			}
-			pEtxNew := pEtxRetry{hash: block.Hash(), retries: retry}
-			sl.pEtxRetryCache.Add(block.Hash(), pEtxNew)
-			return nil, false, false, ErrSubNotSyncedToDom
+			rawdb.WriteInboundEtxs(sl.sliceDb, block.Hash(), newInboundEtxs)
 		}
 	}
 	time5 := common.PrettyDuration(time.Since(start))
@@ -314,9 +319,8 @@ func (sl *Slice) Append(header *types.Header, domPendingHeader *types.Header, do
 			return nil, false, false, err
 		}
 
-		time9 = common.PrettyDuration(time.Since(start))
-
 	}
+	time9 = common.PrettyDuration(time.Since(start))
 	sl.updatePhCache(pendingHeaderWithTermini, true, nil, subReorg, common.NodeLocation)
 
 	var updateDom bool
@@ -584,9 +588,16 @@ func (sl *Slice) CollectNewlyConfirmedEtxs(block *types.Block, location common.L
 	subRollup := types.Transactions{}
 	var err error
 	if nodeCtx < common.ZONE_CTX {
-		subRollup, err = sl.hc.CollectSubRollup(block)
-		if err != nil {
-			return nil, nil, err
+		rollup, exists := sl.hc.subRollupCache.Get(block.Hash())
+		if !exists {
+			subRollup, err = sl.hc.CollectSubRollup(block)
+			if err != nil {
+				return nil, nil, err
+			}
+			sl.hc.subRollupCache.Add(block.Hash(), subRollup)
+		} else {
+			subRollup = rollup.(types.Transactions)
+			log.Info("Found the rollup in cache", "Hash", block.Hash(), "len", len(subRollup))
 		}
 	}
 

core/types/transaction.go

@@ -450,8 +450,7 @@ func (s Transactions) FilterToLocation(l common.Location) Transactions {
 func (s Transactions) FilterToSlice(slice common.Location, minCtx int) Transactions {
 	filteredList := Transactions{}
 	for _, tx := range s {
-		toChain := tx.To().Location()
-		if toChain.InSameSliceAs(slice) {
+		if tx.To().InSameSliceAs(slice) {
 			filteredList = append(filteredList, tx)
 		}
 	}