Merge branch 'master' into wasmstore-rebuilder

cmd/conf/init.go

@@ -1,6 +1,8 @@
 package conf
 
 import (
+	"fmt"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -10,6 +12,8 @@ import (
 type InitConfig struct {
 	Force                    bool          `koanf:"force"`
 	Url                      string        `koanf:"url"`
+	Latest                   string        `koanf:"latest"`
+	LatestBase               string        `koanf:"latest-base"`
 	DownloadPath             string        `koanf:"download-path"`
 	DownloadPoll             time.Duration `koanf:"download-poll"`
 	DevInit                  bool          `koanf:"dev-init"`
@@ -28,6 +32,8 @@ type InitConfig struct {
 var InitConfigDefault = InitConfig{
 	Force:                    false,
 	Url:                      "",
+	Latest:                   "",
+	LatestBase:               "https://snapshot.arbitrum.foundation/",
 	DownloadPath:             "/tmp/",
 	DownloadPoll:             time.Minute,
 	DevInit:                  false,
@@ -46,6 +52,8 @@ var InitConfigDefault = InitConfig{
 func InitConfigAddOptions(prefix string, f *pflag.FlagSet) {
 	f.Bool(prefix+".force", InitConfigDefault.Force, "if true: in case database exists init code will be reexecuted and genesis block compared to database")
 	f.String(prefix+".url", InitConfigDefault.Url, "url to download initialization data - will poll if download fails")
+	f.String(prefix+".latest", InitConfigDefault.Latest, "if set, searches for the latest snapshot of the given kind "+acceptedSnapshotKindsStr)
+	f.String(prefix+".latest-base", InitConfigDefault.LatestBase, "base url used when searching for the latest")
 	f.String(prefix+".download-path", InitConfigDefault.DownloadPath, "path to save temp downloaded file")
 	f.Duration(prefix+".download-poll", InitConfigDefault.DownloadPoll, "how long to wait between polling attempts")
 	f.Bool(prefix+".dev-init", InitConfigDefault.DevInit, "init with dev data (1 account with balance) instead of file import")
@@ -65,5 +73,22 @@ func (c *InitConfig) Validate() error {
 	if c.Force && c.RecreateMissingStateFrom > 0 {
 		log.Warn("force init enabled, recreate-missing-state-from will have no effect")
 	}
+	if c.Latest != "" && !isAcceptedSnapshotKind(c.Latest) {
+		return fmt.Errorf("invalid value for latest option: \"%s\" %s", c.Latest, acceptedSnapshotKindsStr)
+	}
 	return nil
 }
+
+var (
+	acceptedSnapshotKinds    = []string{"archive", "pruned", "genesis"}
+	acceptedSnapshotKindsStr = "(accepted values: \"" + strings.Join(acceptedSnapshotKinds, "\" | \"") + "\")"
+)
+
+func isAcceptedSnapshotKind(kind string) bool {
+	for _, valid := range acceptedSnapshotKinds {
+		if kind == valid {
+			return true
+		}
+	}
+	return false
+}

cmd/nitro/init.go

@@ -13,6 +13,7 @@ import (
 	"io"
 	"math/big"
 	"net/http"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
@@ -142,9 +143,8 @@ func downloadFile(ctx context.Context, initConfig *conf.InitConfig, url string)
 	}
 }
 
-// fetchChecksum performs a GET request to the specified URL using the provided context
-// and returns the checksum as a []byte
-func fetchChecksum(ctx context.Context, url string) ([]byte, error) {
+// httpGet performs a GET request to the specified URL
+func httpGet(ctx context.Context, url string) ([]byte, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
 		return nil, fmt.Errorf("error creating request: %w", err)
@@ -164,6 +164,15 @@ func fetchChecksum(ctx context.Context, url string) ([]byte, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error reading response body: %w", err)
 	}
+	return body, nil
+}
+
+// fetchChecksum performs a GET request to the specified URL and returns the checksum
+func fetchChecksum(ctx context.Context, url string) ([]byte, error) {
+	body, err := httpGet(ctx, url)
+	if err != nil {
+		return nil, err
+	}
 	checksumStr := strings.TrimSpace(string(body))
 	checksum, err := hex.DecodeString(checksumStr)
 	if err != nil {
@@ -235,6 +244,29 @@ func joinArchive(parts []string) (string, error) {
 	return archivePath, nil
 }
 
+// setLatestSnapshotUrl sets the Url in initConfig to the latest one available on the mirror.
+func setLatestSnapshotUrl(ctx context.Context, initConfig *conf.InitConfig, chain string) error {
+	if initConfig.Latest == "" {
+		return nil
+	}
+	if initConfig.Url != "" {
+		return fmt.Errorf("cannot set latest url if url is already set")
+	}
+	baseUrl, err := url.Parse(initConfig.LatestBase)
+	if err != nil {
+		return fmt.Errorf("failed to parse latest mirror \"%s\": %w", initConfig.LatestBase, err)
+	}
+	latestDateUrl := baseUrl.JoinPath(chain, "latest-"+initConfig.Latest+".txt").String()
+	latestDateBytes, err := httpGet(ctx, latestDateUrl)
+	if err != nil {
+		return fmt.Errorf("failed to get latest snapshot at \"%s\": %w", latestDateUrl, err)
+	}
+	latestDate := strings.TrimSpace(string(latestDateBytes))
+	initConfig.Url = baseUrl.JoinPath(chain, latestDate, initConfig.Latest+".tar").String()
+	log.Info("Set latest snapshot url", "url", initConfig.Url)
+	return nil
+}
+
 func validateBlockChain(blockChain *core.BlockChain, chainConfig *params.ChainConfig) error {
 	statedb, err := blockChain.State()
 	if err != nil {
@@ -357,6 +389,10 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo
 		}
 	}
 
+	if err := setLatestSnapshotUrl(ctx, &config.Init, config.Chain.Name); err != nil {
+		return nil, nil, err
+	}
+
 	initFile, err := downloadInit(ctx, &config.Init)
 	if err != nil {
 		return nil, nil, err

cmd/nitro/init_test.go

@@ -13,6 +13,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -117,6 +118,42 @@ func TestDownloadInitInParts(t *testing.T) {
 	}
 }
 
+func TestSetLatestSnapshotUrl(t *testing.T) {
+	const (
+		chain        = "arb1"
+		snapshotKind = "archive"
+		latestDate   = "2024/21"
+		latestFile   = "latest-" + snapshotKind + ".txt"
+		dirPerm      = 0700
+		filePerm     = 0600
+	)
+
+	// Create latest file
+	serverDir := t.TempDir()
+	err := os.Mkdir(filepath.Join(serverDir, chain), dirPerm)
+	Require(t, err)
+	err = os.WriteFile(filepath.Join(serverDir, chain, latestFile), []byte(latestDate), filePerm)
+	Require(t, err)
+
+	// Start HTTP server
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	addr := "http://" + startFileServer(t, ctx, serverDir)
+
+	// Set latest snapshot URL
+	initConfig := conf.InitConfigDefault
+	initConfig.Latest = snapshotKind
+	initConfig.LatestBase = addr
+	err = setLatestSnapshotUrl(ctx, &initConfig, chain)
+	Require(t, err)
+
+	// Check url
+	want := fmt.Sprintf("%s/%s/%s/archive.tar", addr, chain, latestDate)
+	if initConfig.Url != want {
+		t.Errorf("initConfig.Url = %s; want: %s", initConfig.Url, want)
+	}
+}
+
 func startFileServer(t *testing.T, ctx context.Context, dir string) string {
 	t.Helper()
 	ln, err := net.Listen("tcp", "127.0.0.1:0")

cmd/nitro/nitro.go

@@ -966,6 +966,7 @@ func applyChainParameters(ctx context.Context, k *koanf.Koanf, chainId uint64, c
 	}
 	chainDefaults := map[string]interface{}{
 		"persistent.chain": chainInfo.ChainName,
+		"chain.name":       chainInfo.ChainName,
 		"chain.id":         chainInfo.ChainConfig.ChainID.Uint64(),
 		"parent-chain.id":  chainInfo.ParentChainId,
 	}