Merge pull request #2359 from OffchainLabs/remove-ipfs-das

Remove IPFS das factory logic and stubs
OffchainLabs · Jun 4, 2024 · b1edb9b · b1edb9b
2 parents 4146eff + 6356632
commit b1edb9b
Show file tree

Hide file tree

Showing 16 changed files with 38 additions and 925 deletions.
diff --git a/das/das.go b/das/das.go
@@ -41,11 +41,9 @@ type DataAvailabilityConfig struct {
 	LocalCache CacheConfig `koanf:"local-cache"`
 	RedisCache RedisConfig `koanf:"redis-cache"`
 
-	LocalDBStorage     LocalDBStorageConfig     `koanf:"local-db-storage"`
-	LocalFileStorage   LocalFileStorageConfig   `koanf:"local-file-storage"`
-	S3Storage          S3StorageServiceConfig   `koanf:"s3-storage"`
-	IpfsStorage        IpfsStorageServiceConfig `koanf:"ipfs-storage"`
-	RegularSyncStorage RegularSyncStorageConfig `koanf:"regular-sync-storage"`
+	LocalDBStorage   LocalDBStorageConfig   `koanf:"local-db-storage"`
+	LocalFileStorage LocalFileStorageConfig `koanf:"local-file-storage"`
+	S3Storage        S3StorageServiceConfig `koanf:"s3-storage"`
 
 	Key KeyConfig `koanf:"key"`
 
@@ -68,7 +66,6 @@ var DefaultDataAvailabilityConfig = DataAvailabilityConfig{
 	RPCAggregator:                 DefaultAggregatorConfig,
 	ParentChainConnectionAttempts: 15,
 	PanicOnError:                  false,
-	IpfsStorage:                   DefaultIpfsStorageServiceConfig,
 }
 
 func OptionalAddressFromString(s string) (*common.Address, error) {
@@ -115,7 +112,6 @@ func dataAvailabilityConfigAddOptions(prefix string, f *flag.FlagSet, r role) {
 		LocalDBStorageConfigAddOptions(prefix+".local-db-storage", f)
 		LocalFileStorageConfigAddOptions(prefix+".local-file-storage", f)
 		S3ConfigAddOptions(prefix+".s3-storage", f)
-		RegularSyncStorageConfigAddOptions(prefix+".regular-sync-storage", f)
 
 		// Key config for storage
 		KeyConfigAddOptions(prefix+".key", f)
@@ -129,7 +125,6 @@ func dataAvailabilityConfigAddOptions(prefix string, f *flag.FlagSet, r role) {
 	}
 
 	// Both the Nitro node and daserver can use these options.
-	IpfsStorageServiceConfigAddOptions(prefix+".ipfs-storage", f)
 	RestfulClientAggregatorConfigAddOptions(prefix+".rest-aggregator", f)
 
 	f.String(prefix+".parent-chain-node-url", DefaultDataAvailabilityConfig.ParentChainNodeURL, "URL for parent chain node, only used in standalone daserver; when running as part of a node that node's L1 configuration is used")

diff --git a/das/das_test.go b/das/das_test.go
@@ -47,9 +47,7 @@ func testDASStoreRetrieveMultipleInstances(t *testing.T, storageType string) {
 		ParentChainNodeURL: "none",
 	}
 
-	var syncFromStorageServicesFirst []*IterableStorageService
-	var syncToStorageServicesFirst []StorageService
-	storageService, lifecycleManager, err := CreatePersistentStorageService(firstCtx, &config, &syncFromStorageServicesFirst, &syncToStorageServicesFirst)
+	storageService, lifecycleManager, err := CreatePersistentStorageService(firstCtx, &config)
 	Require(t, err)
 	defer lifecycleManager.StopAndWaitUntil(time.Second)
 	daWriter, err := NewSignAfterStoreDASWriter(firstCtx, config, storageService)
@@ -77,9 +75,7 @@ func testDASStoreRetrieveMultipleInstances(t *testing.T, storageType string) {
 	secondCtx, secondCancel := context.WithCancel(context.Background())
 	defer secondCancel()
 
-	var syncFromStorageServicesSecond []*IterableStorageService
-	var syncToStorageServicesSecond []StorageService
-	storageService2, lifecycleManager, err := CreatePersistentStorageService(secondCtx, &config, &syncFromStorageServicesSecond, &syncToStorageServicesSecond)
+	storageService2, lifecycleManager, err := CreatePersistentStorageService(secondCtx, &config)
 	Require(t, err)
 	defer lifecycleManager.StopAndWaitUntil(time.Second)
 	var daReader2 DataAvailabilityServiceReader = storageService2
@@ -140,9 +136,7 @@ func testDASMissingMessage(t *testing.T, storageType string) {
 		ParentChainNodeURL: "none",
 	}
 
-	var syncFromStorageServices []*IterableStorageService
-	var syncToStorageServices []StorageService
-	storageService, lifecycleManager, err := CreatePersistentStorageService(ctx, &config, &syncFromStorageServices, &syncToStorageServices)
+	storageService, lifecycleManager, err := CreatePersistentStorageService(ctx, &config)
 	Require(t, err)
 	defer lifecycleManager.StopAndWaitUntil(time.Second)
 	daWriter, err := NewSignAfterStoreDASWriter(ctx, config, storageService)

diff --git a/das/db_storage_service.go b/das/db_storage_service.go
@@ -20,11 +20,9 @@ import (
 )
 
 type LocalDBStorageConfig struct {
-	Enable                 bool   `koanf:"enable"`
-	DataDir                string `koanf:"data-dir"`
-	DiscardAfterTimeout    bool   `koanf:"discard-after-timeout"`
-	SyncFromStorageService bool   `koanf:"sync-from-storage-service"`
-	SyncToStorageService   bool   `koanf:"sync-to-storage-service"`
+	Enable              bool   `koanf:"enable"`
+	DataDir             string `koanf:"data-dir"`
+	DiscardAfterTimeout bool   `koanf:"discard-after-timeout"`
 
 	// BadgerDB options
 	NumMemtables            int   `koanf:"num-memtables"`
@@ -38,11 +36,9 @@ type LocalDBStorageConfig struct {
 var badgerDefaultOptions = badger.DefaultOptions("")
 
 var DefaultLocalDBStorageConfig = LocalDBStorageConfig{
-	Enable:                 false,
-	DataDir:                "",
-	DiscardAfterTimeout:    false,
-	SyncFromStorageService: false,
-	SyncToStorageService:   false,
+	Enable:              false,
+	DataDir:             "",
+	DiscardAfterTimeout: false,
 
 	NumMemtables:            badgerDefaultOptions.NumMemtables,
 	NumLevelZeroTables:      badgerDefaultOptions.NumLevelZeroTables,
@@ -56,8 +52,6 @@ func LocalDBStorageConfigAddOptions(prefix string, f *flag.FlagSet) {
 	f.Bool(prefix+".enable", DefaultLocalDBStorageConfig.Enable, "enable storage/retrieval of sequencer batch data from a database on the local filesystem")
 	f.String(prefix+".data-dir", DefaultLocalDBStorageConfig.DataDir, "directory in which to store the database")
 	f.Bool(prefix+".discard-after-timeout", DefaultLocalDBStorageConfig.DiscardAfterTimeout, "discard data after its expiry timeout")
-	f.Bool(prefix+".sync-from-storage-service", DefaultLocalDBStorageConfig.SyncFromStorageService, "enable db storage to be used as a source for regular sync storage")
-	f.Bool(prefix+".sync-to-storage-service", DefaultLocalDBStorageConfig.SyncToStorageService, "enable db storage to be used as a sink for regular sync storage")
 
 	f.Int(prefix+".num-memtables", DefaultLocalDBStorageConfig.NumMemtables, "BadgerDB option: sets the maximum number of tables to keep in memory before stalling")
 	f.Int(prefix+".num-level-zero-tables", DefaultLocalDBStorageConfig.NumLevelZeroTables, "BadgerDB option: sets the maximum number of Level 0 tables before compaction starts")
@@ -158,13 +152,6 @@ func (dbs *DBStorageService) Put(ctx context.Context, data []byte, timeout uint6
 	})
 }
 
-func (dbs *DBStorageService) putKeyValue(ctx context.Context, key common.Hash, value []byte) error {
-	return dbs.db.Update(func(txn *badger.Txn) error {
-		e := badger.NewEntry(key.Bytes(), value)
-		return txn.SetEntry(e)
-	})
-}
-
 func (dbs *DBStorageService) Sync(ctx context.Context) error {
 	return dbs.db.Sync()
 }

diff --git a/das/factory.go b/das/factory.go
@@ -22,8 +22,6 @@ import (
 func CreatePersistentStorageService(
 	ctx context.Context,
 	config *DataAvailabilityConfig,
-	syncFromStorageServices *[]*IterableStorageService,
-	syncToStorageServices *[]StorageService,
 ) (StorageService, *LifecycleManager, error) {
 	storageServices := make([]StorageService, 0, 10)
 	var lifecycleManager LifecycleManager
@@ -32,14 +30,6 @@ func CreatePersistentStorageService(
 		if err != nil {
 			return nil, nil, err
 		}
-		if config.LocalDBStorage.SyncFromStorageService {
-			iterableStorageService := NewIterableStorageService(ConvertStorageServiceToIterationCompatibleStorageService(s))
-			*syncFromStorageServices = append(*syncFromStorageServices, iterableStorageService)
-			s = iterableStorageService
-		}
-		if config.LocalDBStorage.SyncToStorageService {
-			*syncToStorageServices = append(*syncToStorageServices, s)
-		}
 		lifecycleManager.Register(s)
 		storageServices = append(storageServices, s)
 	}
@@ -49,14 +39,6 @@ func CreatePersistentStorageService(
 		if err != nil {
 			return nil, nil, err
 		}
-		if config.LocalFileStorage.SyncFromStorageService {
-			iterableStorageService := NewIterableStorageService(ConvertStorageServiceToIterationCompatibleStorageService(s))
-			*syncFromStorageServices = append(*syncFromStorageServices, iterableStorageService)
-			s = iterableStorageService
-		}
-		if config.LocalFileStorage.SyncToStorageService {
-			*syncToStorageServices = append(*syncToStorageServices, s)
-		}
 		lifecycleManager.Register(s)
 		storageServices = append(storageServices, s)
 	}
@@ -67,23 +49,6 @@ func CreatePersistentStorageService(
 			return nil, nil, err
 		}
 		lifecycleManager.Register(s)
-		if config.S3Storage.SyncFromStorageService {
-			iterableStorageService := NewIterableStorageService(ConvertStorageServiceToIterationCompatibleStorageService(s))
-			*syncFromStorageServices = append(*syncFromStorageServices, iterableStorageService)
-			s = iterableStorageService
-		}
-		if config.S3Storage.SyncToStorageService {
-			*syncToStorageServices = append(*syncToStorageServices, s)
-		}
-		storageServices = append(storageServices, s)
-	}
-
-	if config.IpfsStorage.Enable {
-		s, err := NewIpfsStorageService(ctx, config.IpfsStorage)
-		if err != nil {
-			return nil, nil, err
-		}
-		lifecycleManager.Register(s)
 		storageServices = append(storageServices, s)
 	}
 
@@ -105,8 +70,6 @@ func WrapStorageWithCache(
 	ctx context.Context,
 	config *DataAvailabilityConfig,
 	storageService StorageService,
-	syncFromStorageServices *[]*IterableStorageService,
-	syncToStorageServices *[]StorageService,
 	lifecycleManager *LifecycleManager) (StorageService, error) {
 	if storageService == nil {
 		return nil, nil
@@ -120,14 +83,6 @@ func WrapStorageWithCache(
 		if err != nil {
 			return nil, err
 		}
-		if config.RedisCache.SyncFromStorageService {
-			iterableStorageService := NewIterableStorageService(ConvertStorageServiceToIterationCompatibleStorageService(storageService))
-			*syncFromStorageServices = append(*syncFromStorageServices, iterableStorageService)
-			storageService = iterableStorageService
-		}
-		if config.RedisCache.SyncToStorageService {
-			*syncToStorageServices = append(*syncToStorageServices, storageService)
-		}
 	}
 	if config.LocalCache.Enable {
 		storageService = NewCacheStorageService(config.LocalCache, storageService)
@@ -151,10 +106,6 @@ func CreateBatchPosterDAS(
 	if !config.RPCAggregator.Enable || !config.RestAggregator.Enable {
 		return nil, nil, nil, errors.New("--node.data-availability.rpc-aggregator.enable and rest-aggregator.enable must be set when running a Batch Poster in AnyTrust mode")
 	}
-
-	if config.IpfsStorage.Enable {
-		return nil, nil, nil, errors.New("--node.data-availability.ipfs-storage.enable may not be set when running a Nitro AnyTrust node in Batch Poster mode")
-	}
 	// Done checking config requirements
 
 	var daWriter DataAvailabilityServiceWriter
@@ -192,20 +143,17 @@ func CreateDAComponentsForDaserver(
 	// Check config requirements
 	if !config.LocalDBStorage.Enable &&
 		!config.LocalFileStorage.Enable &&
-		!config.S3Storage.Enable &&
-		!config.IpfsStorage.Enable {
-		return nil, nil, nil, nil, nil, errors.New("At least one of --data-availability.(local-db-storage|local-file-storage|s3-storage|ipfs-storage) must be enabled.")
+		!config.S3Storage.Enable {
+		return nil, nil, nil, nil, nil, errors.New("At least one of --data-availability.(local-db-storage|local-file-storage|s3-storage) must be enabled.")
 	}
 	// Done checking config requirements
 
-	var syncFromStorageServices []*IterableStorageService
-	var syncToStorageServices []StorageService
-	storageService, dasLifecycleManager, err := CreatePersistentStorageService(ctx, config, &syncFromStorageServices, &syncToStorageServices)
+	storageService, dasLifecycleManager, err := CreatePersistentStorageService(ctx, config)
 	if err != nil {
 		return nil, nil, nil, nil, nil, err
 	}
 
-	storageService, err = WrapStorageWithCache(ctx, config, storageService, &syncFromStorageServices, &syncToStorageServices, dasLifecycleManager)
+	storageService, err = WrapStorageWithCache(ctx, config, storageService, dasLifecycleManager)
 	if err != nil {
 		return nil, nil, nil, nil, nil, err
 	}
@@ -285,11 +233,6 @@ func CreateDAComponentsForDaserver(
 		}
 	}
 
-	if config.RegularSyncStorage.Enable && len(syncFromStorageServices) != 0 && len(syncToStorageServices) != 0 {
-		regularlySyncStorage := NewRegularlySyncStorage(syncFromStorageServices, syncToStorageServices, config.RegularSyncStorage)
-		regularlySyncStorage.Start(ctx)
-	}
-
 	if seqInboxAddress != nil {
 		daReader, err = NewChainFetchReader(daReader, (*l1Reader).Client(), *seqInboxAddress)
 		if err != nil {
@@ -315,48 +258,22 @@ func CreateDAReaderForNode(
 		return nil, nil, errors.New("node.data-availability.rpc-aggregator is only for Batch Poster mode")
 	}
 
-	if !config.RestAggregator.Enable && !config.IpfsStorage.Enable {
-		return nil, nil, fmt.Errorf("--node.data-availability.enable was set but neither of --node.data-availability.(rest-aggregator|ipfs-storage) were enabled. When running a Nitro Anytrust node in non-Batch Poster mode, some way to get the batch data is required.")
-	}
-
-	if config.RestAggregator.SyncToStorage.Eager {
-		return nil, nil, errors.New("--node.data-availability.rest-aggregator.sync-to-storage.eager can't be used with a Nitro node, only lazy syncing can be used.")
+	if !config.RestAggregator.Enable {
+		return nil, nil, fmt.Errorf("--node.data-availability.enable was set but not --node.data-availability.rest-aggregator. When running a Nitro Anytrust node in non-Batch Poster mode, some way to get the batch data is required.")
 	}
 	// Done checking config requirements
 
-	storageService, dasLifecycleManager, err := CreatePersistentStorageService(ctx, config, nil, nil)
-	if err != nil {
-		return nil, nil, err
-	}
-
+	var lifecycleManager LifecycleManager
 	var daReader DataAvailabilityServiceReader
 	if config.RestAggregator.Enable {
 		var restAgg *SimpleDASReaderAggregator
-		restAgg, err = NewRestfulClientAggregator(ctx, &config.RestAggregator)
+		restAgg, err := NewRestfulClientAggregator(ctx, &config.RestAggregator)
 		if err != nil {
 			return nil, nil, err
 		}
 		restAgg.Start(ctx)
-		dasLifecycleManager.Register(restAgg)
-
-		if storageService != nil {
-			syncConf := &config.RestAggregator.SyncToStorage
-			var retentionPeriodSeconds uint64
-			if uint64(syncConf.RetentionPeriod) == math.MaxUint64 {
-				retentionPeriodSeconds = math.MaxUint64
-			} else {
-				retentionPeriodSeconds = uint64(syncConf.RetentionPeriod.Seconds())
-			}
-
-			// This falls back to REST and updates the local IPFS repo if the data is found.
-			storageService = NewFallbackStorageService(storageService, restAgg, restAgg,
-				retentionPeriodSeconds, syncConf.IgnoreWriteErrors, true)
-			dasLifecycleManager.Register(storageService)
-
-			daReader = storageService
-		} else {
-			daReader = restAgg
-		}
+		lifecycleManager.Register(restAgg)
+		daReader = restAgg
 	}
 
 	if seqInboxAddress != nil {
@@ -370,5 +287,5 @@ func CreateDAReaderForNode(
 		}
 	}
 
-	return daReader, dasLifecycleManager, nil
+	return daReader, &lifecycleManager, nil
 }