address review

cmd/curio/rpc/rpc.go

@@ -29,7 +29,6 @@ import (
 	"github.com/filecoin-project/curio/api/client"
 	"github.com/filecoin-project/curio/deps"
 	"github.com/filecoin-project/curio/lib/paths"
-	"github.com/filecoin-project/curio/market"
 	"github.com/filecoin-project/curio/web"
 
 	"github.com/filecoin-project/lotus/api"
@@ -158,8 +157,9 @@ func (p *CurioAPI) StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsS
 	return p.Stor.FsStat(ctx, id)
 }
 
+// this method is currently unused, might be back when we get markets into curio
 func (p *CurioAPI) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece piece.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
-	di, err := market.NewPieceIngester(ctx, p.Deps.DB, p.Deps.Chain, maddr, true, time.Minute)
+	/*di, err := market.NewPieceIngester(ctx, p.Deps.DB, p.Deps.Chain, maddr, true, time.Minute)
 	if err != nil {
 		return api.SectorOffset{}, xerrors.Errorf("failed to create a piece ingestor")
 	}
@@ -173,8 +173,8 @@ func (p *CurioAPI) AllocatePieceToSector(ctx context.Context, maddr address.Addr
 	if err != nil {
 		return api.SectorOffset{}, xerrors.Errorf("failed to start sealing the sector %d for actor %s", sector.Sector, maddr)
 	}
-
-	return sector, nil
+	*/
+	return api.SectorOffset{}, xerrors.Errorf("not implemented")
 }
 
 // Trigger shutdown

deps/config/types.go

@@ -199,12 +199,23 @@ type CurioSubsystemsConfig struct {
 	// uses all available network (or disk) bandwidth on the machine without causing bottlenecks.
 	MoveStorageMaxTasks int
 
+	// EnableUpdateEncode enables the encoding step of the SnapDeal process on this curio instance.
+	// This step involves encoding the data into the sector and computing updated TreeR (uses gpu).
 	EnableUpdateEncode bool
-	EnableUpdateProve  bool
+
+	// EnableUpdateProve enables the proving step of the SnapDeal process on this curio instance.
+	// This step generates the snark proof for the updated sector.
+	EnableUpdateProve bool
+
+	// EnableUpdateSubmit enables the submission of SnapDeal proofs to the blockchain from this curio instance.
+	// This step submits the generated proofs to the chain.
 	EnableUpdateSubmit bool
 
+	// UpdateEncodeMaxTasks sets the maximum number of concurrent SnapDeal encoding tasks that can run on this instance.
 	UpdateEncodeMaxTasks int
-	UpdateProveMaxTasks  int
+
+	// UpdateProveMaxTasks sets the maximum number of concurrent SnapDeal proving tasks that can run on this instance.
+	UpdateProveMaxTasks int
 
 	// BoostAdapters is a list of tuples of miner address and port/ip to listen for market (e.g. boost) requests.
 	// This interface is compatible with the lotus-miner RPC, implementing a subset needed for storage market operations.
@@ -407,6 +418,10 @@ type CurioIngestConfig struct {
 
 	// Maximum time an open deal sector should wait for more deal before it starts sealing
 	MaxDealWaitTime Duration
+
+	// DoSnap enables the snap deal process for deals ingested by this instance. Unlike in lotus-miner there is no
+	// fallback to porep when no sectors are available to snap into. When enabled all deals will be snap deals.
+	DoSnap bool
 }
 
 type CurioAlertingConfig struct {

documentation/en/configuration/default-curio-configuration.md

@@ -156,18 +156,31 @@ description: The default curio configuration
   # type: int
   #MoveStorageMaxTasks = 0
 
+  # EnableUpdateEncode enables the encoding step of the SnapDeal process on this curio instance.
+  # This step involves encoding the data into the sector and computing updated TreeR (uses gpu).
+  #
   # type: bool
   #EnableUpdateEncode = false
 
+  # EnableUpdateProve enables the proving step of the SnapDeal process on this curio instance.
+  # This step generates the snark proof for the updated sector.
+  #
   # type: bool
   #EnableUpdateProve = false
 
+  # EnableUpdateSubmit enables the submission of SnapDeal proofs to the blockchain from this curio instance.
+  # This step submits the generated proofs to the chain.
+  #
   # type: bool
   #EnableUpdateSubmit = false
 
+  # UpdateEncodeMaxTasks sets the maximum number of concurrent SnapDeal encoding tasks that can run on this instance.
+  #
   # type: int
   #UpdateEncodeMaxTasks = 0
 
+  # UpdateProveMaxTasks sets the maximum number of concurrent SnapDeal proving tasks that can run on this instance.
+  #
   # type: int
   #UpdateProveMaxTasks = 0
 
@@ -404,6 +417,12 @@ description: The default curio configuration
   # type: Duration
   #MaxDealWaitTime = "1h0m0s"
 
+  # DoSnap enables the snap deal process for deals ingested by this instance. Unlike in lotus-miner there is no
+  # fallback to porep when no sectors are available to snap into. When enabled all deals will be snap deals.
+  #
+  # type: bool
+  #DoSnap = false
+
 
 [Journal]
   # Events of the form: "system1:event1,system1:event2[,...]"

harmony/harmonydb/sql/20240611-snap-pipeline.sql

@@ -76,6 +76,7 @@ CREATE TABLE sectors_cc_values (
                                         PRIMARY KEY (reg_seal_proof, cur_unsealed_cid)
 );
 
+-- sectors_cc_values contains CC UnsealedCIDs for every seal proof type / sector size
 INSERT INTO sectors_cc_values (reg_seal_proof, cur_unsealed_cid) VALUES
                                                                            (0, 'baga6ea4seaqpy7usqklokfx2vxuynmupslkeutzexe2uqurdg5vhtebhxqmpqmy'),
                                                                            (1, 'baga6ea4seaqgl4u6lwmnerwdrm4iz7ag3mpwwaqtapc2fciabpooqmvjypweeha'),

lib/dealdata/dealdata.go

@@ -122,7 +122,7 @@ func getDealMetadata(ctx context.Context, db *harmonydb.DB, sc *ffi.SealCalls, s
 				PieceCID: c,
 			})
 
-			offset += abi.UnpaddedPieceSize(*p.DataRawSize)
+			offset += abi.PaddedPieceSize(p.PieceSize).Unpadded()
 
 			// make pieceReader
 			if p.DataUrl != nil {

lib/ffi/sdr_funcs.go

@@ -64,7 +64,7 @@ type storageProvider struct {
 	storageReservations *xsync.MapOf[harmonytask.TaskID, *StorageReservation]
 }
 
-func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask.TaskID, sector storiface.SectorRef, existing, allocate storiface.SectorFileType, sealing storiface.PathType) (fspaths, ids storiface.SectorPaths, release func(dontDeclace ...storiface.SectorFileType), err error) {
+func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask.TaskID, sector storiface.SectorRef, existing, allocate storiface.SectorFileType, sealing storiface.PathType) (fspaths, ids storiface.SectorPaths, release func(dontDeclare ...storiface.SectorFileType), err error) {
 	var sectorPaths, storageIDs storiface.SectorPaths
 	var releaseStorage func()
 
@@ -118,15 +118,15 @@ func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask
 
 	log.Debugf("acquired sector %d (e:%d; a:%d): %v", sector, existing, allocate, sectorPaths)
 
-	return sectorPaths, storageIDs, func(dontDeclace ...storiface.SectorFileType) {
+	return sectorPaths, storageIDs, func(dontDeclare ...storiface.SectorFileType) {
 		releaseStorage()
 
 	nextType:
 		for _, fileType := range storiface.PathTypes {
 			if fileType&allocate == 0 {
 				continue
 			}
-			for _, dont := range dontDeclace {
+			for _, dont := range dontDeclare {
 				if fileType&dont != 0 {
 					continue nextType
 				}

lib/ffi/snap_funcs.go

@@ -70,6 +70,7 @@ func (sb *SealCalls) EncodeUpdate(
 	var cleanupStagedFiles func() error
 	{
 		// hack until we do snap encode ourselves and just call into proofs for CommR
+		// https://github.com/filecoin-project/curio/issues/92
 
 		keyFile, err := os.Create(keyPath)
 		if err != nil {

lib/tarutil/systar.go

@@ -11,6 +11,11 @@ import (
 	"golang.org/x/xerrors"
 )
 
+/*
+This file contains utils for sending sector data directories (cache, update-cache) over the network. Compatible with
+lotus-miner counterpart, but extended to support more advanced processes in curio.
+*/
+
 var log = logging.Logger("tarutil") // nolint
 
 var CacheFileConstraints = map[string]int64{

market/deal_ingest_seal.go

@@ -33,6 +33,7 @@ const loopFrequency = 10 * time.Second
 
 type Ingester interface {
 	AllocatePieceToSector(ctx context.Context, maddr address.Address, piece lpiece.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error)
+	SectorStartSealing(context.Context, abi.SectorNumber) error
 }
 
 type PieceIngesterApi interface {

market/lmrpc/lmrpc.go

@@ -154,7 +154,13 @@ func forEachMarketRPC(cfg *config.CurioConfig, cb func(string, string) error) er
 func ServeCurioMarketRPC(db *harmonydb.DB, full api.Chain, maddr address.Address, conf *config.CurioConfig, listen string) error {
 	ctx := context.Background()
 
-	pin, err := cumarket.NewPieceIngesterSnap(ctx, db, full, maddr, false, time.Duration(conf.Ingest.MaxDealWaitTime))
+	var pin cumarket.Ingester
+	var err error
+	if conf.Ingest.DoSnap {
+		pin, err = cumarket.NewPieceIngesterSnap(ctx, db, full, maddr, false, time.Duration(conf.Ingest.MaxDealWaitTime))
+	} else {
+		pin, err = cumarket.NewPieceIngester(ctx, db, full, maddr, false, time.Duration(conf.Ingest.MaxDealWaitTime))
+	}
 	if err != nil {
 		return xerrors.Errorf("starting piece ingestor")
 	}

market/seal_now.go

@@ -21,6 +21,7 @@ type SealNowNodeApi interface {
 	StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (networkVersion network.Version, err error)
 }
 
+// SealNow is triggerred by the user to seal a sector immediately. Can be from the CLI or the web interface.
 func SealNow(ctx context.Context, node SealNowNodeApi, db *harmonydb.DB, act address.Address, sector abi.SectorNumber, synthetic bool) error {
 	mid, err := address.IDFromAddress(act)
 	if err != nil {

web/api/webrpc/upgrade.go

@@ -21,7 +21,6 @@ type UpgradeSector struct {
 
 func (a *WebRPC) UpgradeSectors(ctx context.Context) ([]UpgradeSector, error) {
 	var sectors []UpgradeSector
-	//err := a.deps.DB.Select(ctx, &sectors, `SELECT sp_id, sector_number FROM sectors_snap_pipeline`)
 	err := a.deps.DB.Select(ctx, &sectors, `SELECT sp_id, sector_number, task_id_encode, after_encode, task_id_prove, after_prove, task_id_submit, after_submit, task_id_move_storage, after_move_storage FROM sectors_snap_pipeline`)
 	if err != nil {
 		return nil, err

web/static/deals/pending-deals.mjs

@@ -2,10 +2,6 @@ import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/al
 import RPCCall from '/lib/jsonrpc.mjs';
 
 class PendingDeals extends LitElement {
-    static properties = {
-        data: { type: Array }
-    };
-
     constructor() {
         super();
         this.data = [];
@@ -14,7 +10,7 @@ class PendingDeals extends LitElement {
 
     async loadData() {
         this.data = await RPCCall('DealsPending');
-        super.requestUpdate();
+        this.requestUpdate();
     }
 
     async sealNow(entry) {

web/static/upgrade/upgrade-sectors.mjs → web/static/snap/upgrade-sectors.mjs

@@ -2,9 +2,6 @@ import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/al
 import RPCCall from '/lib/jsonrpc.mjs';
 
 class UpgradeSectors extends LitElement {
-    static properties = {
-        data: { type: Array }
-    };
 
     constructor() {
         super();
@@ -14,7 +11,7 @@ class UpgradeSectors extends LitElement {
 
     async loadData() {
         this.data = await RPCCall('UpgradeSectors');
-        super.requestUpdate();
+        this.requestUpdate();
     }
 
     render() {

web/static/ux/curio-ux.mjs

@@ -67,7 +67,7 @@ class CurioUX extends LitElement {
                 <a class="nav-link" href="/pipeline_porep.html">PoRep</a>
               </li>
               <li class="nav-item">
-                <a class="nav-link" href="/upgrade/">Upgrade</a>
+                <a class="nav-link" href="/snap/">Snap</a>
               </li> 
               <li>
                 <hr class="dropdown-divider">