Merge pull request #722 from SiaFoundation/chris/wallet-fund-unconfir…

…med-txns

Allow for funding transactions with unconfirmed outputs

api/bus.go

@@ -195,8 +195,9 @@ type SlabBuffer struct {
 
 // WalletFundRequest is the request type for the /wallet/fund endpoint.
 type WalletFundRequest struct {
-	Transaction types.Transaction `json:"transaction"`
-	Amount      types.Currency    `json:"amount"`
+	Transaction        types.Transaction `json:"transaction"`
+	Amount             types.Currency    `json:"amount"`
+	UseUnconfirmedTxns bool              `json:"useUnconfirmedTxns"`
 }
 
 // WalletFundResponse is the response type for the /wallet/fund endpoint.

autopilot/contractor.go

@@ -352,7 +352,7 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) (
 		if err != nil {
 			c.logger.Errorf("failed to fetch wallet, err: %v", err)
 			return false, err
-		} else if wallet.Confirmed.IsZero() {
+		} else if wallet.Confirmed.IsZero() && wallet.Unconfirmed.IsZero() {
 			c.logger.Warn("contract formations skipped, wallet is empty")
 		} else {
 			formed, err = c.runContractFormations(ctx, w, candidates, usedHosts, unusableHosts, state.cfg.Contracts.Amount-uint64(len(updatedSet)), &remaining)

bus/bus.go

@@ -81,7 +81,7 @@ type (
 	Wallet interface {
 		Address() types.Address
 		Balance() (spendable, confirmed, unconfirmed types.Currency, _ error)
-		FundTransaction(cs consensus.State, txn *types.Transaction, amount types.Currency, pool []types.Transaction) ([]types.Hash256, error)
+		FundTransaction(cs consensus.State, txn *types.Transaction, amount types.Currency, useUnconfirmedTxns bool) ([]types.Hash256, error)
 		Height() uint64
 		Redistribute(cs consensus.State, outputs int, amount, feePerByte types.Currency, pool []types.Transaction) (types.Transaction, []types.Hash256, error)
 		ReleaseInputs(txn types.Transaction)
@@ -383,7 +383,7 @@ func (b *bus) walletFundHandler(jc jape.Context) {
 		fee := b.tp.RecommendedFee().Mul64(b.cm.TipState().TransactionWeight(txn))
 		txn.MinerFees = []types.Currency{fee}
 	}
-	toSign, err := b.w.FundTransaction(b.cm.TipState(), &txn, wfr.Amount.Add(txn.MinerFees[0]), b.tp.Transactions())
+	toSign, err := b.w.FundTransaction(b.cm.TipState(), &txn, wfr.Amount.Add(txn.MinerFees[0]), wfr.UseUnconfirmedTxns)
 	if jc.Check("couldn't fund transaction", err) != nil {
 		return
 	}
@@ -467,7 +467,7 @@ func (b *bus) walletPrepareFormHandler(jc jape.Context) {
 		FileContracts: []types.FileContract{fc},
 	}
 	txn.MinerFees = []types.Currency{b.tp.RecommendedFee().Mul64(cs.TransactionWeight(txn))}
-	toSign, err := b.w.FundTransaction(cs, &txn, cost.Add(txn.MinerFees[0]), b.tp.Transactions())
+	toSign, err := b.w.FundTransaction(cs, &txn, cost.Add(txn.MinerFees[0]), true)
 	if jc.Check("couldn't fund transaction", err) != nil {
 		return
 	}
@@ -520,7 +520,7 @@ func (b *bus) walletPrepareRenewHandler(jc jape.Context) {
 	// Fund the txn. We are not signing it yet since it's not complete. The host
 	// still needs to complete it and the revision + contract are signed with
 	// the renter key by the worker.
-	toSign, err := b.w.FundTransaction(cs, &txn, cost, b.tp.Transactions())
+	toSign, err := b.w.FundTransaction(cs, &txn, cost, true)
 	if jc.Check("couldn't fund transaction", err) != nil {
 		return
 	}

bus/client/wallet.go

@@ -14,15 +14,15 @@ import (
 )
 
 // SendSiacoins is a helper method that sends siacoins to the given outputs.
-func (c *Client) SendSiacoins(ctx context.Context, scos []types.SiacoinOutput) (err error) {
+func (c *Client) SendSiacoins(ctx context.Context, scos []types.SiacoinOutput, useUnconfirmedTxns bool) (err error) {
 	var value types.Currency
 	for _, sco := range scos {
 		value = value.Add(sco.Value)
 	}
 	txn := types.Transaction{
 		SiacoinOutputs: scos,
 	}
-	toSign, parents, err := c.WalletFund(ctx, &txn, value)
+	toSign, parents, err := c.WalletFund(ctx, &txn, value, useUnconfirmedTxns)
 	if err != nil {
 		return err
 	}
@@ -51,10 +51,11 @@ func (c *Client) WalletDiscard(ctx context.Context, txn types.Transaction) error
 }
 
 // WalletFund funds txn using inputs controlled by the wallet.
-func (c *Client) WalletFund(ctx context.Context, txn *types.Transaction, amount types.Currency) ([]types.Hash256, []types.Transaction, error) {
+func (c *Client) WalletFund(ctx context.Context, txn *types.Transaction, amount types.Currency, useUnconfirmedTransactions bool) ([]types.Hash256, []types.Transaction, error) {
 	req := api.WalletFundRequest{
-		Transaction: *txn,
-		Amount:      amount,
+		Transaction:        *txn,
+		Amount:             amount,
+		UseUnconfirmedTxns: useUnconfirmedTransactions,
 	}
 	var resp api.WalletFundResponse
 	err := c.c.WithContext(ctx).POST("/wallet/fund", req, &resp)

internal/testing/cluster.go

@@ -256,13 +256,14 @@ func (c *TestCluster) UpdateAutopilotConfig(ctx context.Context, cfg api.Autopil
 }
 
 type testClusterOptions struct {
-	dbName        string
-	dir           string
-	funding       *bool
-	hosts         int
-	logger        *zap.Logger
-	uploadPacking bool
-	walletKey     *types.PrivateKey
+	dbName               string
+	dir                  string
+	funding              *bool
+	hosts                int
+	logger               *zap.Logger
+	uploadPacking        bool
+	skipSettingAutopilot bool
+	walletKey            *types.PrivateKey
 
 	autopilotCfg      *node.AutopilotConfig
 	autopilotSettings *api.AutopilotConfig
@@ -505,10 +506,12 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster {
 	tt.OK(busClient.SetContractSet(context.Background(), testContractSet, []types.FileContractID{}))
 
 	// Update the autopilot to use test settings
-	tt.OK(busClient.UpdateAutopilot(context.Background(), api.Autopilot{
-		ID:     apCfg.ID,
-		Config: apSettings,
-	}))
+	if !opts.skipSettingAutopilot {
+		tt.OK(busClient.UpdateAutopilot(context.Background(), api.Autopilot{
+			ID:     apCfg.ID,
+			Config: apSettings,
+		}))
+	}
 
 	// Update the bus settings.
 	tt.OK(busClient.UpdateSetting(context.Background(), api.SettingGouging, testGougingSettings))
@@ -740,7 +743,7 @@ func (c *TestCluster) AddHost(h *Host) {
 			Address: h.WalletAddress(),
 		})
 	}
-	c.tt.OK(c.Bus.SendSiacoins(context.Background(), scos))
+	c.tt.OK(c.Bus.SendSiacoins(context.Background(), scos, true))
 
 	// Mine transaction.
 	c.MineBlocks(1)

internal/testing/cluster_test.go

@@ -25,6 +25,7 @@ import (
 	"go.sia.tech/renterd/api"
 	"go.sia.tech/renterd/hostdb"
 	"go.sia.tech/renterd/object"
+	"go.sia.tech/renterd/wallet"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"lukechampine.com/frand"
@@ -1629,7 +1630,7 @@ func TestWallet(t *testing.T) {
 		},
 		MinerFees: []types.Currency{minerFee},
 	}
-	toSign, parents, err := b.WalletFund(context.Background(), &txn, txn.SiacoinOutputs[0].Value)
+	toSign, parents, err := b.WalletFund(context.Background(), &txn, txn.SiacoinOutputs[0].Value, false)
 	tt.OK(err)
 	err = b.WalletSign(context.Background(), &txn, toSign, types.CoveredFields{WholeTransaction: true})
 	tt.OK(err)
@@ -1950,3 +1951,126 @@ func TestMultipartUploads(t *testing.T) {
 		t.Fatal("unexpected data:", cmp.Diff(data, expectedData))
 	}
 }
+
+func TestWalletSendUnconfirmed(t *testing.T) {
+	cluster := newTestCluster(t, clusterOptsDefault)
+	defer cluster.Shutdown()
+	b := cluster.Bus
+	tt := cluster.tt
+
+	wr, err := b.Wallet(context.Background())
+	tt.OK(err)
+
+	// check balance
+	if !wr.Unconfirmed.IsZero() {
+		t.Fatal("wallet should not have unconfirmed balance")
+	} else if wr.Confirmed.IsZero() {
+		t.Fatal("wallet should have confirmed balance")
+	}
+
+	// send the full balance back to the weallet
+	toSend := wr.Confirmed.Sub(types.Siacoins(1).Div64(100)) // leave some for the fee
+	tt.OK(b.SendSiacoins(context.Background(), []types.SiacoinOutput{
+		{
+			Address: wr.Address,
+			Value:   toSend,
+		},
+	}, false))
+
+	// the unconfirmed balance should have changed to slightly more than toSend
+	// since we paid a fee
+	wr, err = b.Wallet(context.Background())
+	tt.OK(err)
+
+	if wr.Unconfirmed.Cmp(toSend) < 0 || wr.Unconfirmed.Add(types.Siacoins(1)).Cmp(toSend) < 0 {
+		t.Fatal("wallet should have unconfirmed balance")
+	}
+	fmt.Println(wr.Confirmed, wr.Unconfirmed)
+
+	// try again - this should fail
+	err = b.SendSiacoins(context.Background(), []types.SiacoinOutput{
+		{
+			Address: wr.Address,
+			Value:   toSend,
+		},
+	}, false)
+	tt.AssertIs(err, wallet.ErrInsufficientBalance)
+
+	// try again - this time using unconfirmed transactions
+	tt.OK(b.SendSiacoins(context.Background(), []types.SiacoinOutput{
+		{
+			Address: wr.Address,
+			Value:   toSend,
+		},
+	}, true))
+
+	// the unconfirmed balance should be almost the same
+	wr, err = b.Wallet(context.Background())
+	tt.OK(err)
+
+	if wr.Unconfirmed.Cmp(toSend) < 0 || wr.Unconfirmed.Add(types.Siacoins(1)).Cmp(toSend) < 0 {
+		t.Fatal("wallet should have unconfirmed balance")
+	}
+	fmt.Println(wr.Confirmed, wr.Unconfirmed)
+
+	// mine a block, this should confirm the transactions
+	cluster.MineBlocks(1)
+	tt.Retry(100, time.Millisecond, func() error {
+		wr, err = b.Wallet(context.Background())
+		tt.OK(err)
+
+		if !wr.Unconfirmed.IsZero() {
+			return fmt.Errorf("wallet should not have unconfirmed balance")
+		} else if wr.Confirmed.Cmp(toSend) < 0 || wr.Confirmed.Add(types.Siacoins(1)).Cmp(toSend) < 0 {
+			return fmt.Errorf("wallet should have almost the same confirmed balance as in the beginning")
+		}
+		return nil
+	})
+}
+
+func TestWalletFormUnconfirmed(t *testing.T) {
+	// New cluster with autopilot disabled
+	cfg := clusterOptsDefault
+	cfg.skipSettingAutopilot = true
+	cluster := newTestCluster(t, cfg)
+	defer cluster.Shutdown()
+	b := cluster.Bus
+	tt := cluster.tt
+
+	// Add a host.
+	cluster.AddHosts(1)
+
+	// Send the full balance back to the wallet to make sure it's all
+	// unconfirmed.
+	wr, err := b.Wallet(context.Background())
+	tt.OK(err)
+	tt.OK(b.SendSiacoins(context.Background(), []types.SiacoinOutput{
+		{
+			Address: wr.Address,
+			Value:   wr.Confirmed.Sub(types.Siacoins(1).Div64(100)), // leave some for the fee
+		},
+	}, false))
+
+	// There should be hardly any money in the wallet.
+	wr, err = b.Wallet(context.Background())
+	tt.OK(err)
+	if wr.Confirmed.Sub(wr.Unconfirmed).Cmp(types.Siacoins(1).Div64(100)) > 0 {
+		t.Fatal("wallet should have hardly any confirmed balance")
+	}
+
+	// There shouldn't be any contracts at this point.
+	contracts, err := b.Contracts(context.Background())
+	tt.OK(err)
+	if len(contracts) != 0 {
+		t.Fatal("expected 0 contracts", len(contracts))
+	}
+
+	// Enable autopilot by setting it.
+	cluster.UpdateAutopilotConfig(context.Background(), testAutopilotConfig)
+
+	// Wait for a contract to form.
+	contractsFormed := cluster.WaitForContracts()
+	if len(contractsFormed) != 1 {
+		t.Fatal("expected 1 contract", len(contracts))
+	}
+}

wallet/wallet.go

@@ -168,7 +168,9 @@ func (w *SingleAddressWallet) Balance() (spendable, confirmed, unconfirmed types
 		confirmed = confirmed.Add(sce.Value)
 	}
 	for _, sco := range w.tpoolUtxos {
-		unconfirmed = unconfirmed.Add(sco.Value)
+		if !w.isOutputUsed(sco.ID) {
+			unconfirmed = unconfirmed.Add(sco.Value)
+		}
 	}
 	return
 }
@@ -204,33 +206,34 @@ func (w *SingleAddressWallet) Transactions(before, since time.Time, offset, limi
 // FundTransaction adds siacoin inputs worth at least the requested amount to
 // the provided transaction. A change output is also added, if necessary. The
 // inputs will not be available to future calls to FundTransaction unless
-// ReleaseInputs is called.
-func (w *SingleAddressWallet) FundTransaction(cs consensus.State, txn *types.Transaction, amount types.Currency, pool []types.Transaction) ([]types.Hash256, error) {
+// ReleaseInputs is called or enough time has passed.
+func (w *SingleAddressWallet) FundTransaction(cs consensus.State, txn *types.Transaction, amount types.Currency, useUnconfirmedTxns bool) ([]types.Hash256, error) {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	if amount.IsZero() {
 		return nil, nil
 	}
 
-	// avoid reusing any inputs currently in the transaction pool
-	inPool := make(map[types.Hash256]bool)
-	for _, ptxn := range pool {
-		for _, in := range ptxn.SiacoinInputs {
-			inPool[types.Hash256(in.ParentID)] = true
-		}
-	}
-
+	// fetch all unspent siacoin elements
 	utxos, err := w.store.UnspentSiacoinElements(false)
 	if err != nil {
 		return nil, err
 	}
+
 	// choose outputs randomly
 	frand.Shuffle(len(utxos), reflect.Swapper(utxos))
 
+	// add all unconfirmed outputs to the end of the slice as a last resort
+	if useUnconfirmedTxns {
+		for _, sco := range w.tpoolUtxos {
+			utxos = append(utxos, sco)
+		}
+	}
+
 	var outputSum types.Currency
 	var fundingElements []SiacoinElement
 	for _, sce := range utxos {
-		if w.isOutputUsed(sce.ID) || inPool[sce.ID] || cs.Index.Height < sce.MaturityHeight {
+		if w.isOutputUsed(sce.ID) || w.tpoolSpent[types.SiacoinOutputID(sce.ID)] || cs.Index.Height < sce.MaturityHeight {
 			continue
 		}
 		fundingElements = append(fundingElements, sce)
@@ -393,11 +396,12 @@ func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amou
 }
 
 func (w *SingleAddressWallet) isOutputUsed(id types.Hash256) bool {
+	inPool := w.tpoolSpent[types.SiacoinOutputID(id)]
 	lastUsed := w.lastUsed[id]
 	if w.usedUTXOExpiry == 0 {
-		return !lastUsed.IsZero()
+		return !lastUsed.IsZero() || inPool
 	}
-	return time.Since(lastUsed) <= w.usedUTXOExpiry
+	return time.Since(lastUsed) <= w.usedUTXOExpiry || inPool
 }
 
 // ReceiveUpdatedUnconfirmedTransactions implements modules.TransactionPoolSubscriber.

worker/worker.go

@@ -166,7 +166,7 @@ type Bus interface {
 	FinishUpload(ctx context.Context, uID api.UploadID) error
 
 	WalletDiscard(ctx context.Context, txn types.Transaction) error
-	WalletFund(ctx context.Context, txn *types.Transaction, amount types.Currency) ([]types.Hash256, []types.Transaction, error)
+	WalletFund(ctx context.Context, txn *types.Transaction, amount types.Currency, useUnconfirmedTxns bool) ([]types.Hash256, []types.Transaction, error)
 	WalletPrepareForm(ctx context.Context, renterAddress types.Address, renterKey types.PublicKey, renterFunds, hostCollateral types.Currency, hostKey types.PublicKey, hostSettings rhpv2.HostSettings, endHeight uint64) (txns []types.Transaction, err error)
 	WalletPrepareRenew(ctx context.Context, revision types.FileContractRevision, hostAddress, renterAddress types.Address, renterKey types.PrivateKey, renterFunds, newCollateral types.Currency, pt rhpv3.HostPriceTable, endHeight, windowSize uint64) (api.WalletPrepareRenewResponse, error)
 	WalletSign(ctx context.Context, txn *types.Transaction, toSign []types.Hash256, cf types.CoveredFields) error
@@ -592,7 +592,7 @@ func (w *worker) rhpBroadcastHandler(jc jape.Context) {
 	}
 	// Fund the txn. We pass 0 here since we only need the wallet to fund
 	// the fee.
-	toSign, parents, err := w.bus.WalletFund(ctx, &txn, types.ZeroCurrency)
+	toSign, parents, err := w.bus.WalletFund(ctx, &txn, types.ZeroCurrency, true)
 	if jc.Check("failed to fund transaction", err) != nil {
 		return
 	}