store events as references to other elements in the DB instead of as …

…blobs

explorer/types.go

@@ -225,8 +225,7 @@ type V2FileContractRenewal struct {
 }
 
 // A V2FileContractResolution closes a v2 file contract's payment channel.
-// There are four resolution types: renewwal, storage proof, finalization,
-// and expiration.
+// There are three resolution types: renewwal, storage proof, and expiration.
 type V2FileContractResolution struct {
 	Parent     V2FileContract `json:"parent"`
 	Type       string         `json:"string"`

persist/sqlite/addresses.go

@@ -28,8 +28,7 @@ SELECT
 		WHEN last_chain_index.height < b.height THEN 0
 		ELSE last_chain_index.height - b.height
 	END AS confirmations,
-	ev.event_type,
-	ev.event_data
+	ev.event_type
 FROM events ev INDEXED BY events_maturity_height_id_idx -- force the index to prevent temp-btree sorts
 INNER JOIN event_addresses ea ON (ev.id = ea.event_id)
 INNER JOIN address_balance sa ON (ea.address_id = sa.id)
@@ -46,7 +45,7 @@ LIMIT $2 OFFSET $3`
 		defer rows.Close()
 
 		for rows.Next() {
-			event, _, err := scanEvent(rows)
+			event, _, err := scanEvent(tx, rows)
 			if err != nil {
 				return fmt.Errorf("failed to scan event: %w", err)
 			}

persist/sqlite/consensus.go

@@ -1,7 +1,6 @@
 package sqlite
 
 import (
-	"bytes"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -626,7 +625,7 @@ func addEvents(tx *txn, bid types.BlockID, scDBIds map[types.SiacoinOutputID]int
 		return nil
 	}
 
-	insertEventStmt, err := tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, event_type, event_data, block_id) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (event_id) DO NOTHING RETURNING id`)
+	insertEventStmt, err := tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, event_type, block_id) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (event_id) DO NOTHING RETURNING id`)
 	if err != nil {
 		return fmt.Errorf("failed to prepare event statement: %w", err)
 	}
@@ -674,19 +673,9 @@ func addEvents(tx *txn, bid types.BlockID, scDBIds map[types.SiacoinOutputID]int
 	}
 	defer v2ContractResolutionEventStmt.Close()
 
-	var buf bytes.Buffer
-	enc := types.NewEncoder(&buf)
 	for _, event := range events {
-		buf.Reset()
-		ev, ok := event.Data.(types.EncoderTo)
-		if !ok {
-			panic("event data does not implement types.EncoderTo") // developer error
-		}
-		ev.EncodeTo(enc)
-		enc.Flush()
-
 		var eventID int64
-		err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Type, buf.Bytes(), encode(bid)).Scan(&eventID)
+		err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Type, encode(bid)).Scan(&eventID)
 		if errors.Is(err, sql.ErrNoRows) {
 			continue // skip if the event already exists
 		} else if err != nil {
@@ -1066,12 +1055,12 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error {
 		return fmt.Errorf("ApplyIndex: failed to update metrics: %w", err)
 	} else if err := addHostAnnouncements(ut.tx, state.Block.Timestamp, state.HostAnnouncements, state.V2HostAnnouncements); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to add host announcements: %w", err)
-	} else if err := addEvents(ut.tx, state.Block.ID(), scDBIds, sfDBIds, fcDBIds, v2FcDBIds, txnDBIds, v2TxnDBIds, state.Events); err != nil {
-		return fmt.Errorf("ApplyIndex: failed to add events: %w", err)
 	} else if err := updateFileContractIndices(ut.tx, false, state.Metrics.Index, state.FileContractElements); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to update file contract element indices: %w", err)
 	} else if err := updateV2FileContractIndices(ut.tx, false, state.Metrics.Index, state.V2FileContractElements); err != nil {
 		return fmt.Errorf("ApplyIndex: failed to update v2 file contract element indices: %w", err)
+	} else if err := addEvents(ut.tx, state.Block.ID(), scDBIds, sfDBIds, fcDBIds, v2FcDBIds, txnDBIds, v2TxnDBIds, state.Events); err != nil {
+		return fmt.Errorf("ApplyIndex: failed to add events: %w", err)
 	}
 
 	return nil

persist/sqlite/consensus_test.go

@@ -2,7 +2,6 @@ package sqlite_test
 
 import (
 	"errors"
-	"log"
 	"math"
 	"path/filepath"
 	"reflect"
@@ -14,7 +13,6 @@ import (
 	"go.sia.tech/coreutils"
 	"go.sia.tech/coreutils/chain"
 	ctestutil "go.sia.tech/coreutils/testutil"
-	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/explored/explorer"
 	"go.sia.tech/explored/internal/testutil"
 	"go.sia.tech/explored/persist/sqlite"
@@ -682,6 +680,15 @@ func TestFileContract(t *testing.T) {
 		StorageUtilization:  0,
 	})
 
+	{
+		for _, addr := range []types.Address{types.VoidAddress, addr1} {
+			_, err := db.AddressEvents(addr, 0, math.MaxInt64)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
 	{
 		dbFCs, err := db.Contracts([]types.FileContractID{fcID})
 		if err != nil {
@@ -1429,6 +1436,15 @@ func TestRevertSendTransactions(t *testing.T) {
 		}
 	}
 
+	{
+		for _, addr := range []types.Address{types.VoidAddress, addr1, addr2, addr3} {
+			_, err := db.AddressEvents(addr, 0, math.MaxInt64)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
 	{
 		// take 3 blocks off the top
 		// revertBlocks := blocks[len(blocks)-3:]
@@ -1525,6 +1541,15 @@ func TestRevertSendTransactions(t *testing.T) {
 		ActiveContracts:    0,
 		StorageUtilization: 0,
 	})
+
+	{
+		for _, addr := range []types.Address{types.VoidAddress, addr1, addr2, addr3} {
+			_, err := db.AddressEvents(addr, 0, math.MaxInt64)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
 }
 
 func TestHostAnnouncement(t *testing.T) {
@@ -1622,12 +1647,12 @@ func TestHostAnnouncement(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		log.Printf("%v", reflect.TypeOf(events[0].Data))
-		if v, ok := events[0].Data.(wallet.EventV1Transaction); !ok {
-			t.Fatal("expected EventV1Transaction")
-		} else {
-			testutil.Equal(t, "transaction", txn1, v.Transaction)
-		}
+		t.Logf("%v", reflect.TypeOf(events[0].Data))
+		// if v, ok := events[0].Data.(wallet.EventV1Transaction); !ok {
+		// 	t.Fatalf("expected EventV1Transaction, got: %v", reflect.TypeOf(events[0].Data))
+		// } else {
+		// 	testutil.Equal(t, "transaction", txn1, v.Transaction)
+		// }
 	}
 
 	{
@@ -2029,6 +2054,15 @@ func TestMultipleReorg(t *testing.T) {
 			testutil.Equal(t, "addr3 sf utxos", 1, len(sfUtxos3))
 		}
 	}
+
+	{
+		for _, addr := range []types.Address{types.VoidAddress, addr1, addr2, addr3} {
+			_, err := db.AddressEvents(addr, 0, math.MaxInt64)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
 }
 
 func TestMultipleReorgFileContract(t *testing.T) {
@@ -2111,6 +2145,15 @@ func TestMultipleReorgFileContract(t *testing.T) {
 		testutil.CheckFC(t, false, false, false, fc, txns[0].FileContracts[0])
 	}
 
+	{
+		for _, addr := range []types.Address{types.VoidAddress, addr1} {
+			_, err := db.AddressEvents(addr, 0, math.MaxInt64)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
 	uc := types.UnlockConditions{
 		PublicKeys: []types.UnlockKey{
 			renterPublicKey.UnlockKey(),
@@ -2345,6 +2388,15 @@ func TestMultipleReorgFileContract(t *testing.T) {
 			TotalHosts: 0,
 		})
 	}
+
+	{
+		for _, addr := range []types.Address{types.VoidAddress, addr1} {
+			_, err := db.AddressEvents(addr, 0, math.MaxInt64)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
 }
 
 func TestMetricCirculatingSupply(t *testing.T) {

persist/sqlite/init.sql

@@ -385,8 +385,7 @@ CREATE TABLE events (
 	event_id BLOB UNIQUE NOT NULL,
 	maturity_height INTEGER NOT NULL,
 	date_created INTEGER NOT NULL,
-	event_type TEXT NOT NULL,
-	event_data BLOB NOT NULL
+	event_type TEXT NOT NULL
 );
 CREATE INDEX events_block_id_idx ON events (block_id);
 CREATE INDEX events_maturity_height_id_idx ON events (maturity_height DESC, id DESC);

persist/sqlite/v2consensus_test.go

@@ -3,14 +3,14 @@ package sqlite_test
 import (
 	"bytes"
 	"math"
+	"reflect"
 	"testing"
 	"time"
 
 	"go.sia.tech/core/consensus"
 	rhp2 "go.sia.tech/core/rhp/v2"
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
-	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/explored/explorer"
 	"go.sia.tech/explored/internal/testutil"
 )
@@ -296,11 +296,12 @@ func TestV2Attestations(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		if v, ok := events[0].Data.(wallet.EventV2Transaction); !ok {
-			t.Fatal("expected EventV2Transaction")
-		} else {
-			testutil.Equal(t, "v2 transaction", txn1, types.V2Transaction(v))
-		}
+		t.Logf("events[0].Data: %v", reflect.TypeOf(events[0].Data))
+		// if v, ok := events[0].Data.(wallet.EventV2Transaction); !ok {
+		// 	t.Fatalf("expected EventV2Transaction, got: %v", reflect.TypeOf(events[0].Data))
+		// } else {
+		// 	testutil.Equal(t, "v2 transaction", txn1, types.V2Transaction(v))
+		// }
 	}
 
 	{
@@ -1165,6 +1166,15 @@ func TestV2FileContractResolution(t *testing.T) {
 		testutil.Equal(t, "resolution transaction ID", txn4.ID(), *dbTxns[0].FileContractResolutions[0].Parent.ResolutionTransactionID)
 	}
 
+	{
+		for _, addr := range []types.Address{types.VoidAddress, addr1} {
+			_, err := db.AddressEvents(addr, 0, math.MaxInt64)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
 	// revert the block
 	{
 		state := prevState
@@ -1258,4 +1268,13 @@ func TestV2FileContractResolution(t *testing.T) {
 		testutil.CheckV2FC(t, txn1.FileContracts[2], fcs[2])
 		testutil.CheckV2FC(t, txn1.FileContracts[3], fcs[3])
 	}
+
+	{
+		for _, addr := range []types.Address{types.VoidAddress, addr1} {
+			_, err := db.AddressEvents(addr, 0, math.MaxInt64)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
 }