Merge remote-tracking branch 'origin/main' into cli/restore-preamble

cmd/soroban-rpc/internal/daemon/daemon.go

@@ -28,6 +28,7 @@ import (
 	"github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/ledgerbucketwindow"
 	"github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/preflight"
 	"github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/transactions"
+	"github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/util"
 )
 
 const (
@@ -281,19 +282,20 @@ func (d *Daemon) Run() {
 		"addr":    d.server.Addr,
 	}).Info("starting Soroban JSON RPC server")
 
-	go func() {
+	panicGroup := util.UnrecoverablePanicGroup.Log(d.logger)
+	panicGroup.Go(func() {
 		if err := d.server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
 			// Error starting or closing listener:
 			d.logger.WithError(err).Fatal("soroban JSON RPC server encountered fatal error")
 		}
-	}()
+	})
 
 	if d.adminServer != nil {
-		go func() {
+		panicGroup.Go(func() {
 			if err := d.adminServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
 				d.logger.WithError(err).Error("soroban admin server encountered fatal error")
 			}
-		}()
+		})
 	}
 
 	// Shutdown gracefully when we receive an interrupt signal.

cmd/soroban-rpc/internal/db/ledgerentry.go

@@ -190,7 +190,7 @@ func (l *ledgerEntryReadTx) GetLatestLedgerSequence() (uint32, error) {
 		return l.cachedLatestLedgerSeq, nil
 	}
 	latestLedgerSeq, err := getLatestLedgerSequence(context.Background(), l.tx)
-	if err != nil {
+	if err == nil {
 		l.cachedLatestLedgerSeq = latestLedgerSeq
 	}
 	return latestLedgerSeq, err

cmd/soroban-rpc/internal/ingest/service.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/daemon/interfaces"
 	"github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/db"
+	"github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/util"
 
 	"github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/events"
 	"github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/transactions"
@@ -81,7 +82,8 @@ func NewService(cfg Config) *Service {
 		ledgerStatsMetric:       ledgerStatsMetric,
 	}
 	service.wg.Add(1)
-	go func() {
+	panicGroup := util.UnrecoverablePanicGroup.Log(cfg.Logger)
+	panicGroup.Go(func() {
 		defer service.wg.Done()
 		// Retry running ingestion every second for 5 seconds.
 		constantBackoff := backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5)
@@ -101,7 +103,7 @@ func NewService(cfg Config) *Service {
 		if err != nil && !errors.Is(err, context.Canceled) {
 			service.logger.WithError(err).Fatal("could not run ingestion")
 		}
-	}()
+	})
 	return service
 }
 
@@ -170,9 +172,10 @@ func (s *Service) maybeFillEntriesFromCheckpoint(ctx context.Context, archive hi
 		// DB is empty, let's fill it from the History Archive, using the latest available checkpoint
 		// Do it in parallel with the upcoming captive core preparation to save time
 		s.logger.Infof("found an empty database, creating ledger-entry baseline from the most recent checkpoint (%d). This can take up to 30 minutes, depending on the network", checkpointLedger)
-		go func() {
+		panicGroup := util.UnrecoverablePanicGroup.Log(s.logger)
+		panicGroup.Go(func() {
 			checkPointFillErr <- s.fillEntriesFromCheckpoint(ctx, archive, checkpointLedger)
-		}()
+		})
 		return checkpointLedger + 1, checkPointFillErr, nil
 	} else if err != nil {
 		return 0, checkPointFillErr, err

cmd/soroban-rpc/internal/methods/simulate_transaction.go

@@ -2,6 +2,7 @@ package methods
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 
 	"github.com/creachadair/jrpc2"
@@ -121,25 +122,25 @@ func NewSimulateTransactionHandler(logger *log.Entry, ledgerEntryReader db.Ledge
 		}
 
 		var results []SimulateHostFunctionResult
-		if result.Result != "" {
+		if len(result.Result) != 0 {
 			results = append(results, SimulateHostFunctionResult{
-				XDR:  result.Result,
-				Auth: result.Auth,
+				XDR:  base64.StdEncoding.EncodeToString(result.Result),
+				Auth: base64EncodeSlice(result.Auth),
 			})
 		}
 		restorePreable := RestorePreamble{}
-		if result.PreRestoreTransactionData != "" {
+		if len(result.PreRestoreTransactionData) != 0 {
 			restorePreable = RestorePreamble{
-				TransactionData: result.PreRestoreTransactionData,
+				TransactionData: base64.StdEncoding.EncodeToString(result.PreRestoreTransactionData),
 				MinResourceFee:  result.PreRestoreMinFee,
 			}
 		}
 
 		return SimulateTransactionResponse{
 			Error:           result.Error,
 			Results:         results,
-			Events:          result.Events,
-			TransactionData: result.TransactionData,
+			Events:          base64EncodeSlice(result.Events),
+			TransactionData: base64.StdEncoding.EncodeToString(result.TransactionData),
 			MinResourceFee:  result.MinFee,
 			Cost: SimulateTransactionCost{
 				CPUInstructions: result.CPUInstructions,
@@ -151,6 +152,14 @@ func NewSimulateTransactionHandler(logger *log.Entry, ledgerEntryReader db.Ledge
 	})
 }
 
+func base64EncodeSlice(in [][]byte) []string {
+	result := make([]string, len(in))
+	for i, v := range in {
+		result[i] = base64.StdEncoding.EncodeToString(v)
+	}
+	return result
+}
+
 func getBucketListSize(ctx context.Context, ledgerReader db.LedgerReader, latestLedger uint32) (uint64, error) {
 	// obtain bucket size
 	var closeMeta, ok, err = ledgerReader.GetLedger(ctx, latestLedger)

cmd/soroban-rpc/internal/preflight/preflight.go

@@ -39,31 +39,35 @@ type snapshotSourceHandle struct {
 // It's used by the Rust preflight code to obtain ledger entries.
 //
 //export SnapshotSourceGet
-func SnapshotSourceGet(handle C.uintptr_t, cLedgerKey *C.char, includeExpired C.int) *C.char {
+func SnapshotSourceGet(handle C.uintptr_t, cLedgerKey C.xdr_t, includeExpired C.int) C.xdr_t {
 	h := cgo.Handle(handle).Value().(snapshotSourceHandle)
-	ledgerKeyB64 := C.GoString(cLedgerKey)
+	ledgerKeyXDR := GoXDR(cLedgerKey)
 	var ledgerKey xdr.LedgerKey
-	if err := xdr.SafeUnmarshalBase64(ledgerKeyB64, &ledgerKey); err != nil {
+	if err := xdr.SafeUnmarshal(ledgerKeyXDR, &ledgerKey); err != nil {
 		panic(err)
 	}
 	present, entry, err := h.readTx.GetLedgerEntry(ledgerKey, includeExpired != 0)
 	if err != nil {
 		h.logger.WithError(err).Error("SnapshotSourceGet(): GetLedgerEntry() failed")
-		return nil
+		return C.xdr_t{}
 	}
 	if !present {
-		return nil
+		return C.xdr_t{}
 	}
-	out, err := xdr.MarshalBase64(entry)
+	out, err := entry.MarshalBinary()
 	if err != nil {
 		panic(err)
 	}
-	return C.CString(out)
+
+	return C.xdr_t{
+		xdr: (*C.uchar)(C.CBytes(out)),
+		len: C.size_t(len(out)),
+	}
 }
 
-//export FreeGoCString
-func FreeGoCString(str *C.char) {
-	C.free(unsafe.Pointer(str))
+//export FreeGoXDR
+func FreeGoXDR(xdr C.xdr_t) {
+	C.free(unsafe.Pointer(xdr.xdr))
 }
 
 type PreflightParameters struct {
@@ -78,31 +82,33 @@ type PreflightParameters struct {
 
 type Preflight struct {
 	Error                     string
-	Events                    []string // DiagnosticEvents XDR in base64
-	TransactionData           string   // SorobanTransactionData XDR in base64
+	Events                    [][]byte // DiagnosticEvents XDR
+	TransactionData           []byte   // SorobanTransactionData XDR
 	MinFee                    int64
-	Result                    string   // XDR SCVal in base64
-	Auth                      []string // SorobanAuthorizationEntrys XDR in base64
+	Result                    []byte   // XDR SCVal in base64
+	Auth                      [][]byte // SorobanAuthorizationEntries XDR
 	CPUInstructions           uint64
 	MemoryBytes               uint64
-	PreRestoreTransactionData string // SorobanTransactionData XDR in base64
+	PreRestoreTransactionData []byte // SorobanTransactionData XDR
 	PreRestoreMinFee          int64
 }
 
-// GoNullTerminatedStringSlice transforms a C NULL-terminated char** array to a Go string slice
-func GoNullTerminatedStringSlice(str **C.char) []string {
-	var result []string
-	if str != nil {
-		// CGo doesn't have an easy way to do pointer arithmetic so,
-		// we are better off transforming the memory buffer into a large slice
-		// and finding the NULL termination after that
-		for _, a := range unsafe.Slice(str, 1<<20) {
-			if a == nil {
-				// we found the ending nil
-				break
-			}
-			result = append(result, C.GoString(a))
-		}
+func CXDR(xdr []byte) C.xdr_t {
+	return C.xdr_t{
+		xdr: (*C.uchar)(C.CBytes(xdr)),
+		len: C.size_t(len(xdr)),
+	}
+}
+
+func GoXDR(xdr C.xdr_t) []byte {
+	return C.GoBytes(unsafe.Pointer(xdr.xdr), C.int(xdr.len))
+}
+
+func GoXDRVector(xdrVector C.xdr_vector_t) [][]byte {
+	result := make([][]byte, xdrVector.len)
+	inputSlice := unsafe.Slice(xdrVector.array, xdrVector.len)
+	for i, v := range inputSlice {
+		result[i] = GoXDR(v)
 	}
 	return result
 }
@@ -119,16 +125,16 @@ func GetPreflight(ctx context.Context, params PreflightParameters) (Preflight, e
 }
 
 func getFootprintExpirationPreflight(params PreflightParameters) (Preflight, error) {
-	opBodyB64, err := xdr.MarshalBase64(params.OpBody)
+	opBodyXDR, err := params.OpBody.MarshalBinary()
 	if err != nil {
 		return Preflight{}, err
 	}
-	opBodyCString := C.CString(opBodyB64)
-	footprintB64, err := xdr.MarshalBase64(params.Footprint)
+	opBodyCXDR := CXDR(opBodyXDR)
+	footprintXDR, err := params.Footprint.MarshalBinary()
 	if err != nil {
 		return Preflight{}, err
 	}
-	footprintCString := C.CString(footprintB64)
+	footprintCXDR := CXDR(footprintXDR)
 	handle := cgo.NewHandle(snapshotSourceHandle{params.LedgerEntryReadTx, params.Logger})
 	defer handle.Delete()
 
@@ -140,13 +146,13 @@ func getFootprintExpirationPreflight(params PreflightParameters) (Preflight, err
 	res := C.preflight_footprint_expiration_op(
 		C.uintptr_t(handle),
 		C.uint64_t(params.BucketListSize),
-		opBodyCString,
-		footprintCString,
+		opBodyCXDR,
+		footprintCXDR,
 		C.uint32_t(simulationLedgerSeq),
 	)
 
-	C.free(unsafe.Pointer(opBodyCString))
-	C.free(unsafe.Pointer(footprintCString))
+	FreeGoXDR(opBodyCXDR)
+	FreeGoXDR(footprintCXDR)
 
 	return GoPreflight(res), nil
 }
@@ -164,15 +170,16 @@ func getSimulationLedgerSeq(readTx db.LedgerEntryReadTx) (uint32, error) {
 }
 
 func getInvokeHostFunctionPreflight(params PreflightParameters) (Preflight, error) {
-	invokeHostFunctionB64, err := xdr.MarshalBase64(params.OpBody.MustInvokeHostFunctionOp())
+	invokeHostFunctionXDR, err := params.OpBody.MustInvokeHostFunctionOp().MarshalBinary()
 	if err != nil {
 		return Preflight{}, err
 	}
-	invokeHostFunctionCString := C.CString(invokeHostFunctionB64)
-	sourceAccountB64, err := xdr.MarshalBase64(params.SourceAccount)
+	invokeHostFunctionCXDR := CXDR(invokeHostFunctionXDR)
+	sourceAccountXDR, err := params.SourceAccount.MarshalBinary()
 	if err != nil {
 		return Preflight{}, err
 	}
+	sourceAccountCXDR := CXDR(sourceAccountXDR)
 
 	hasConfig, stateExpirationConfig, err := params.LedgerEntryReadTx.GetLedgerEntry(xdr.LedgerKey{
 		Type: xdr.LedgerEntryTypeConfigSetting,
@@ -193,7 +200,7 @@ func getInvokeHostFunctionPreflight(params PreflightParameters) (Preflight, erro
 	}
 
 	stateExpiration := stateExpirationConfig.Data.MustConfigSetting().MustStateExpirationSettings()
-	li := C.CLedgerInfo{
+	li := C.ledger_info_t{
 		network_passphrase: C.CString(params.NetworkPassphrase),
 		sequence_number:    C.uint32_t(simulationLedgerSeq),
 		protocol_version:   20,
@@ -206,35 +213,34 @@ func getInvokeHostFunctionPreflight(params PreflightParameters) (Preflight, erro
 		auto_bump_ledgers:               C.uint(stateExpiration.AutoBumpLedgers),
 	}
 
-	sourceAccountCString := C.CString(sourceAccountB64)
 	handle := cgo.NewHandle(snapshotSourceHandle{params.LedgerEntryReadTx, params.Logger})
 	defer handle.Delete()
 	res := C.preflight_invoke_hf_op(
 		C.uintptr_t(handle),
 		C.uint64_t(params.BucketListSize),
-		invokeHostFunctionCString,
-		sourceAccountCString,
+		invokeHostFunctionCXDR,
+		sourceAccountCXDR,
 		li,
 	)
-	C.free(unsafe.Pointer(invokeHostFunctionCString))
-	C.free(unsafe.Pointer(sourceAccountCString))
+	FreeGoXDR(invokeHostFunctionCXDR)
+	FreeGoXDR(sourceAccountCXDR)
 
 	return GoPreflight(res), nil
 }
 
-func GoPreflight(result *C.CPreflightResult) Preflight {
+func GoPreflight(result *C.preflight_result_t) Preflight {
 	defer C.free_preflight_result(result)
 
 	preflight := Preflight{
 		Error:                     C.GoString(result.error),
-		Events:                    GoNullTerminatedStringSlice(result.events),
-		TransactionData:           C.GoString(result.transaction_data),
+		Events:                    GoXDRVector(result.events),
+		TransactionData:           GoXDR(result.transaction_data),
 		MinFee:                    int64(result.min_fee),
-		Result:                    C.GoString(result.result),
-		Auth:                      GoNullTerminatedStringSlice(result.auth),
+		Result:                    GoXDR(result.result),
+		Auth:                      GoXDRVector(result.auth),
 		CPUInstructions:           uint64(result.cpu_instructions),
 		MemoryBytes:               uint64(result.memory_bytes),
-		PreRestoreTransactionData: C.GoString(result.pre_restore_transaction_data),
+		PreRestoreTransactionData: GoXDR(result.pre_restore_transaction_data),
 		PreRestoreMinFee:          int64(result.pre_restore_min_fee),
 	}
 	return preflight

cmd/soroban-rpc/internal/preflight/preflight_test.go

@@ -362,6 +362,8 @@ func benchmark(b *testing.B, config benchmarkConfig) {
 			disableCache: config.useDB.disableCache,
 		}
 	}
+
+	b.ResetTimer()
 	b.StopTimer()
 	for i := 0; i < b.N; i++ {
 		params := getPreflightParameters(b, dbConfig)