From e4885d771d56cde8d224c061ceb433d40da41f35 Mon Sep 17 00:00:00 2001 From: Willem Wyndham Date: Mon, 28 Aug 2023 12:28:29 -0400 Subject: [PATCH 1/2] feat(CLI): add dotenv so directories can now set CLI args (#894) --- Cargo.lock | 7 +++ cmd/crates/soroban-test/tests/it/dotenv.rs | 73 ++++++++++++++++++++++ cmd/crates/soroban-test/tests/it/main.rs | 1 + cmd/soroban-cli/Cargo.toml | 1 + cmd/soroban-cli/src/bin/main.rs | 5 +- 5 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 cmd/crates/soroban-test/tests/it/dotenv.rs diff --git a/Cargo.lock b/Cargo.lock index 8aa7fc7b..05d30f61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -741,6 +741,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "downcast-rs" version = "1.2.0" @@ -2443,6 +2449,7 @@ dependencies = [ "crate-git-revision 0.0.4", "csv", "dirs", + "dotenvy", "ed25519-dalek", "ethnum", "heck", diff --git a/cmd/crates/soroban-test/tests/it/dotenv.rs b/cmd/crates/soroban-test/tests/it/dotenv.rs new file mode 100644 index 00000000..eb7b5a68 --- /dev/null +++ b/cmd/crates/soroban-test/tests/it/dotenv.rs @@ -0,0 +1,73 @@ +use soroban_test::TestEnv; + +use crate::util::HELLO_WORLD; + +const SOROBAN_CONTRACT_ID: &str = "SOROBAN_CONTRACT_ID=1"; + +fn deploy(e: &TestEnv, id: &str) { + e.new_assert_cmd("contract") + .arg("deploy") + .arg("--wasm") + .arg(HELLO_WORLD.path()) + .arg("--id") + .arg(id) + .assert() + .success(); +} + +fn write_env_file(e: &TestEnv, contents: &str) { + let env_file = e.dir().join(".env"); + std::fs::write(&env_file, contents).unwrap(); + assert_eq!(contents, std::fs::read_to_string(env_file).unwrap()); +} + +#[test] +fn can_read_file() { + TestEnv::with_default(|e| { + deploy(e, "1"); + write_env_file(e, SOROBAN_CONTRACT_ID); + e.new_assert_cmd("contract") + .arg("invoke") + .arg("--") + .arg("hello") + .arg("--world=world") + .assert() + .stdout("[\"Hello\",\"world\"]\n") + .success(); + }); +} + +#[test] +fn current_env_not_overwritten() { + TestEnv::with_default(|e| { + deploy(e, "1"); + write_env_file(e, SOROBAN_CONTRACT_ID); + + e.new_assert_cmd("contract") + .env("SOROBAN_CONTRACT_ID", "2") + .arg("invoke") + .arg("--") + .arg("hello") + .arg("--world=world") + .assert() + .stderr("error: parsing contract spec: contract spec not found\n"); + }); +} + +#[test] +fn cli_args_have_priority() { + TestEnv::with_default(|e| { + deploy(e, "2"); + write_env_file(e, SOROBAN_CONTRACT_ID); + e.new_assert_cmd("contract") + .env("SOROBAN_CONTRACT_ID", "3") + .arg("invoke") + .arg("--id") + .arg("2") + .arg("--") + .arg("hello") + .arg("--world=world") + .assert() + .stdout("[\"Hello\",\"world\"]\n"); + }); +} diff --git a/cmd/crates/soroban-test/tests/it/main.rs b/cmd/crates/soroban-test/tests/it/main.rs index 6059caa7..71683c4f 100644 --- a/cmd/crates/soroban-test/tests/it/main.rs +++ b/cmd/crates/soroban-test/tests/it/main.rs @@ -2,6 +2,7 @@ mod arg_parsing; mod config; mod custom_types; +mod dotenv; mod invoke_sandbox; mod plugin; mod util; diff --git a/cmd/soroban-cli/Cargo.toml b/cmd/soroban-cli/Cargo.toml index c41e5663..62cdae8e 100644 --- a/cmd/soroban-cli/Cargo.toml +++ b/cmd/soroban-cli/Cargo.toml @@ -91,6 +91,7 @@ tracing-appender = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } cargo_metadata = "0.15.4" pathdiff = "0.2.1" +dotenvy = "0.15.7" # For hyper-tls [target.'cfg(unix)'.dependencies] openssl = { version = "0.10.55", features = ["vendored"] } diff --git a/cmd/soroban-cli/src/bin/main.rs b/cmd/soroban-cli/src/bin/main.rs index c8bda845..6abf501c 100644 --- a/cmd/soroban-cli/src/bin/main.rs +++ b/cmd/soroban-cli/src/bin/main.rs @@ -1,10 +1,11 @@ use clap::{CommandFactory, Parser}; -use tracing_subscriber::{fmt, EnvFilter}; - +use dotenvy::dotenv; use soroban_cli::{commands::plugin, Root}; +use tracing_subscriber::{fmt, EnvFilter}; #[tokio::main] async fn main() { + let _ = dotenv().unwrap_or_default(); let mut root = Root::try_parse().unwrap_or_else(|e| { use clap::error::ErrorKind; match e.kind() { From 0d3fbcbe5ac1d789711aac17e2779d1516e9258e Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Tue, 29 Aug 2023 01:14:54 +0200 Subject: [PATCH 2/2] soroban-rpc: Stream ledgers on initialization (#904) This halves the startup ledger memory requirements (since we don't no longer need to store all the ledgers in memory before initializing the stores). --- cmd/soroban-rpc/internal/daemon/daemon.go | 34 +++++++++---------- cmd/soroban-rpc/internal/db/ledger.go | 26 ++++++++++---- cmd/soroban-rpc/internal/db/ledger_test.go | 6 +++- cmd/soroban-rpc/internal/db/ledgerentry.go | 1 + .../ledgerbucketwindow/ledgerbucketwindow.go | 2 +- .../methods/get_latest_ledger_test.go | 4 +-- 6 files changed, 46 insertions(+), 27 deletions(-) diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index 68efba97..ad946008 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -19,6 +19,7 @@ import ( "github.com/stellar/go/ingest/ledgerbackend" supporthttp "github.com/stellar/go/support/http" supportlog "github.com/stellar/go/support/log" + "github.com/stellar/go/xdr" "github.com/stellar/soroban-tools/cmd/soroban-rpc/internal" "github.com/stellar/soroban-tools/cmd/soroban-rpc/internal/config" @@ -182,35 +183,34 @@ func MustNew(cfg *config.Config) *Daemon { cfg.TransactionLedgerRetentionWindow, ) - maxRetentionWindow := cfg.EventLedgerRetentionWindow - if cfg.TransactionLedgerRetentionWindow > maxRetentionWindow { - maxRetentionWindow = cfg.TransactionLedgerRetentionWindow - } else if cfg.EventLedgerRetentionWindow == 0 && cfg.TransactionLedgerRetentionWindow > ledgerbucketwindow.DefaultEventLedgerRetentionWindow { - maxRetentionWindow = ledgerbucketwindow.DefaultEventLedgerRetentionWindow - } - // initialize the stores using what was on the DB readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout) defer cancelReadTxMeta() - txmetas, err := db.NewLedgerReader(dbConn).GetAllLedgers(readTxMetaCtx) - if err != nil { - logger.WithError(err).Fatal("could not obtain txmeta cache from the database") - } - for _, txmeta := range txmetas { - // NOTE: We could optimize this to avoid unnecessary ingestion calls - // (len(txmetas) can be larger than the store retention windows) - // but it's probably not worth the pain. + // NOTE: We could optimize this to avoid unnecessary ingestion calls + // (the range of txmetads can be larger than the store retention windows) + // but it's probably not worth the pain. + err = db.NewLedgerReader(dbConn).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error { if err := eventStore.IngestEvents(txmeta); err != nil { - logger.WithError(err).Fatal("could initialize event memory store") + logger.WithError(err).Fatal("could not initialize event memory store") } if err := transactionStore.IngestTransactions(txmeta); err != nil { - logger.WithError(err).Fatal("could initialize transaction memory store") + logger.WithError(err).Fatal("could not initialize transaction memory store") } + return nil + }) + if err != nil { + logger.WithError(err).Fatal("could not obtain txmeta cache from the database") } onIngestionRetry := func(err error, dur time.Duration) { logger.WithError(err).Error("could not run ingestion. Retrying") } + maxRetentionWindow := cfg.EventLedgerRetentionWindow + if cfg.TransactionLedgerRetentionWindow > maxRetentionWindow { + maxRetentionWindow = cfg.TransactionLedgerRetentionWindow + } else if cfg.EventLedgerRetentionWindow == 0 && cfg.TransactionLedgerRetentionWindow > ledgerbucketwindow.DefaultEventLedgerRetentionWindow { + maxRetentionWindow = ledgerbucketwindow.DefaultEventLedgerRetentionWindow + } ingestService := ingest.NewService(ingest.Config{ Logger: logger, DB: db.NewReadWriter(dbConn, maxLedgerEntryWriteBatchSize, maxRetentionWindow), diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 2c40ff09..1b4b0aa2 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -13,9 +13,11 @@ const ( ledgerCloseMetaTableName = "ledger_close_meta" ) +type StreamLedgerFn func(xdr.LedgerCloseMeta) error + type LedgerReader interface { GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) - GetAllLedgers(ctx context.Context) ([]xdr.LedgerCloseMeta, error) + StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error } type LedgerWriter interface { @@ -30,12 +32,24 @@ func NewLedgerReader(db *DB) LedgerReader { return ledgerReader{db: db} } -// GetAllLedgers returns all ledgers in the database. -func (r ledgerReader) GetAllLedgers(ctx context.Context) ([]xdr.LedgerCloseMeta, error) { - var results []xdr.LedgerCloseMeta +// StreamAllLedgers runs f over all the ledgers in the database (until f errors or signals it's done). +func (r ledgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error { sql := sq.Select("meta").From(ledgerCloseMetaTableName).OrderBy("sequence asc") - err := r.db.Select(ctx, &results, sql) - return results, err + q, err := r.db.Query(ctx, sql) + if err != nil { + return err + } + defer q.Close() + for q.Next() { + var closeMeta xdr.LedgerCloseMeta + if err = q.Scan(&closeMeta); err != nil { + return err + } + if err = f(closeMeta); err != nil { + return err + } + } + return nil } // GetLedger fetches a single ledger from the db. diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index 906ad9ff..362f3365 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -28,7 +28,11 @@ func createLedger(ledgerSequence uint32) xdr.LedgerCloseMeta { } func assertLedgerRange(t *testing.T, reader LedgerReader, start, end uint32) { - allLedgers, err := reader.GetAllLedgers(context.Background()) + var allLedgers []xdr.LedgerCloseMeta + err := reader.StreamAllLedgers(context.Background(), func(txmeta xdr.LedgerCloseMeta) error { + allLedgers = append(allLedgers, txmeta) + return nil + }) assert.NoError(t, err) for i := start - 1; i <= end+1; i++ { ledger, exists, err := reader.GetLedger(context.Background(), i) diff --git a/cmd/soroban-rpc/internal/db/ledgerentry.go b/cmd/soroban-rpc/internal/db/ledgerentry.go index 0c650a56..6363e251 100644 --- a/cmd/soroban-rpc/internal/db/ledgerentry.go +++ b/cmd/soroban-rpc/internal/db/ledgerentry.go @@ -233,6 +233,7 @@ func (l *ledgerEntryReadTx) getRawLedgerEntries(keys ...string) (map[string]stri if err != nil { return nil, err } + defer q.Close() for q.Next() { var key, entry string if err = q.Scan(&key, &entry); err != nil { diff --git a/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go b/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go index 08b9dc9d..0d447e71 100644 --- a/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go +++ b/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go @@ -7,7 +7,7 @@ import ( // LedgerBucketWindow is a sequence of buckets associated to a ledger window. type LedgerBucketWindow[T any] struct { // buckets is a circular buffer where each cell represents - // all events occurring within a specific ledger. + // the content stored for a specific ledger. buckets []LedgerBucket[T] // start is the index of the head in the circular buffer. start uint32 diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index a57ad4fe..1beb15b0 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -54,8 +54,8 @@ func (ledgerReader *ConstantLedgerReader) GetLedger(ctx context.Context, sequenc return createLedger(sequence, expectedLatestLedgerProtocolVersion, expectedLatestLedgerHashBytes), true, nil } -func (ledgerReader *ConstantLedgerReader) GetAllLedgers(ctx context.Context) ([]xdr.LedgerCloseMeta, error) { - return []xdr.LedgerCloseMeta{createLedger(expectedLatestLedgerSequence, expectedLatestLedgerProtocolVersion, expectedLatestLedgerHashBytes)}, nil +func (ledgerReader *ConstantLedgerReader) StreamAllLedgers(ctx context.Context, f db.StreamLedgerFn) error { + return nil } func createLedger(ledgerSequence uint32, protocolVersion uint32, hash byte) xdr.LedgerCloseMeta {