From 17adede9cb8e091555a45db23fd9f819c6215536 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 28 Oct 2024 16:13:05 +0100 Subject: [PATCH 01/45] Test adding nextest --- .github/workflows/ci.yml | 27 +++++++++++++++---------- bin/e2e-test-client/src/tests/script.rs | 6 +++--- tests/tests/poa.rs | 1 - 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 943f36c810d..fa5825c7c09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,20 +118,20 @@ jobs: args: --all-features --workspace --no-deps - command: make args: check --locked - - command: test + - command: nextest args: --workspace - - command: test - args: --all-features --workspace - - command: test - args: -p fuel-core --no-default-features - - command: test - args: -p fuel-core --lib executor --features wasm-executor + - command: nextest + args: run --retries 10 --all-features --workspace + - command: nextest + args: run --retries 10 -p fuel-core --no-default-features + - command: nextest + args: run --retries 10 -p fuel-core --lib executor --features wasm-executor env: FUEL_ALWAYS_USE_WASM=true - - command: test - args: -p fuel-core-client --no-default-features - - command: test - args: -p fuel-core-chain-config --no-default-features + - command: nextest + args: run --retries 10 -p fuel-core-client --no-default-features + - command: nextest + args: run --retries 10 -p fuel-core-chain-config --no-default-features # Don't split this command; this is a workaround. # We need to run `cargo check` first to fetch the locked dependencies # for `fuel-core 0.26.0`(because of the bug with `--offline` @@ -177,6 +177,11 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.command }}-${{ matrix.args }}-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - uses: actions-rs/cargo@v1 + if: ${{ matrix.command == 'nextest' }} + with: + command: install + args: cargo-nextest --locked - name: ${{ matrix.command }} ${{ matrix.args }} run: ${{ matrix.env }} cargo ${{ matrix.command }} ${{ matrix.args }} - uses: FuelLabs/.github/.github/actions/slack-notify-template@master diff --git a/bin/e2e-test-client/src/tests/script.rs b/bin/e2e-test-client/src/tests/script.rs index a14268627e0..3c5b8a26f83 100644 --- a/bin/e2e-test-client/src/tests/script.rs +++ b/bin/e2e-test-client/src/tests/script.rs @@ -137,7 +137,7 @@ pub async fn run_contract_large_state(ctx: &TestContext) -> Result<(), Failed> { if result?.is_none() { let deployment_request = ctx.bob.deploy_contract(contract_config, salt); - timeout(Duration::from_secs(20), deployment_request).await??; + timeout(Duration::from_secs(90), deployment_request).await??; } _dry_runs(ctx, &[dry_run], 100, DryRunResult::MayFail).await @@ -199,9 +199,9 @@ async fn _dry_runs( }); } - // All queries should be resolved for 60 seconds. + // All queries should be resolved for 90 seconds. let queries = - tokio::time::timeout(Duration::from_secs(60), futures::future::join_all(queries)) + tokio::time::timeout(Duration::from_secs(90), futures::future::join_all(queries)) .await?; let chain_info = ctx.alice.client.chain_info().await?; diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index 0f4f5a4fb97..566c5759eb2 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -259,7 +259,6 @@ mod p2p { // Then starts second_producer that uses the first one as a reserved peer. // second_producer should not produce blocks while the first one is producing // after the first_producer stops, second_producer should start producing blocks - #[ignore = "seems to be flaky, issue: https://github.com/FuelLabs/fuel-core/issues/2351"] #[tokio::test(flavor = "multi_thread")] async fn test_poa_multiple_producers() { const SYNC_TIMEOUT: u64 = 30; From 42f543945a00543e75abd878b6582cb28d98bd82 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 28 Oct 2024 16:34:48 +0100 Subject: [PATCH 02/45] Remove nextest addition --- .github/workflows/ci.yml | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa5825c7c09..943f36c810d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,20 +118,20 @@ jobs: args: --all-features --workspace --no-deps - command: make args: check --locked - - command: nextest + - command: test args: --workspace - - command: nextest - args: run --retries 10 --all-features --workspace - - command: nextest - args: run --retries 10 -p fuel-core --no-default-features - - command: nextest - args: run --retries 10 -p fuel-core --lib executor --features wasm-executor + - command: test + args: --all-features --workspace + - command: test + args: -p fuel-core --no-default-features + - command: test + args: -p fuel-core --lib executor --features wasm-executor env: FUEL_ALWAYS_USE_WASM=true - - command: nextest - args: run --retries 10 -p fuel-core-client --no-default-features - - command: nextest - args: run --retries 10 -p fuel-core-chain-config --no-default-features + - command: test + args: -p fuel-core-client --no-default-features + - command: test + args: -p fuel-core-chain-config --no-default-features # Don't split this command; this is a workaround. # We need to run `cargo check` first to fetch the locked dependencies # for `fuel-core 0.26.0`(because of the bug with `--offline` @@ -177,11 +177,6 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.command }}-${{ matrix.args }}-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - uses: actions-rs/cargo@v1 - if: ${{ matrix.command == 'nextest' }} - with: - command: install - args: cargo-nextest --locked - name: ${{ matrix.command }} ${{ matrix.args }} run: ${{ matrix.env }} cargo ${{ matrix.command }} ${{ matrix.args }} - uses: FuelLabs/.github/.github/actions/slack-notify-template@master From 0b63735449bd01a6bfdd6ac803a66f7f6be9eaa0 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 28 Oct 2024 16:48:44 +0100 Subject: [PATCH 03/45] Readd nextest without retried to add timeout --- .config/nextest.toml | 2 ++ .github/workflows/ci.yml | 29 +++++++++++++++++------------ 2 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000000..73484acfdb3 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,2 @@ +[profile.default] +slow-timeout = { period = "60s", terminate-after = 5 } \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 943f36c810d..61d31fe121f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,20 +118,20 @@ jobs: args: --all-features --workspace --no-deps - command: make args: check --locked - - command: test - args: --workspace - - command: test - args: --all-features --workspace - - command: test - args: -p fuel-core --no-default-features - - command: test - args: -p fuel-core --lib executor --features wasm-executor + - command: nextest + args: run --workspace + - command: nextest + args: run --all-features --workspace + - command: nextest + args: run -p fuel-core --no-default-features + - command: nextest + args: run -p fuel-core --lib executor --features wasm-executor env: FUEL_ALWAYS_USE_WASM=true - - command: test - args: -p fuel-core-client --no-default-features - - command: test - args: -p fuel-core-chain-config --no-default-features + - command: nextest + args: run -p fuel-core-client --no-default-features + - command: nextest + args: run -p fuel-core-chain-config --no-default-features # Don't split this command; this is a workaround. # We need to run `cargo check` first to fetch the locked dependencies # for `fuel-core 0.26.0`(because of the bug with `--offline` @@ -177,6 +177,11 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.command }}-${{ matrix.args }}-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - uses: actions-rs/cargo@v1 + if: ${{ matrix.command == 'nextest' }} + with: + command: install + args: cargo-nextest --locked - name: ${{ matrix.command }} ${{ matrix.args }} run: ${{ matrix.env }} cargo ${{ matrix.command }} ${{ matrix.args }} - uses: FuelLabs/.github/.github/actions/slack-notify-template@master From f9bb547a768674b36c063a91244c4000a784de4d Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 28 Oct 2024 17:18:21 +0100 Subject: [PATCH 04/45] Improve robustness backpressure tests --- crates/services/sync/src/import/back_pressure_tests.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/services/sync/src/import/back_pressure_tests.rs b/crates/services/sync/src/import/back_pressure_tests.rs index 7974890015f..039e9bb4ef3 100644 --- a/crates/services/sync/src/import/back_pressure_tests.rs +++ b/crates/services/sync/src/import/back_pressure_tests.rs @@ -29,7 +29,7 @@ struct Input { )] #[test_case( Input { - headers: Duration::from_millis(10), + headers: Duration::from_millis(100), ..Default::default() }, State::new(None, 0), @@ -42,7 +42,7 @@ struct Input { )] #[test_case( Input { - headers: Duration::from_millis(10), + headers: Duration::from_millis(100), ..Default::default() }, State::new(None, 1000), @@ -55,7 +55,7 @@ struct Input { )] #[test_case( Input { - transactions: Duration::from_millis(10), + transactions: Duration::from_millis(100), ..Default::default() }, State::new(None, 1000), @@ -68,7 +68,7 @@ struct Input { )] #[test_case( Input { - consensus: Duration::from_millis(10), + consensus: Duration::from_millis(100), ..Default::default() }, State::new(None, 1000), @@ -81,7 +81,7 @@ struct Input { )] #[test_case( Input { - executes: Duration::from_millis(10), + executes: Duration::from_millis(100), ..Default::default() }, State::new(None, 1000), From e96b25e51d8323234d4b8b43debda9f12dd5777b Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 28 Oct 2024 17:20:35 +0100 Subject: [PATCH 05/45] change timing back pressure --- crates/services/sync/src/import/back_pressure_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/services/sync/src/import/back_pressure_tests.rs b/crates/services/sync/src/import/back_pressure_tests.rs index 039e9bb4ef3..9610d552dba 100644 --- a/crates/services/sync/src/import/back_pressure_tests.rs +++ b/crates/services/sync/src/import/back_pressure_tests.rs @@ -81,7 +81,7 @@ struct Input { )] #[test_case( Input { - executes: Duration::from_millis(100), + executes: Duration::from_millis(20), ..Default::default() }, State::new(None, 1000), From 8ded514467226f1e5a865d5dd65edac5c2a19f23 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 28 Oct 2024 17:28:12 +0100 Subject: [PATCH 06/45] Remove timeout in poa test --- tests/tests/poa.rs | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index 566c5759eb2..dc6ab306ef1 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -253,6 +253,7 @@ mod p2p { fuel_tx::Input, fuel_types::Address, }; + use tracing::info; use std::time::Duration; // Starts first_producer which creates some blocks @@ -261,9 +262,13 @@ mod p2p { // after the first_producer stops, second_producer should start producing blocks #[tokio::test(flavor = "multi_thread")] async fn test_poa_multiple_producers() { - const SYNC_TIMEOUT: u64 = 30; - const TIME_UNTIL_SYNCED: u64 = SYNC_TIMEOUT + 10; + const TIME_UNTIL_SYNCED: u64 = 2; + // Tracing layer with debug + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .init(); + info!("Starting test_poa_multiple_producers"); let mut rng = StdRng::seed_from_u64(2222); // Create a producer and a validator that share the same key pair. @@ -311,12 +316,7 @@ mod p2p { // Start the second producer after 3 blocks. // The second producer should synchronize 3 blocks produced by the first producer. let second_producer = make_node(second_producer_config, vec![]).await; - tokio::time::timeout( - Duration::from_secs(SYNC_TIMEOUT), - second_producer.wait_for_blocks(3, false /* is_local */), - ) - .await - .expect("The second should sync with the first"); + second_producer.wait_for_blocks(3, false /* is_local */).await; let start_time = tokio::time::Instant::now(); // Stop the first producer. @@ -347,12 +347,7 @@ mod p2p { // it should sync remotely 5 blocks. let first_producer = make_node(make_node_config("First Producer reborn"), vec![]).await; - tokio::time::timeout( - Duration::from_secs(SYNC_TIMEOUT), - first_producer.wait_for_blocks(5, false /* is_local */), - ) - .await - .expect("The first should reborn and sync with the second"); + first_producer.wait_for_blocks(5, false /* is_local */).await; } fn update_signing_key(config: &mut Config, key: Address) { From e3ad9c6403a79e095bd28194791b65e085a68e19 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 28 Oct 2024 18:01:54 +0100 Subject: [PATCH 07/45] Readd timeout and fix ci --- .github/workflows/ci.yml | 1 + tests/tests/poa.rs | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61d31fe121f..1b24063422f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,6 +182,7 @@ jobs: with: command: install args: cargo-nextest --locked + continue-on-error: true - name: ${{ matrix.command }} ${{ matrix.args }} run: ${{ matrix.env }} cargo ${{ matrix.command }} ${{ matrix.args }} - uses: FuelLabs/.github/.github/actions/slack-notify-template@master diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index dc6ab306ef1..4d7bc356dce 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -262,11 +262,12 @@ mod p2p { // after the first_producer stops, second_producer should start producing blocks #[tokio::test(flavor = "multi_thread")] async fn test_poa_multiple_producers() { - const TIME_UNTIL_SYNCED: u64 = 2; // Tracing layer with debug tracing_subscriber::fmt() .with_max_level(tracing::Level::INFO) .init(); + const SYNC_TIMEOUT: u64 = 5; + const TIME_UNTIL_SYNCED: u64 = SYNC_TIMEOUT + 5; info!("Starting test_poa_multiple_producers"); let mut rng = StdRng::seed_from_u64(2222); @@ -316,7 +317,12 @@ mod p2p { // Start the second producer after 3 blocks. // The second producer should synchronize 3 blocks produced by the first producer. let second_producer = make_node(second_producer_config, vec![]).await; - second_producer.wait_for_blocks(3, false /* is_local */).await; + tokio::time::timeout( + Duration::from_secs(SYNC_TIMEOUT), + second_producer.wait_for_blocks(3, false /* is_local */), + ) + .await + .expect("The second should sync with the first"); let start_time = tokio::time::Instant::now(); // Stop the first producer. @@ -347,7 +353,12 @@ mod p2p { // it should sync remotely 5 blocks. let first_producer = make_node(make_node_config("First Producer reborn"), vec![]).await; - first_producer.wait_for_blocks(5, false /* is_local */).await; + tokio::time::timeout( + Duration::from_secs(SYNC_TIMEOUT), + first_producer.wait_for_blocks(5, false /* is_local */), + ) + .await + .expect("The first should reborn and sync with the second"); } fn update_signing_key(config: &mut Config, key: Address) { From 0e69bfdbf4017d0896bc615e1cbd102848089a08 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 11 Nov 2024 12:24:30 +0100 Subject: [PATCH 08/45] try to debug --- .github/workflows/ci.yml | 4 +- crates/services/sync/src/import.rs | 1 + crates/services/sync/src/import/tests.rs | 82 +++++++++++++++++++++++- crates/services/sync/src/state.rs | 8 +++ crates/services/sync/src/sync.rs | 2 + tests/tests/poa.rs | 4 -- 6 files changed, 92 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b24063422f..e3df425055e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,9 +119,9 @@ jobs: - command: make args: check --locked - command: nextest - args: run --workspace + args: run --workspace -- --nocapture - command: nextest - args: run --all-features --workspace + args: run --all-features --workspace -- --nocapture - command: nextest args: run -p fuel-core --no-default-features - command: nextest diff --git a/crates/services/sync/src/import.rs b/crates/services/sync/src/import.rs index a0aebc6dc8f..ff9ca5eb9e8 100644 --- a/crates/services/sync/src/import.rs +++ b/crates/services/sync/src/import.rs @@ -162,6 +162,7 @@ where pub async fn import(&self, shutdown: &mut StateWatcher) -> anyhow::Result { self.import_inner(shutdown).await?; + dbg!("waiting for notify or shutdown"); Ok(wait_for_notify_or_shutdown(&self.notify, shutdown).await) } diff --git a/crates/services/sync/src/import/tests.rs b/crates/services/sync/src/import/tests.rs index 0db7d8c5532..323493f6ee1 100644 --- a/crates/services/sync/src/import/tests.rs +++ b/crates/services/sync/src/import/tests.rs @@ -5,15 +5,16 @@ use crate::{ import::test_helpers::{ empty_header, random_peer, - }, - ports::{ + }, ports::{ MockBlockImporterPort, MockConsensusPort, MockPeerToPeerPort, PeerReportReason, - }, + }, state::Status, sync::SyncHeights }; +use fuel_core_services::stream::IntoBoxStream; use fuel_core_types::services::p2p::Transactions; +use mockall::Sequence; use std::time::Duration; use super::*; @@ -275,6 +276,81 @@ async fn import__signature_fails_on_header_4_only() { assert_eq!((State::new(3, None), false), res); } +#[tokio::test] +async fn test() { + let (_tx, shutdown) = tokio::sync::watch::channel(fuel_core_services::State::Started); + let mut watcher = shutdown.into(); + // given + let observed_height_stream = + futures::stream::iter([2, 2, 2, 2, 2].into_iter().map(BlockHeight::from)).into_boxed(); + let committed_height_stream = + futures::stream::iter([]).into_boxed(); + let mut p2p = MockPeerToPeerPort::default(); + let mut seq = Sequence::new(); + p2p.expect_get_sealed_block_headers() + .times(1) + .in_sequence(&mut seq) + .returning(|_| { + dbg!("error p2p"); + Err(anyhow::anyhow!("Some network error")) + }); + p2p.expect_get_sealed_block_headers() + .times(1) + .in_sequence(&mut seq) + .returning(|_| { + let peer = random_peer(); + let headers = Some(vec![empty_header(2)]); + let headers = peer.bind(headers); + Ok(headers) + }); + p2p.expect_get_transactions() + .times(1) + .in_sequence(&mut seq) + .returning(|block_ids| { + let data = block_ids.data; + let v = data.into_iter().map(|_| Transactions::default()).collect(); + Ok(Some(v)) + }); + p2p.expect_report_peer().returning(|_, _| Ok(())); + let state = SharedMutex::new(State::new(1, None)); + let notify = Arc::new(Notify::new()); + let p2p = Arc::new(p2p); + let executor: Arc = Arc::new(DefaultMocks::times([1])); + let consensus: Arc = Arc::new(DefaultMocks::times([1])); + let params = Config { + block_stream_buffer_size: 10, + header_batch_size: 10, + }; + let import = Import { + state: state.clone(), + notify: notify.clone(), + params, + p2p, + executor, + consensus, + }; + let mut sync_heights = SyncHeights::new(observed_height_stream, committed_height_stream, state.clone(), notify.clone()); + + tokio::spawn(async move { + let mut should_continue = true; + while should_continue { + should_continue = import.import(&mut watcher).await.unwrap_or(true); + } + }); + sync_heights.sync().await.unwrap(); + let mut verif = false; + while !verif { + verif = state.lock().status() == &Status::Committed(1); + tokio::time::sleep(Duration::from_millis(100)).await; + } + sync_heights.sync().await.unwrap(); + let mut verif = false; + while !verif { + verif = state.lock().status() == &Status::Committed(2); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + #[tokio::test] async fn import__header_not_found() { // given diff --git a/crates/services/sync/src/state.rs b/crates/services/sync/src/state.rs index 7bf704c2b66..6c3a547fe3e 100644 --- a/crates/services/sync/src/state.rs +++ b/crates/services/sync/src/state.rs @@ -120,6 +120,7 @@ impl State { #[tracing::instrument] /// Record that a block has been observed. pub fn observe(&mut self, height: u32) -> bool { + dbg!(&height); let new_status = match &self.status { // Currently uninitialized so process from the start to the observed height. Status::Uninitialized => Some(Status::Processing(0..=height)), @@ -140,6 +141,7 @@ impl State { (!r.is_empty()).then_some(Status::Processing(r)) }), }; + dbg!(&new_status); let status_change = new_status.is_some(); self.apply_status(new_status); status_change @@ -218,4 +220,10 @@ impl State { _ => None, } } + + #[cfg(test)] + /// Get the current status. + pub fn status(&self) -> &Status { + &self.status + } } diff --git a/crates/services/sync/src/sync.rs b/crates/services/sync/src/sync.rs index c7f39918c59..a16edb45af2 100644 --- a/crates/services/sync/src/sync.rs +++ b/crates/services/sync/src/sync.rs @@ -19,6 +19,7 @@ use crate::state::State; #[cfg(test)] mod tests; +#[derive(Debug)] pub(crate) enum IncomingHeight { Observed(BlockHeight), Committed(BlockHeight), @@ -63,6 +64,7 @@ impl SyncHeights { IncomingHeight::Observed(height) => self.state.apply(|s| s.observe(*height)), }; if state_change { + dbg!("notify"); self.notify.notify_one(); } Some(()) diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index 4d7bc356dce..6391ac91a8d 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -262,10 +262,6 @@ mod p2p { // after the first_producer stops, second_producer should start producing blocks #[tokio::test(flavor = "multi_thread")] async fn test_poa_multiple_producers() { - // Tracing layer with debug - tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .init(); const SYNC_TIMEOUT: u64 = 5; const TIME_UNTIL_SYNCED: u64 = SYNC_TIMEOUT + 5; From e8137a40bbe20563878806647d24935de88b3071 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 11 Nov 2024 12:27:02 +0100 Subject: [PATCH 09/45] fmt, spellcheck --- .../client/assets/debugAdapterProtocol.json | 2 +- crates/services/sync/src/import/tests.rs | 34 ++++++++++++------- tests/tests/poa.rs | 2 +- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/crates/client/assets/debugAdapterProtocol.json b/crates/client/assets/debugAdapterProtocol.json index 44a0c2eed9c..b435aef0f85 100644 --- a/crates/client/assets/debugAdapterProtocol.json +++ b/crates/client/assets/debugAdapterProtocol.json @@ -1440,7 +1440,7 @@ { "$ref": "#/definitions/Request" }, { "type": "object", - "description": "Replaces all existing instruction breakpoints. Typically, instruction breakpoints would be set from a diassembly window. \nTo clear all instruction breakpoints, specify an empty array.\nWhen an instruction breakpoint is hit, a 'stopped' event (with reason 'instruction breakpoint') is generated.\nClients should only call this request if the capability 'supportsInstructionBreakpoints' is true.", + "description": "Replaces all existing instruction breakpoints. Typically, instruction breakpoints would be set from a disassembly window. \nTo clear all instruction breakpoints, specify an empty array.\nWhen an instruction breakpoint is hit, a 'stopped' event (with reason 'instruction breakpoint') is generated.\nClients should only call this request if the capability 'supportsInstructionBreakpoints' is true.", "properties": { "command": { "type": "string", diff --git a/crates/services/sync/src/import/tests.rs b/crates/services/sync/src/import/tests.rs index 323493f6ee1..dd827ca1bbc 100644 --- a/crates/services/sync/src/import/tests.rs +++ b/crates/services/sync/src/import/tests.rs @@ -5,12 +5,15 @@ use crate::{ import::test_helpers::{ empty_header, random_peer, - }, ports::{ + }, + ports::{ MockBlockImporterPort, MockConsensusPort, MockPeerToPeerPort, PeerReportReason, - }, state::Status, sync::SyncHeights + }, + state::Status, + sync::SyncHeights, }; use fuel_core_services::stream::IntoBoxStream; use fuel_core_types::services::p2p::Transactions; @@ -282,9 +285,9 @@ async fn test() { let mut watcher = shutdown.into(); // given let observed_height_stream = - futures::stream::iter([2, 2, 2, 2, 2].into_iter().map(BlockHeight::from)).into_boxed(); - let committed_height_stream = - futures::stream::iter([]).into_boxed(); + futures::stream::iter([2, 2, 2, 2, 2].into_iter().map(BlockHeight::from)) + .into_boxed(); + let committed_height_stream = futures::stream::iter([]).into_boxed(); let mut p2p = MockPeerToPeerPort::default(); let mut seq = Sequence::new(); p2p.expect_get_sealed_block_headers() @@ -304,13 +307,13 @@ async fn test() { Ok(headers) }); p2p.expect_get_transactions() - .times(1) - .in_sequence(&mut seq) - .returning(|block_ids| { - let data = block_ids.data; - let v = data.into_iter().map(|_| Transactions::default()).collect(); - Ok(Some(v)) - }); + .times(1) + .in_sequence(&mut seq) + .returning(|block_ids| { + let data = block_ids.data; + let v = data.into_iter().map(|_| Transactions::default()).collect(); + Ok(Some(v)) + }); p2p.expect_report_peer().returning(|_, _| Ok(())); let state = SharedMutex::new(State::new(1, None)); let notify = Arc::new(Notify::new()); @@ -329,7 +332,12 @@ async fn test() { executor, consensus, }; - let mut sync_heights = SyncHeights::new(observed_height_stream, committed_height_stream, state.clone(), notify.clone()); + let mut sync_heights = SyncHeights::new( + observed_height_stream, + committed_height_stream, + state.clone(), + notify.clone(), + ); tokio::spawn(async move { let mut should_continue = true; diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index 6391ac91a8d..c0147cd4ea0 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -253,8 +253,8 @@ mod p2p { fuel_tx::Input, fuel_types::Address, }; - use tracing::info; use std::time::Duration; + use tracing::info; // Starts first_producer which creates some blocks // Then starts second_producer that uses the first one as a reserved peer. From a97ad1155abe9fb257093229bf509338152d73ac Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Mon, 11 Nov 2024 12:43:19 +0100 Subject: [PATCH 10/45] use nocapture --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3df425055e..59168d59126 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,9 +119,9 @@ jobs: - command: make args: check --locked - command: nextest - args: run --workspace -- --nocapture + args: run --workspace --nocapture - command: nextest - args: run --all-features --workspace -- --nocapture + args: run --all-features --workspace --nocapture - command: nextest args: run -p fuel-core --no-default-features - command: nextest From 4dec2aa80e4478716a56ebf1f42381a633e2d505 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 09:42:56 +0100 Subject: [PATCH 11/45] Add base last height to p2p --- crates/fuel-core/src/service/sub_services.rs | 1 + crates/services/p2p/src/service.rs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/crates/fuel-core/src/service/sub_services.rs b/crates/fuel-core/src/service/sub_services.rs index 342783c2bae..4fd45534b24 100644 --- a/crates/fuel-core/src/service/sub_services.rs +++ b/crates/fuel-core/src/service/sub_services.rs @@ -214,6 +214,7 @@ pub fn init_sub_services( |(p2p_config, (shared_state, request_receiver))| { fuel_core_p2p::service::new_service( chain_id, + last_height, p2p_config, shared_state, request_receiver, diff --git a/crates/services/p2p/src/service.rs b/crates/services/p2p/src/service.rs index 30c5dd9310a..e3785f9098e 100644 --- a/crates/services/p2p/src/service.rs +++ b/crates/services/p2p/src/service.rs @@ -395,6 +395,7 @@ impl Broadcast for SharedState { /// Uninitialized task for the p2p that can be upgraded later into [`Task`]. pub struct UninitializedTask { chain_id: ChainId, + last_height: BlockHeight, view_provider: V, next_block_height: BoxStream, /// Receive internal Task Requests @@ -438,6 +439,7 @@ pub struct HeartbeatPeerReputationConfig { impl UninitializedTask { pub fn new( chain_id: ChainId, + last_height: BlockHeight, config: Config, shared_state: SharedState, request_receiver: Receiver, @@ -449,6 +451,7 @@ impl UninitializedTask { Self { chain_id, + last_height, view_provider, tx_pool, next_block_height, @@ -768,6 +771,7 @@ where ) -> anyhow::Result { let Self { chain_id, + last_height, view_provider, next_block_height, request_receiver, @@ -805,6 +809,7 @@ where PostcardCodec::new(max_block_size), ) .await?; + p2p_service.update_block_height(last_height); p2p_service.start().await?; let next_check_time = @@ -1292,6 +1297,7 @@ pub fn build_shared_state( pub fn new_service( chain_id: ChainId, + last_height: BlockHeight, p2p_config: Config, shared_state: SharedState, request_receiver: Receiver, @@ -1307,6 +1313,7 @@ where { let task = UninitializedTask::new( chain_id, + last_height, p2p_config, shared_state, request_receiver, @@ -1436,6 +1443,7 @@ pub mod tests { let (shared_state, request_receiver) = build_shared_state(p2p_config.clone()); let service = new_service( ChainId::default(), + 0.into(), p2p_config, shared_state, request_receiver, From fd4bc3bbd39cfe34744921e19d085ad1f0ca3a8d Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 09:54:37 +0100 Subject: [PATCH 12/45] allow clippy --- crates/services/p2p/src/service.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/services/p2p/src/service.rs b/crates/services/p2p/src/service.rs index e3785f9098e..ed3c73528ff 100644 --- a/crates/services/p2p/src/service.rs +++ b/crates/services/p2p/src/service.rs @@ -437,6 +437,7 @@ pub struct HeartbeatPeerReputationConfig { } impl UninitializedTask { + #[allow(clippy::too_many_arguments)] pub fn new( chain_id: ChainId, last_height: BlockHeight, @@ -1295,6 +1296,7 @@ pub fn build_shared_state( ) } +#[allow(clippy::too_many_arguments)] pub fn new_service( chain_id: ChainId, last_height: BlockHeight, From 40ada1a5298a17e1343f71e8465978ed7ab2db69 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 11:10:42 +0100 Subject: [PATCH 13/45] Clean up branch and split docker production to remove it and only allow manual in future commit --- .github/workflows/ci.yml | 412 +---------------- .github/workflows/docker-images.yml | 427 ++++++++++++++++++ crates/fuel-core/src/executor.rs | 1 - .../src/v0/tests/algorithm_v0_tests.rs | 1 - .../src/v1/tests/algorithm_v1_tests.rs | 1 - .../poa/src/service_test/trigger_tests.rs | 1 - crates/services/sync/src/import.rs | 1 - crates/services/sync/src/import/tests.rs | 86 +--- crates/services/sync/src/state.rs | 2 - crates/services/sync/src/sync.rs | 2 - tests/tests/dos.rs | 1 - tests/tests/poa.rs | 4 +- 12 files changed, 432 insertions(+), 507 deletions(-) create mode 100644 .github/workflows/docker-images.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59168d59126..006c91594c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,9 +119,9 @@ jobs: - command: make args: check --locked - command: nextest - args: run --workspace --nocapture + args: run --workspace - command: nextest - args: run --all-features --workspace --nocapture + args: run --all-features --workspace - command: nextest args: run -p fuel-core --no-default-features - command: nextest @@ -340,385 +340,6 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} slack_webhook: ${{ secrets.SLACK_WEBHOOK_NOTIFY_BUILD }} - build-docker-images: - needs: - - publish-crates-check - strategy: - matrix: - arch: [ - # build on native runners instead of using emulation - { platform: linux/amd64, runner: buildjet-8vcpu-ubuntu-2204 }, - { platform: linux/arm64, runner: buildjet-16vcpu-ubuntu-2204-arm } - ] - runs-on: ${{ matrix.arch.runner }} - permissions: - contents: read - packages: write - steps: - - name: Setup environment - run: | - echo "REGISTRY_URL=${REGISTRY@L}/${GIT_REPO@L}" >>${GITHUB_ENV} - platform=${{ matrix.arch.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the ghcr.io registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to the docker.io registry - uses: docker/login-action@v3 - with: - username: fuellabs - password: ${{ secrets.DOCKER_IO_READ_ONLY_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY_URL }} - - - name: Setup Rust build cache - id: cache - uses: buildjet/cache@v3 - with: - path: | - home-cargo-bin - home-cargo-registry-index - home-cargo-registry-cache - home-cargo-git-db - target - key: ${{ env.PLATFORM_PAIR }}-${{ hashFiles('**/Cargo.lock') }} - - - name: Inject cache into docker - uses: reproducible-containers/buildkit-cache-dance@v3.1.2 - with: - cache-map: | - { - "home-cargo-bin": "/usr/local/cargo/bin", - "home-cargo-registry-index": "/usr/local/cargo/registry/index", - "home-cargo-registry-cache": "/usr/local/cargo/registry/cache", - "home-cargo-git-db": "/usr/local/cargo/git/db", - "target": "/build/target" - } - skip-extraction: ${{ steps.cache.outputs.cache-hit }} - - - name: Build Docker image - id: build - uses: docker/build-push-action@v6 - with: - context: . - platforms: ${{ matrix.arch.platform }} - file: deployment/Dockerfile - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache:latest-${{ matrix.arch.runner }} - cache-to: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache:latest-${{ matrix.arch.runner }},mode=max,image-manifest=true,oci-mediatypes=true - outputs: | - type=image,name=${{ env.REGISTRY_URL }},push-by-digest=true,name-canonical=true,push=true - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - publish-docker-image: - needs: - - build-docker-images - - verifications-complete - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - id-token: write - steps: - - name: Setup environment - run: | - echo "REGISTRY_URL=${REGISTRY@L}/${GIT_REPO@L}" >>${GITHUB_ENV} - - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-* - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Configure AWS credentials for ECR publishing - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.AWS_ROLE_ARN }} - aws-region: us-east-1 # ecr public is only in us-east-1 - - - name: Login to Amazon ECR Public - id: login-ecr-public - uses: aws-actions/amazon-ecr-login@v2 - with: - registry-type: public - - - name: Log in to the ghcr.io registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to the docker.io registry - uses: docker/login-action@v3 - with: - username: fuellabs - password: ${{ secrets.DOCKER_IO_READ_ONLY_TOKEN }} - - - name: Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY_URL }} - ${{ steps.login-ecr-public.outputs.registry }}/${{ env.AWS_ECR_ORG }}/${{ env.GIT_REPO_NAME }} - tags: | - type=sha - type=ref,event=branch - type=ref,event=tag - type=semver,pattern={{raw}} - type=raw,value=sha-{{sha}}-{{date 'YYYYMMDDhhmmss'}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Create manifest list and push to all registries - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.REGISTRY_URL }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.REGISTRY_URL }}:${{ steps.meta.outputs.version }} - - # duplicate of publish-docker-image, but with profiling features enabled - # this is split into a separate action since it takes longer to build - publish-docker-image-profiling: - needs: - - verifications-complete - runs-on: buildjet-16vcpu-ubuntu-2204 - permissions: - contents: read - packages: write - id-token: write - steps: - - name: Setup environment - run: | - echo "REGISTRY_URL=${REGISTRY@L}/${GIT_REPO@L}" >>${GITHUB_ENV} - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Configure AWS credentials for ECR publishing - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.AWS_ROLE_ARN }} - aws-region: us-east-1 # ecr public is only in us-east-1 - - - name: Login to Amazon ECR Public - id: login-ecr-public - uses: aws-actions/amazon-ecr-login@v2 - with: - registry-type: public - - - name: Log in to the ghcr.io registry - uses: docker/login-action@v1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to the docker.io registry - uses: docker/login-action@v2 - with: - username: fuellabs - password: ${{ secrets.DOCKER_IO_READ_ONLY_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY_URL }}-debug - ${{ steps.login-ecr-public.outputs.registry }}/${{ env.AWS_ECR_ORG }}/${{ env.GIT_REPO_NAME }}-debug - tags: | - type=sha - type=ref,event=branch - type=ref,event=tag - type=semver,pattern={{raw}} - type=raw,value=sha-{{sha}}-{{date 'YYYYMMDDhhmmss'}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Setup Rust build cache - id: cache - uses: buildjet/cache@v3 - with: - path: | - home-cargo-bin - home-cargo-registry-index - home-cargo-registry-cache - home-cargo-git-db - target - key: publish-docker-image-profiling-${{ hashFiles('**/Cargo.lock') }} - - - name: Inject cache into docker - uses: reproducible-containers/buildkit-cache-dance@v3.1.2 - with: - cache-map: | - { - "home-cargo-bin": "/usr/local/cargo/bin", - "home-cargo-registry-index": "/usr/local/cargo/registry/index", - "home-cargo-registry-cache": "/usr/local/cargo/registry/cache", - "home-cargo-git-db": "/usr/local/cargo/git/db", - "target": "/build/target" - } - skip-extraction: ${{ steps.cache.outputs.cache-hit }} - - - name: Build & push Docker image - id: build - uses: docker/build-push-action@v6 - with: - context: . - file: deployment/Dockerfile - build-args: "DEBUG_SYMBOLS=true" - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-debug:latest - cache-to: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-debug:latest,mode=max,image-manifest=true,oci-mediatypes=true - - - uses: FuelLabs/.github/.github/actions/slack-notify-template@master - if: always() && (github.ref == 'refs/heads/master' || github.ref_type == 'tag') - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - slack_webhook: ${{ secrets.SLACK_WEBHOOK_NOTIFY_BUILD }} - - publish-e2e-client-docker-image: - needs: - - verifications-complete - runs-on: buildjet-4vcpu-ubuntu-2204 - permissions: - contents: read - packages: write - id-token: write - steps: - - name: Setup environment - run: | - echo "REGISTRY_URL=${REGISTRY@L}/${GIT_REPO@L}" >>${GITHUB_ENV} - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Configure AWS credentials for ECR publishing - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.AWS_ROLE_ARN }} - aws-region: us-east-1 # ecr public is only in us-east-1 - - - name: Login to Amazon ECR Public - id: login-ecr-public - uses: aws-actions/amazon-ecr-login@v2 - with: - registry-type: public - - - name: Log in to the ghcr.io registry - uses: docker/login-action@v1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to the docker.io registry - uses: docker/login-action@v2 - with: - username: fuellabs - password: ${{ secrets.DOCKER_IO_READ_ONLY_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY_URL }}-e2e-client - ${{ steps.login-ecr-public.outputs.registry }}/${{ env.AWS_ECR_ORG }}/${{ env.GIT_REPO_NAME }}-e2e-client - tags: | - type=sha - type=ref,event=branch - type=ref,event=tag - type=semver,pattern={{raw}} - type=raw,value=sha-{{sha}}-{{date 'YYYYMMDDhhmmss'}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Setup Rust build cache - id: cache - uses: buildjet/cache@v3 - with: - path: | - home-cargo-bin - home-cargo-registry-index - home-cargo-registry-cache - home-cargo-git-db - target - key: publish-e2e-client-docker-image-${{ hashFiles('**/Cargo.lock') }} - - - name: Inject cache into docker - uses: reproducible-containers/buildkit-cache-dance@v3.1.2 - with: - cache-map: | - { - "home-cargo-bin": "/usr/local/cargo/bin", - "home-cargo-registry-index": "/usr/local/cargo/registry/index", - "home-cargo-registry-cache": "/usr/local/cargo/registry/cache", - "home-cargo-git-db": "/usr/local/cargo/git/db", - "target": "/build/target" - } - skip-extraction: ${{ steps.cache.outputs.cache-hit }} - - - name: Build & push Docker image - id: build - uses: docker/build-push-action@v6 - with: - context: . - file: deployment/e2e-client.Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-e2e:latest - cache-to: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-e2e:latest,mode=max,image-manifest=true,oci-mediatypes=true - - - uses: FuelLabs/.github/.github/actions/slack-notify-template@master - if: always() && (github.ref == 'refs/heads/master' || github.ref_type == 'tag') - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - slack_webhook: ${{ secrets.SLACK_WEBHOOK_NOTIFY_BUILD }} - publish-fuel-core-binary: name: Release fuel-core binaries runs-on: ${{ matrix.job.os }} @@ -864,35 +485,6 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} slack_webhook: ${{ secrets.SLACK_WEBHOOK_NOTIFY_BUILD }} - - # Deploy Fuel Core Ephemeral Developer Environment - deploy-eph-env: - if: startsWith(github.head_ref, 'preview/') - needs: - - publish-docker-image - runs-on: buildjet-4vcpu-ubuntu-2204 - steps: - - name: Set Environment Variables - run: | - tag=(`echo $GITHUB_SHA | cut -c1-7`) - echo "IMAGE_TAG=`echo sha-$tag`" >> $GITHUB_ENV - echo "DEPLOYMENT_VERSION=$(echo $GITHUB_SHA)" >> $GITHUB_ENV - echo "NAMESPACE=$(echo ${GITHUB_HEAD_REF} | cut -c 9-)" >> $GITHUB_ENV - - - name: Deploy Fuel Core Ephemeral Developer Environment - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: Deploy Fuel-Core on k8s - repo: FuelLabs/fuel-deployment - ref: refs/heads/master - token: ${{ secrets.REPO_TOKEN }} - inputs: '{ "k8s-type": "${{ env.K8S }}", "config-directory": "${{ env.CONFIG }}", "config-env": "${{ env.ENV }}", "deployment-version": "${{ env.DEPLOYMENT_VERSION }}", "image-tag": "${{ env.IMAGE_TAG }}", "namespace": "${{ env.NAMESPACE }}", "delete-infra": "${{ env.DELETE_INFRA }}" }' - env: - K8S: 'eks' - CONFIG: 'fuel-dev1' - ENV: 'fueldevsway.env' - DELETE_INFRA: true - cargo-audit: runs-on: ubuntu-latest continue-on-error: true diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml new file mode 100644 index 00000000000..4f333d02e7f --- /dev/null +++ b/.github/workflows/docker-images.yml @@ -0,0 +1,427 @@ +name: Docker Images + +on: + workflow_dispatch: + #TODO: Remove to only trigger on workflow dispatch + push: + branches: + - master + pull_request: + types: [ opened, synchronize, reopened, labeled, unlabeled ] + release: + types: [ published ] + +env: + GIT_BRANCH: ${{ github.head_ref || github.ref_name }} + GIT_REPO_OWNER: ${{ github.repository_owner }} + GIT_REPO: ${{ github.repository }} + GIT_REPO_NAME: ${{ github.event.repository.name }} + AWS_ROLE_ARN: arn:aws:iam::024848458133:role/github_oidc_FuelLabs_fuel-core + AWS_ECR_ORG: fuellabs + CARGO_TERM_COLOR: always + RUST_VERSION: 1.79.0 + RUST_VERSION_FMT: nightly-2023-10-29 + RUST_VERSION_COV: nightly-2024-06-05 + RUSTFLAGS: -D warnings + REGISTRY: ghcr.io + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2 + +jobs: + build-docker-images: + strategy: + matrix: + arch: [ + # build on native runners instead of using emulation + { platform: linux/amd64, runner: buildjet-8vcpu-ubuntu-2204 }, + { platform: linux/arm64, runner: buildjet-16vcpu-ubuntu-2204-arm } + ] + runs-on: ${{ matrix.arch.runner }} + permissions: + contents: read + packages: write + steps: + - name: Setup environment + run: | + echo "REGISTRY_URL=${REGISTRY@L}/${GIT_REPO@L}" >>${GITHUB_ENV} + platform=${{ matrix.arch.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the ghcr.io registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to the docker.io registry + uses: docker/login-action@v3 + with: + username: fuellabs + password: ${{ secrets.DOCKER_IO_READ_ONLY_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_URL }} + + - name: Setup Rust build cache + id: cache + uses: buildjet/cache@v3 + with: + path: | + home-cargo-bin + home-cargo-registry-index + home-cargo-registry-cache + home-cargo-git-db + target + key: ${{ env.PLATFORM_PAIR }}-${{ hashFiles('**/Cargo.lock') }} + + - name: Inject cache into docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "home-cargo-bin": "/usr/local/cargo/bin", + "home-cargo-registry-index": "/usr/local/cargo/registry/index", + "home-cargo-registry-cache": "/usr/local/cargo/registry/cache", + "home-cargo-git-db": "/usr/local/cargo/git/db", + "target": "/build/target" + } + skip-extraction: ${{ steps.cache.outputs.cache-hit }} + + - name: Build Docker image + id: build + uses: docker/build-push-action@v6 + with: + context: . + platforms: ${{ matrix.arch.platform }} + file: deployment/Dockerfile + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache:latest-${{ matrix.arch.runner }} + cache-to: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache:latest-${{ matrix.arch.runner }},mode=max,image-manifest=true,oci-mediatypes=true + outputs: | + type=image,name=${{ env.REGISTRY_URL }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + publish-docker-image: + needs: + - build-docker-images + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + steps: + - name: Setup environment + run: | + echo "REGISTRY_URL=${REGISTRY@L}/${GIT_REPO@L}" >>${GITHUB_ENV} + + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials for ECR publishing + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: us-east-1 # ecr public is only in us-east-1 + + - name: Login to Amazon ECR Public + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Log in to the ghcr.io registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to the docker.io registry + uses: docker/login-action@v3 + with: + username: fuellabs + password: ${{ secrets.DOCKER_IO_READ_ONLY_TOKEN }} + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY_URL }} + ${{ steps.login-ecr-public.outputs.registry }}/${{ env.AWS_ECR_ORG }}/${{ env.GIT_REPO_NAME }} + tags: | + type=sha + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{raw}} + type=raw,value=sha-{{sha}}-{{date 'YYYYMMDDhhmmss'}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Create manifest list and push to all registries + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_URL }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_URL }}:${{ steps.meta.outputs.version }} + + # duplicate of publish-docker-image, but with profiling features enabled + # this is split into a separate action since it takes longer to build + publish-docker-image-profiling: + runs-on: buildjet-16vcpu-ubuntu-2204 + permissions: + contents: read + packages: write + id-token: write + steps: + - name: Setup environment + run: | + echo "REGISTRY_URL=${REGISTRY@L}/${GIT_REPO@L}" >>${GITHUB_ENV} + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials for ECR publishing + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: us-east-1 # ecr public is only in us-east-1 + + - name: Login to Amazon ECR Public + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Log in to the ghcr.io registry + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to the docker.io registry + uses: docker/login-action@v2 + with: + username: fuellabs + password: ${{ secrets.DOCKER_IO_READ_ONLY_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY_URL }}-debug + ${{ steps.login-ecr-public.outputs.registry }}/${{ env.AWS_ECR_ORG }}/${{ env.GIT_REPO_NAME }}-debug + tags: | + type=sha + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{raw}} + type=raw,value=sha-{{sha}}-{{date 'YYYYMMDDhhmmss'}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Setup Rust build cache + id: cache + uses: buildjet/cache@v3 + with: + path: | + home-cargo-bin + home-cargo-registry-index + home-cargo-registry-cache + home-cargo-git-db + target + key: publish-docker-image-profiling-${{ hashFiles('**/Cargo.lock') }} + + - name: Inject cache into docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "home-cargo-bin": "/usr/local/cargo/bin", + "home-cargo-registry-index": "/usr/local/cargo/registry/index", + "home-cargo-registry-cache": "/usr/local/cargo/registry/cache", + "home-cargo-git-db": "/usr/local/cargo/git/db", + "target": "/build/target" + } + skip-extraction: ${{ steps.cache.outputs.cache-hit }} + + - name: Build & push Docker image + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: deployment/Dockerfile + build-args: "DEBUG_SYMBOLS=true" + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-debug:latest + cache-to: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-debug:latest,mode=max,image-manifest=true,oci-mediatypes=true + + - uses: FuelLabs/.github/.github/actions/slack-notify-template@master + if: always() && (github.ref == 'refs/heads/master' || github.ref_type == 'tag') + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + slack_webhook: ${{ secrets.SLACK_WEBHOOK_NOTIFY_BUILD }} + + publish-e2e-client-docker-image: + runs-on: buildjet-4vcpu-ubuntu-2204 + permissions: + contents: read + packages: write + id-token: write + steps: + - name: Setup environment + run: | + echo "REGISTRY_URL=${REGISTRY@L}/${GIT_REPO@L}" >>${GITHUB_ENV} + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials for ECR publishing + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: us-east-1 # ecr public is only in us-east-1 + + - name: Login to Amazon ECR Public + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Log in to the ghcr.io registry + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to the docker.io registry + uses: docker/login-action@v2 + with: + username: fuellabs + password: ${{ secrets.DOCKER_IO_READ_ONLY_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY_URL }}-e2e-client + ${{ steps.login-ecr-public.outputs.registry }}/${{ env.AWS_ECR_ORG }}/${{ env.GIT_REPO_NAME }}-e2e-client + tags: | + type=sha + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{raw}} + type=raw,value=sha-{{sha}}-{{date 'YYYYMMDDhhmmss'}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Setup Rust build cache + id: cache + uses: buildjet/cache@v3 + with: + path: | + home-cargo-bin + home-cargo-registry-index + home-cargo-registry-cache + home-cargo-git-db + target + key: publish-e2e-client-docker-image-${{ hashFiles('**/Cargo.lock') }} + + - name: Inject cache into docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "home-cargo-bin": "/usr/local/cargo/bin", + "home-cargo-registry-index": "/usr/local/cargo/registry/index", + "home-cargo-registry-cache": "/usr/local/cargo/registry/cache", + "home-cargo-git-db": "/usr/local/cargo/git/db", + "target": "/build/target" + } + skip-extraction: ${{ steps.cache.outputs.cache-hit }} + + - name: Build & push Docker image + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: deployment/e2e-client.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-e2e:latest + cache-to: type=registry,ref=${{ env.REGISTRY_URL }}-build-cache-e2e:latest,mode=max,image-manifest=true,oci-mediatypes=true + + - uses: FuelLabs/.github/.github/actions/slack-notify-template@master + if: always() && (github.ref == 'refs/heads/master' || github.ref_type == 'tag') + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + slack_webhook: ${{ secrets.SLACK_WEBHOOK_NOTIFY_BUILD }} + # Deploy Fuel Core Ephemeral Developer Environment + deploy-eph-env: + if: startsWith(github.head_ref, 'preview/') + needs: + - publish-docker-image + runs-on: buildjet-4vcpu-ubuntu-2204 + steps: + - name: Set Environment Variables + run: | + tag=(`echo $GITHUB_SHA | cut -c1-7`) + echo "IMAGE_TAG=`echo sha-$tag`" >> $GITHUB_ENV + echo "DEPLOYMENT_VERSION=$(echo $GITHUB_SHA)" >> $GITHUB_ENV + echo "NAMESPACE=$(echo ${GITHUB_HEAD_REF} | cut -c 9-)" >> $GITHUB_ENV + + - name: Deploy Fuel Core Ephemeral Developer Environment + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: Deploy Fuel-Core on k8s + repo: FuelLabs/fuel-deployment + ref: refs/heads/master + token: ${{ secrets.REPO_TOKEN }} + inputs: '{ "k8s-type": "${{ env.K8S }}", "config-directory": "${{ env.CONFIG }}", "config-env": "${{ env.ENV }}", "deployment-version": "${{ env.DEPLOYMENT_VERSION }}", "image-tag": "${{ env.IMAGE_TAG }}", "namespace": "${{ env.NAMESPACE }}", "delete-infra": "${{ env.DELETE_INFRA }}" }' + env: + K8S: 'eks' + CONFIG: 'fuel-dev1' + ENV: 'fueldevsway.env' + DELETE_INFRA: true diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 412d13c18cf..c951a711549 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -2599,7 +2599,6 @@ mod tests { // One of two transactions is skipped. assert_eq!(skipped_transactions.len(), 1); let err = &skipped_transactions[0].1; - dbg!(err); assert!(matches!( err, &ExecutorError::TransactionValidity( diff --git a/crates/fuel-gas-price-algorithm/src/v0/tests/algorithm_v0_tests.rs b/crates/fuel-gas-price-algorithm/src/v0/tests/algorithm_v0_tests.rs index 0fad86ed2d6..a903a8efe6b 100644 --- a/crates/fuel-gas-price-algorithm/src/v0/tests/algorithm_v0_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v0/tests/algorithm_v0_tests.rs @@ -42,7 +42,6 @@ fn _worst_case__correctly_calculates_value( let change_amount = expected.saturating_mul(percentage).saturating_div(100); expected = expected.saturating_add(change_amount); } - dbg!(actual, expected); assert!(actual >= expected); } diff --git a/crates/fuel-gas-price-algorithm/src/v1/tests/algorithm_v1_tests.rs b/crates/fuel-gas-price-algorithm/src/v1/tests/algorithm_v1_tests.rs index 5578af6bdc2..9e6be41073b 100644 --- a/crates/fuel-gas-price-algorithm/src/v1/tests/algorithm_v1_tests.rs +++ b/crates/fuel-gas-price-algorithm/src/v1/tests/algorithm_v1_tests.rs @@ -72,7 +72,6 @@ fn _worst_case__correctly_calculates_value( let expected = expected_exec_price.saturating_add(expected_da_gas_price); - dbg!(actual, expected); assert!(actual >= expected); } diff --git a/crates/services/consensus_module/poa/src/service_test/trigger_tests.rs b/crates/services/consensus_module/poa/src/service_test/trigger_tests.rs index 8b403e97fc1..b1fa82ac2c3 100644 --- a/crates/services/consensus_module/poa/src/service_test/trigger_tests.rs +++ b/crates/services/consensus_module/poa/src/service_test/trigger_tests.rs @@ -502,7 +502,6 @@ async fn interval_trigger_even_if_queued_tx_events() { let block_creation_waiter = block_creation_notifier.clone(); tokio::task::spawn(async move { ctx.block_import.recv().await.unwrap(); - dbg!("First block produced"); block_creation_notifier.notify_waiters(); }); block_creation_waiter.notified().await; diff --git a/crates/services/sync/src/import.rs b/crates/services/sync/src/import.rs index ff9ca5eb9e8..a0aebc6dc8f 100644 --- a/crates/services/sync/src/import.rs +++ b/crates/services/sync/src/import.rs @@ -162,7 +162,6 @@ where pub async fn import(&self, shutdown: &mut StateWatcher) -> anyhow::Result { self.import_inner(shutdown).await?; - dbg!("waiting for notify or shutdown"); Ok(wait_for_notify_or_shutdown(&self.notify, shutdown).await) } diff --git a/crates/services/sync/src/import/tests.rs b/crates/services/sync/src/import/tests.rs index dd827ca1bbc..f4a00617f5c 100644 --- a/crates/services/sync/src/import/tests.rs +++ b/crates/services/sync/src/import/tests.rs @@ -11,13 +11,9 @@ use crate::{ MockConsensusPort, MockPeerToPeerPort, PeerReportReason, - }, - state::Status, - sync::SyncHeights, + } }; -use fuel_core_services::stream::IntoBoxStream; use fuel_core_types::services::p2p::Transactions; -use mockall::Sequence; use std::time::Duration; use super::*; @@ -279,86 +275,6 @@ async fn import__signature_fails_on_header_4_only() { assert_eq!((State::new(3, None), false), res); } -#[tokio::test] -async fn test() { - let (_tx, shutdown) = tokio::sync::watch::channel(fuel_core_services::State::Started); - let mut watcher = shutdown.into(); - // given - let observed_height_stream = - futures::stream::iter([2, 2, 2, 2, 2].into_iter().map(BlockHeight::from)) - .into_boxed(); - let committed_height_stream = futures::stream::iter([]).into_boxed(); - let mut p2p = MockPeerToPeerPort::default(); - let mut seq = Sequence::new(); - p2p.expect_get_sealed_block_headers() - .times(1) - .in_sequence(&mut seq) - .returning(|_| { - dbg!("error p2p"); - Err(anyhow::anyhow!("Some network error")) - }); - p2p.expect_get_sealed_block_headers() - .times(1) - .in_sequence(&mut seq) - .returning(|_| { - let peer = random_peer(); - let headers = Some(vec![empty_header(2)]); - let headers = peer.bind(headers); - Ok(headers) - }); - p2p.expect_get_transactions() - .times(1) - .in_sequence(&mut seq) - .returning(|block_ids| { - let data = block_ids.data; - let v = data.into_iter().map(|_| Transactions::default()).collect(); - Ok(Some(v)) - }); - p2p.expect_report_peer().returning(|_, _| Ok(())); - let state = SharedMutex::new(State::new(1, None)); - let notify = Arc::new(Notify::new()); - let p2p = Arc::new(p2p); - let executor: Arc = Arc::new(DefaultMocks::times([1])); - let consensus: Arc = Arc::new(DefaultMocks::times([1])); - let params = Config { - block_stream_buffer_size: 10, - header_batch_size: 10, - }; - let import = Import { - state: state.clone(), - notify: notify.clone(), - params, - p2p, - executor, - consensus, - }; - let mut sync_heights = SyncHeights::new( - observed_height_stream, - committed_height_stream, - state.clone(), - notify.clone(), - ); - - tokio::spawn(async move { - let mut should_continue = true; - while should_continue { - should_continue = import.import(&mut watcher).await.unwrap_or(true); - } - }); - sync_heights.sync().await.unwrap(); - let mut verif = false; - while !verif { - verif = state.lock().status() == &Status::Committed(1); - tokio::time::sleep(Duration::from_millis(100)).await; - } - sync_heights.sync().await.unwrap(); - let mut verif = false; - while !verif { - verif = state.lock().status() == &Status::Committed(2); - tokio::time::sleep(Duration::from_millis(100)).await; - } -} - #[tokio::test] async fn import__header_not_found() { // given diff --git a/crates/services/sync/src/state.rs b/crates/services/sync/src/state.rs index 6c3a547fe3e..0cfbfe8e8f5 100644 --- a/crates/services/sync/src/state.rs +++ b/crates/services/sync/src/state.rs @@ -120,7 +120,6 @@ impl State { #[tracing::instrument] /// Record that a block has been observed. pub fn observe(&mut self, height: u32) -> bool { - dbg!(&height); let new_status = match &self.status { // Currently uninitialized so process from the start to the observed height. Status::Uninitialized => Some(Status::Processing(0..=height)), @@ -141,7 +140,6 @@ impl State { (!r.is_empty()).then_some(Status::Processing(r)) }), }; - dbg!(&new_status); let status_change = new_status.is_some(); self.apply_status(new_status); status_change diff --git a/crates/services/sync/src/sync.rs b/crates/services/sync/src/sync.rs index a16edb45af2..c7f39918c59 100644 --- a/crates/services/sync/src/sync.rs +++ b/crates/services/sync/src/sync.rs @@ -19,7 +19,6 @@ use crate::state::State; #[cfg(test)] mod tests; -#[derive(Debug)] pub(crate) enum IncomingHeight { Observed(BlockHeight), Committed(BlockHeight), @@ -64,7 +63,6 @@ impl SyncHeights { IncomingHeight::Observed(height) => self.state.apply(|s| s.observe(*height)), }; if state_change { - dbg!("notify"); self.notify.notify_one(); } Some(()) diff --git a/tests/tests/dos.rs b/tests/tests/dos.rs index 05c0e0fc030..03092699ef4 100644 --- a/tests/tests/dos.rs +++ b/tests/tests/dos.rs @@ -321,7 +321,6 @@ async fn complex_queries__100_block_headers__works() { let url = format!("http://{}/v1/graphql", node.bound_address); let result = send_graph_ql_query(&url, query).await; - dbg!(&result); assert!(result.contains("transactions")); } diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index c0147cd4ea0..bd89f09b720 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -262,8 +262,8 @@ mod p2p { // after the first_producer stops, second_producer should start producing blocks #[tokio::test(flavor = "multi_thread")] async fn test_poa_multiple_producers() { - const SYNC_TIMEOUT: u64 = 5; - const TIME_UNTIL_SYNCED: u64 = SYNC_TIMEOUT + 5; + const SYNC_TIMEOUT: u64 = 30; + const TIME_UNTIL_SYNCED: u64 = SYNC_TIMEOUT + 10; info!("Starting test_poa_multiple_producers"); let mut rng = StdRng::seed_from_u64(2222); From 6bb56a4b8b5123e07bad368e6d68137a3f8487ee Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 11:11:59 +0100 Subject: [PATCH 14/45] fmt --- crates/services/sync/src/import/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/services/sync/src/import/tests.rs b/crates/services/sync/src/import/tests.rs index f4a00617f5c..0db7d8c5532 100644 --- a/crates/services/sync/src/import/tests.rs +++ b/crates/services/sync/src/import/tests.rs @@ -11,7 +11,7 @@ use crate::{ MockConsensusPort, MockPeerToPeerPort, PeerReportReason, - } + }, }; use fuel_core_types::services::p2p::Transactions; use std::time::Duration; From 3f30d900d26cb42b0ed2a631d463305f25d56742 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 11:31:48 +0100 Subject: [PATCH 15/45] Fix flaky gas price test --- tests/tests/gas_price.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index acd8b7d2ad4..ca979b3dae4 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -297,6 +297,7 @@ async fn latest_gas_price__if_node_restarts_gets_latest_value() { let LatestGasPrice { gas_price, .. } = new_latest_gas_price; let actual = gas_price; assert_eq!(expected, actual); + recovered_driver.kill().await; } #[tokio::test] From d2fce0f037c5068703dacef51a790b388b86e9f0 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 11:32:37 +0100 Subject: [PATCH 16/45] remove launch of docker builds --- .github/workflows/docker-images.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 4f333d02e7f..31ceffc464a 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -2,14 +2,6 @@ name: Docker Images on: workflow_dispatch: - #TODO: Remove to only trigger on workflow dispatch - push: - branches: - - master - pull_request: - types: [ opened, synchronize, reopened, labeled, unlabeled ] - release: - types: [ published ] env: GIT_BRANCH: ${{ github.head_ref || github.ref_name }} From dc9c2262133d80e6cde7de00d5d49c9183aee7f8 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 11:41:10 +0100 Subject: [PATCH 17/45] remove cancel of other jobs when one fails --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 006c91594c1..96380bac26c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,6 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true env: GIT_BRANCH: ${{ github.head_ref || github.ref_name }} From efb84bae8b44bfb89ceb8a8808c947b5aea74fcb Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 11:47:13 +0100 Subject: [PATCH 18/45] fix disable of cancel for jobs --- .github/workflows/ci.yml | 2 ++ tests/tests/gas_price.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96380bac26c..5106d7ae33f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true env: GIT_BRANCH: ${{ github.head_ref || github.ref_name }} @@ -151,6 +152,7 @@ jobs: args: -p fuel-core-chain-config --target wasm32-unknown-unknown --no-default-features - command: check args: -p fuel-core-executor --target wasm32-unknown-unknown --no-default-features --features alloc + fail-fast: false # disallow any job that takes longer than 45 minutes timeout-minutes: 45 diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index ca979b3dae4..13305a7e127 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -298,6 +298,7 @@ async fn latest_gas_price__if_node_restarts_gets_latest_value() { let actual = gas_price; assert_eq!(expected, actual); recovered_driver.kill().await; + tokio::time::sleep(Duration::from_millis(10)).await; } #[tokio::test] From 53114098c2f799f2e67ad93da4d8b51860e6f067 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 12:04:08 +0100 Subject: [PATCH 19/45] Fix test gas price --- tests/tests/gas_price.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 13305a7e127..3a5c7cd3c4a 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -260,7 +260,7 @@ async fn estimate_gas_price__returns_min_gas_price_if_starting_gas_price_is_zero assert_eq!(MIN_GAS_PRICE, actual) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn latest_gas_price__if_node_restarts_gets_latest_value() { // given let args = vec![ @@ -297,8 +297,6 @@ async fn latest_gas_price__if_node_restarts_gets_latest_value() { let LatestGasPrice { gas_price, .. } = new_latest_gas_price; let actual = gas_price; assert_eq!(expected, actual); - recovered_driver.kill().await; - tokio::time::sleep(Duration::from_millis(10)).await; } #[tokio::test] From d3763504e5ced04142b7e10f964de9848f96bca4 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 12:36:34 +0100 Subject: [PATCH 20/45] Split codecov out of CI and increase timing before timeout --- .config/nextest.toml | 2 +- .github/workflows/ci.yml | 42 ------------------ .github/workflows/publish-codecov.yml | 61 +++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/publish-codecov.yml diff --git a/.config/nextest.toml b/.config/nextest.toml index 73484acfdb3..bc97d1f6931 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,2 +1,2 @@ [profile.default] -slow-timeout = { period = "60s", terminate-after = 5 } \ No newline at end of file +slow-timeout = { period = "60s", terminate-after = 8 } \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5106d7ae33f..28a8634d14f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,48 +244,6 @@ jobs: - name: Run integration tests for kms only run: cargo test -p fuel-core-tests --features aws-kms -- kms - publish-codecov: - name: Publish code coverage report on GitHub pages branch - runs-on: buildjet-4vcpu-ubuntu-2204 - needs: - - cargo-verifications - permissions: # Write access to push changes to pages - contents: write - steps: - - uses: actions/checkout@v4 - - name: Install latest Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ env.RUST_VERSION_COV }} - targets: wasm32-unknown-unknown - - - name: Install cargo-llvm-codecov - uses: taiki-e/install-action@cargo-llvm-cov - - - name: Code coverage report - run: cargo +${{ env.RUST_VERSION_COV }} llvm-cov --all-features --html --branch - - - name: Checkout the repo again for pushing pages revision - uses: actions/checkout@v4 - with: - ref: 'codecov-pages' - path: 'pages-branch' - - - name: Push codecov report to pages branch - working-directory: ./pages-branch - run: | - export BRANCH_B64=$(echo -n "${{ env.GIT_BRANCH }}" | basenc --base64url) - git config user.email "2204863+Dentosal@users.noreply.github.com" - git config user.name "Dentosal" - cp -r ../target/llvm-cov/html "$BRANCH_B64" - python3 ../.github/workflows/scripts/generate_pages_index.py > index.html - git add . - git commit -m "Update codecov for ${{ env.GIT_BRANCH }}" - git push - export PAGES_URL="https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/$BRANCH_B64/index.html" - echo "$PAGES_URL" - echo "Codecov report $PAGES_URL" >> $GITHUB_STEP_SUMMARY - verifications-complete: needs: - cargo-verifications diff --git a/.github/workflows/publish-codecov.yml b/.github/workflows/publish-codecov.yml new file mode 100644 index 00000000000..42230a15b4d --- /dev/null +++ b/.github/workflows/publish-codecov.yml @@ -0,0 +1,61 @@ +name: Publish Codecov Report + +on: + workflow_dispatch: + +env: + GIT_BRANCH: ${{ github.head_ref || github.ref_name }} + GIT_REPO_OWNER: ${{ github.repository_owner }} + GIT_REPO: ${{ github.repository }} + GIT_REPO_NAME: ${{ github.event.repository.name }} + AWS_ROLE_ARN: arn:aws:iam::024848458133:role/github_oidc_FuelLabs_fuel-core + AWS_ECR_ORG: fuellabs + CARGO_TERM_COLOR: always + RUST_VERSION: 1.79.0 + RUST_VERSION_FMT: nightly-2023-10-29 + RUST_VERSION_COV: nightly-2024-06-05 + RUSTFLAGS: -D warnings + REGISTRY: ghcr.io + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2 + +jobs: + publish-codecov: + name: Publish code coverage report on GitHub pages branch + runs-on: buildjet-4vcpu-ubuntu-2204 + permissions: # Write access to push changes to pages + contents: write + steps: + - uses: actions/checkout@v4 + - name: Install latest Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.RUST_VERSION_COV }} + targets: wasm32-unknown-unknown + + - name: Install cargo-llvm-codecov + uses: taiki-e/install-action@cargo-llvm-cov + + - name: Code coverage report + run: cargo +${{ env.RUST_VERSION_COV }} llvm-cov --all-features --html --branch + + - name: Checkout the repo again for pushing pages revision + uses: actions/checkout@v4 + with: + ref: 'codecov-pages' + path: 'pages-branch' + + - name: Push codecov report to pages branch + working-directory: ./pages-branch + run: | + export BRANCH_B64=$(echo -n "${{ env.GIT_BRANCH }}" | basenc --base64url) + git config user.email "2204863+Dentosal@users.noreply.github.com" + git config user.name "Dentosal" + cp -r ../target/llvm-cov/html "$BRANCH_B64" + python3 ../.github/workflows/scripts/generate_pages_index.py > index.html + git add . + git commit -m "Update codecov for ${{ env.GIT_BRANCH }}" + git push + export PAGES_URL="https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/$BRANCH_B64/index.html" + echo "$PAGES_URL" + echo "Codecov report $PAGES_URL" >> $GITHUB_STEP_SUMMARY + \ No newline at end of file From fc8b8f7e018ed323a948e923e269bf0b69fa6517 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 12:56:46 +0100 Subject: [PATCH 21/45] Try to fix gas price test --- tests/tests/gas_price.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 3a5c7cd3c4a..40c35276e6b 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -297,6 +297,7 @@ async fn latest_gas_price__if_node_restarts_gets_latest_value() { let LatestGasPrice { gas_price, .. } = new_latest_gas_price; let actual = gas_price; assert_eq!(expected, actual); + recovered_driver.kill().await; } #[tokio::test] From e86c0b4b664ec83c5616ce0538af179716fd0943 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 13:02:01 +0100 Subject: [PATCH 22/45] Fix all gas price tests --- tests/tests/gas_price.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tests/gas_price.rs b/tests/tests/gas_price.rs index 40c35276e6b..5b24470ca34 100644 --- a/tests/tests/gas_price.rs +++ b/tests/tests/gas_price.rs @@ -358,7 +358,7 @@ async fn dry_run_opt__zero_gas_price_equal_to_none_gas_price() { assert_eq!(total_gas, total_gas_zero_gas_price); } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn startup__can_override_gas_price_values_by_changing_config() { // given let args = vec![ @@ -414,4 +414,5 @@ async fn startup__can_override_gas_price_values_by_changing_config() { l2_block_height, .. } = new_metadata.try_into().unwrap(); assert_eq!(l2_block_height, new_height); + recovered_driver.kill().await; } From 3179ead42caa29f4c709c549d131f73ae5e3e913 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 13:03:01 +0100 Subject: [PATCH 23/45] remove unused tracing --- tests/tests/poa.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index bd89f09b720..566c5759eb2 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -254,7 +254,6 @@ mod p2p { fuel_types::Address, }; use std::time::Duration; - use tracing::info; // Starts first_producer which creates some blocks // Then starts second_producer that uses the first one as a reserved peer. @@ -265,7 +264,6 @@ mod p2p { const SYNC_TIMEOUT: u64 = 30; const TIME_UNTIL_SYNCED: u64 = SYNC_TIMEOUT + 10; - info!("Starting test_poa_multiple_producers"); let mut rng = StdRng::seed_from_u64(2222); // Create a producer and a validator that share the same key pair. From de94b6508372d3fa8eb85df475b3d262acb5e9b1 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 13:49:55 +0100 Subject: [PATCH 24/45] kill all nodes on test finish and increase a timeout --- bin/e2e-test-client/src/tests/transfers.rs | 2 +- tests/tests/regenesis.rs | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bin/e2e-test-client/src/tests/transfers.rs b/bin/e2e-test-client/src/tests/transfers.rs index b9619453649..a0430d6b05a 100644 --- a/bin/e2e-test-client/src/tests/transfers.rs +++ b/bin/e2e-test-client/src/tests/transfers.rs @@ -49,7 +49,7 @@ pub async fn transfer_back(ctx: &TestContext) -> Result<(), Failed> { } // wait until alice sees the transaction timeout( - ctx.config.sync_timeout(), + ctx.config.sync_timeout().saturating_mul(2), ctx.alice.client.await_transaction_commit(&result.tx_id), ) .await??; diff --git a/tests/tests/regenesis.rs b/tests/tests/regenesis.rs index 3ba03d27366..3a02107fe75 100644 --- a/tests/tests/regenesis.rs +++ b/tests/tests/regenesis.rs @@ -176,6 +176,7 @@ async fn test_regenesis_old_blocks_are_preserved() -> anyhow::Result<()> { .expect("The block and all related data should migrate"); } + core.kill().await; Ok(()) } @@ -334,6 +335,7 @@ async fn test_regenesis_processed_transactions_are_preserved() -> anyhow::Result "Unexpected message {reason:?}" ); + core.kill().await; Ok(()) } @@ -481,6 +483,7 @@ async fn test_regenesis_message_proofs_are_preserved() -> anyhow::Result<()> { )); } + core.kill().await; Ok(()) } @@ -542,6 +545,7 @@ async fn starting_node_with_same_chain_config_keeps_genesis() -> anyhow::Result< .consensus; assert_eq!(original_consensus, non_modified_consensus); + core.kill().await; Ok(()) } @@ -603,6 +607,7 @@ async fn starting_node_with_new_chain_config_updates_genesis() -> anyhow::Result .consensus; assert_ne!(original_consensus, modified_consensus); + core.kill().await; Ok(()) } @@ -691,6 +696,7 @@ async fn starting_node_with_overwritten_old_poa_key_doesnt_rollback_the_state( .height; assert_eq!(original_block_height, block_height_after_override); + core.kill().await; Ok(()) } @@ -732,6 +738,7 @@ async fn starting_empty_node_with_overwritten_poa_works() -> anyhow::Result<()> let core = result.expect("Failed to start the node"); produce_block_with_tx(&mut rng, &core.client).await; + core.kill().await; Ok(()) } @@ -814,7 +821,7 @@ async fn starting_node_with_overwritten_new_poa_key_rollbacks_the_state( .height; assert_ne!(original_block_height, block_height_after_override); assert_eq!(override_height - 1, block_height_after_override); - + core.kill().await; Ok(()) } @@ -897,5 +904,6 @@ async fn starting_node_with_overwritten_new_poa_key_from_the_future_doesnt_rollb .height; assert_eq!(original_block_height, block_height_after_override); + core.kill().await; Ok(()) } From 6e89c35ef7716c6a2217c3f222f7df0200727d8e Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 14:34:51 +0100 Subject: [PATCH 25/45] try some tweak for p2p params --- crates/services/p2p/src/discovery.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/services/p2p/src/discovery.rs b/crates/services/p2p/src/discovery.rs index 725024c4939..8b60c824a61 100644 --- a/crates/services/p2p/src/discovery.rs +++ b/crates/services/p2p/src/discovery.rs @@ -253,7 +253,7 @@ mod tests { use libp2p_swarm_test::SwarmExt; - const MAX_PEERS: usize = 50; + const MAX_PEERS: usize = 1000; fn build_behavior_fn( bootstrap_nodes: Vec, @@ -264,7 +264,8 @@ mod tests { config .max_peers_connected(MAX_PEERS) .with_bootstrap_nodes(bootstrap_nodes) - .with_random_walk(Duration::from_millis(500)); + .with_random_walk(Duration::from_millis(200)) + .set_connection_idle_timeout(Duration::from_secs(30)); config.finish().expect("Config should be valid") } @@ -309,7 +310,6 @@ mod tests { for _ in 1..num_of_swarms { let (swarm, peer_addr, peer_id) = build_fuel_discovery(vec![bootstrap_addr.clone()]); - discovery_swarms.push((swarm, peer_addr, peer_id)); } From 91b40fdfb6db0398924b8efd36fd4cce22426174 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 14:58:44 +0100 Subject: [PATCH 26/45] Add more kill and try to reconnect on p2p --- crates/services/p2p/src/discovery.rs | 27 +++++++++++++++++++++++---- tests/tests/local_node.rs | 1 + tests/tests/poa.rs | 2 +- tests/tests/recovery.rs | 4 ++++ tests/tests/state_rewind.rs | 2 ++ 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/crates/services/p2p/src/discovery.rs b/crates/services/p2p/src/discovery.rs index 8b60c824a61..168c2c30dce 100644 --- a/crates/services/p2p/src/discovery.rs +++ b/crates/services/p2p/src/discovery.rs @@ -253,7 +253,7 @@ mod tests { use libp2p_swarm_test::SwarmExt; - const MAX_PEERS: usize = 1000; + const MAX_PEERS: usize = 50; fn build_behavior_fn( bootstrap_nodes: Vec, @@ -264,8 +264,7 @@ mod tests { config .max_peers_connected(MAX_PEERS) .with_bootstrap_nodes(bootstrap_nodes) - .with_random_walk(Duration::from_millis(200)) - .set_connection_idle_timeout(Duration::from_secs(30)); + .with_random_walk(Duration::from_millis(500)); config.finish().expect("Config should be valid") } @@ -369,7 +368,27 @@ mod tests { .add_address(&peer_id, unroutable_peer_addr.clone()); } SwarmEvent::ConnectionClosed { peer_id, .. } => { - panic!("PeerId {peer_id:?} disconnected"); + // reconnect to the peer + let unroutable_peer_addr = discovery_swarms + .iter() + .find_map(|(_, next_addr, next_peer_id)| { + // identify the peer + if next_peer_id == &peer_id { + // and return it's address + Some(next_addr.clone()) + } else { + None + } + }) + .unwrap(); + + // kademlia must be informed of a peer's address before + // adding it to the routing table + discovery_swarms[swarm_index] + .0 + .behaviour_mut() + .kademlia + .add_address(&peer_id, unroutable_peer_addr.clone()); } _ => {} } diff --git a/tests/tests/local_node.rs b/tests/tests/local_node.rs index 6176ada75c5..88467b57ff0 100644 --- a/tests/tests/local_node.rs +++ b/tests/tests/local_node.rs @@ -104,4 +104,5 @@ async fn start_local_node_without_any_arguments() { block.header.state_transition_bytecode_version, LATEST_STATE_TRANSITION_VERSION ); + service.kill().await; } diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index 566c5759eb2..85334b005da 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -229,7 +229,7 @@ async fn starting_node_with_predefined_nodes_produces_these_predefined_blocks( }) .collect(); assert_eq!(predefined_blocks, blocks_from_new_node); - + new_core.kill().await; Ok(()) } diff --git a/tests/tests/recovery.rs b/tests/tests/recovery.rs index d1f9cb9d7b9..623c5b832f1 100644 --- a/tests/tests/recovery.rs +++ b/tests/tests/recovery.rs @@ -61,6 +61,7 @@ async fn off_chain_worker_can_recover_on_start_up_when_is_behind() -> anyhow::Re Some(BlockHeight::new(HEIGHTS)) ); + recovered_driver.kill().await; Ok(()) } @@ -125,6 +126,7 @@ async fn _gas_price_updater__can_recover_on_startup_when_gas_price_db_is_ahead( assert_eq!(actual_onchain_height, expected_onchain_height); assert_eq!(actual_gas_price_height, expected_gas_price_height); + recovered_driver.kill().await; Ok(()) } @@ -190,6 +192,7 @@ async fn _gas_price_updater__can_recover_on_startup_when_gas_price_db_is_behind( Some(BlockHeight::new(height)) ); + recovered_driver.kill().await; Ok(()) } @@ -263,6 +266,7 @@ async fn gas_price_updater__if_no_metadata_history_start_from_current_block( Some(BlockHeight::new(next_height)) ); + recovered_driver.kill().await; Ok(()) } diff --git a/tests/tests/state_rewind.rs b/tests/tests/state_rewind.rs index 3c8a7af810b..f8553f351f5 100644 --- a/tests/tests/state_rewind.rs +++ b/tests/tests/state_rewind.rs @@ -114,6 +114,7 @@ async fn validate_block_at_any_height__only_transfers() -> anyhow::Result<()> { assert_eq!(&actual_changes, expected_changes); } + driver.kill().await; Ok(()) } @@ -211,6 +212,7 @@ async fn rollback_existing_chain_to_target_height_and_verify( .latest_height_from_metadata(); assert_eq!(Ok(Some(BlockHeight::new(target_height))), latest_height); + driver.kill().await; Ok(()) } From 5aa70775a7348caef6584bb86c2065814296d379 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 15:28:10 +0100 Subject: [PATCH 27/45] add kill --- tests/tests/regenesis.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests/regenesis.rs b/tests/tests/regenesis.rs index 3a02107fe75..e648a35d686 100644 --- a/tests/tests/regenesis.rs +++ b/tests/tests/regenesis.rs @@ -268,6 +268,7 @@ async fn test_regenesis_spent_messages_are_preserved() -> anyhow::Result<()> { .expect("Failed to get message status"); assert_eq!(status, MessageStatus::Spent); + core.kill().await; Ok(()) } From c72493f96b9a50023df2ba93ed15caaf41caa62b Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 16:12:28 +0100 Subject: [PATCH 28/45] Change timeouts --- .config/nextest.toml | 6 +++++- crates/services/upgradable-executor/src/executor.rs | 2 -- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index bc97d1f6931..3e3c1dd8944 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,2 +1,6 @@ [profile.default] -slow-timeout = { period = "60s", terminate-after = 8 } \ No newline at end of file +slow-timeout = { period = "60s", terminate-after = 5 } + +[[profile.default.overrides]] +filter = 'test(too_many_transactions_are_split_in_blocks | block_producer_never_includes_more_than_max_tx_count_transactions)' +slow-timeout = { period = "60s", terminate-after = 20 } \ No newline at end of file diff --git a/crates/services/upgradable-executor/src/executor.rs b/crates/services/upgradable-executor/src/executor.rs index 38ebabe0283..e29dc2389b6 100644 --- a/crates/services/upgradable-executor/src/executor.rs +++ b/crates/services/upgradable-executor/src/executor.rs @@ -1111,7 +1111,6 @@ mod test { // If it doesn't cache the modules, the test will fail with a timeout. #[test] #[cfg(not(coverage))] // Too slow for coverage - #[ntest::timeout(60_000)] fn reuse_cached_compiled_module__native_strategy() { // Given let next_version = Executor::::VERSION + 1; @@ -1132,7 +1131,6 @@ mod test { // If it doesn't cache the modules, the test will fail with a timeout. #[test] #[cfg(not(coverage))] // Too slow for coverage - #[ntest::timeout(60_000)] fn reuse_cached_compiled_module__wasm_strategy() { // Given let next_version = Executor::::VERSION + 1; From 843f7eac657d87d85c7bd8a3cc1df0ff82c21155 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 16:19:55 +0100 Subject: [PATCH 29/45] remove unused ntest --- Cargo.lock | 34 ------------------- .../services/upgradable-executor/Cargo.toml | 1 - 2 files changed, 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67a97f72c3e..c764158c37a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3852,7 +3852,6 @@ dependencies = [ "fuel-core-storage", "fuel-core-types 0.40.0", "fuel-core-wasm-executor", - "ntest", "parking_lot", "postcard", "tracing", @@ -6312,39 +6311,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "ntest" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb183f0a1da7a937f672e5ee7b7edb727bf52b8a52d531374ba8ebb9345c0330" -dependencies = [ - "ntest_test_cases", - "ntest_timeout", -] - -[[package]] -name = "ntest_test_cases" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d0d3f2a488592e5368ebbe996e7f1d44aa13156efad201f5b4d84e150eaa93" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "ntest_timeout" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc7c92f190c97f79b4a332f5e81dcf68c8420af2045c936c9be0bc9de6f63b5" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "nu-ansi-term" version = "0.46.0" diff --git a/crates/services/upgradable-executor/Cargo.toml b/crates/services/upgradable-executor/Cargo.toml index 4b340df0a25..16eeb0fc391 100644 --- a/crates/services/upgradable-executor/Cargo.toml +++ b/crates/services/upgradable-executor/Cargo.toml @@ -34,7 +34,6 @@ wasmtime = { version = "23.0.2", default-features = false, features = [ anyhow = { workspace = true } fuel-core-storage = { workspace = true, features = ["test-helpers"] } fuel-core-types = { workspace = true, features = ["test-helpers"] } -ntest = "0.9.2" [build-dependencies] fuel-core-wasm-executor = { workspace = true, optional = true, default-features = false } From 79de5a00e18d7dc6e49ca3dce6bc7a078da15b10 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 16:20:14 +0100 Subject: [PATCH 30/45] fmt --- crates/services/upgradable-executor/src/executor.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/services/upgradable-executor/src/executor.rs b/crates/services/upgradable-executor/src/executor.rs index e29dc2389b6..1bea9cea4c6 100644 --- a/crates/services/upgradable-executor/src/executor.rs +++ b/crates/services/upgradable-executor/src/executor.rs @@ -740,8 +740,6 @@ where #[cfg(test)] mod test { #[cfg(coverage)] - use ntest as _; // Only used outside cdg(coverage) - use super::*; use fuel_core_storage::{ kv_store::Value, @@ -946,7 +944,6 @@ mod test { mod native { use super::*; use crate::executor::Executor; - use ntest as _; #[test] fn can_validate_block() { From 1333c9968b799ef72e678916519d9df04823df0d Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 16:45:31 +0100 Subject: [PATCH 31/45] Fix compil and remove useless work --- crates/services/p2p/src/discovery.rs | 24 +------------------ .../upgradable-executor/src/executor.rs | 1 - 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/crates/services/p2p/src/discovery.rs b/crates/services/p2p/src/discovery.rs index 168c2c30dce..67eff3ddecb 100644 --- a/crates/services/p2p/src/discovery.rs +++ b/crates/services/p2p/src/discovery.rs @@ -367,29 +367,7 @@ mod tests { .kademlia .add_address(&peer_id, unroutable_peer_addr.clone()); } - SwarmEvent::ConnectionClosed { peer_id, .. } => { - // reconnect to the peer - let unroutable_peer_addr = discovery_swarms - .iter() - .find_map(|(_, next_addr, next_peer_id)| { - // identify the peer - if next_peer_id == &peer_id { - // and return it's address - Some(next_addr.clone()) - } else { - None - } - }) - .unwrap(); - - // kademlia must be informed of a peer's address before - // adding it to the routing table - discovery_swarms[swarm_index] - .0 - .behaviour_mut() - .kademlia - .add_address(&peer_id, unroutable_peer_addr.clone()); - } + SwarmEvent::ConnectionClosed { .. } => {} _ => {} } continue 'polling diff --git a/crates/services/upgradable-executor/src/executor.rs b/crates/services/upgradable-executor/src/executor.rs index 1bea9cea4c6..39d778bc650 100644 --- a/crates/services/upgradable-executor/src/executor.rs +++ b/crates/services/upgradable-executor/src/executor.rs @@ -739,7 +739,6 @@ where #[allow(unexpected_cfgs)] // for cfg(coverage) #[cfg(test)] mod test { - #[cfg(coverage)] use super::*; use fuel_core_storage::{ kv_store::Value, From b2e110c7ac8f350aa1f08d09390259f00735df1d Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Tue, 12 Nov 2024 18:26:54 +0100 Subject: [PATCH 32/45] Enhance test informations and add doc in Contributing. --- CONTRIBUTING.md | 4 ++++ bin/e2e-test-client/src/tests/transfers.rs | 2 +- crates/services/p2p/src/discovery.rs | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce3d6df8cd2..f379e61081a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,6 +153,10 @@ Multiple issues should use full syntax for each issue and separate by a comma, l close #123, ref #456 ``` +### Long tests issues + +If your test if taking more than 5 minutes on the CI, you need to edit the `.config/nextest.toml` file to allow your test to have a bigger timeout. See [nextest documentation](https://nexte.st/docs/configuration/?h=config) for format + ### Releasing Each release should have its own new version of the `fuel_core_upgradable_executor::Executor` regardless of minor or major release. The version of the executor should grow without gaps. diff --git a/bin/e2e-test-client/src/tests/transfers.rs b/bin/e2e-test-client/src/tests/transfers.rs index a0430d6b05a..5bede8e0ace 100644 --- a/bin/e2e-test-client/src/tests/transfers.rs +++ b/bin/e2e-test-client/src/tests/transfers.rs @@ -49,7 +49,7 @@ pub async fn transfer_back(ctx: &TestContext) -> Result<(), Failed> { } // wait until alice sees the transaction timeout( - ctx.config.sync_timeout().saturating_mul(2), + ctx.config.sync_timeout().checked_mul(2).unwrap(), ctx.alice.client.await_transaction_commit(&result.tx_id), ) .await??; diff --git a/crates/services/p2p/src/discovery.rs b/crates/services/p2p/src/discovery.rs index 67eff3ddecb..e41d228883d 100644 --- a/crates/services/p2p/src/discovery.rs +++ b/crates/services/p2p/src/discovery.rs @@ -367,7 +367,9 @@ mod tests { .kademlia .add_address(&peer_id, unroutable_peer_addr.clone()); } - SwarmEvent::ConnectionClosed { .. } => {} + SwarmEvent::ConnectionClosed { peer_id, .. } => { + dbg!(peer_id); + } _ => {} } continue 'polling From f29e216b2868999fe5e2a274708fde228aa9887c Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Wed, 13 Nov 2024 07:50:45 +0100 Subject: [PATCH 33/45] Readd ntest with bigger timeout and reconnect on peer discovery --- Cargo.lock | 34 +++++++++++++++++++ crates/services/p2p/src/discovery.rs | 2 +- .../services/upgradable-executor/Cargo.toml | 1 + .../upgradable-executor/src/executor.rs | 6 ++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c764158c37a..67a97f72c3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3852,6 +3852,7 @@ dependencies = [ "fuel-core-storage", "fuel-core-types 0.40.0", "fuel-core-wasm-executor", + "ntest", "parking_lot", "postcard", "tracing", @@ -6311,6 +6312,39 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "ntest" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb183f0a1da7a937f672e5ee7b7edb727bf52b8a52d531374ba8ebb9345c0330" +dependencies = [ + "ntest_test_cases", + "ntest_timeout", +] + +[[package]] +name = "ntest_test_cases" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d0d3f2a488592e5368ebbe996e7f1d44aa13156efad201f5b4d84e150eaa93" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ntest_timeout" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc7c92f190c97f79b4a332f5e81dcf68c8420af2045c936c9be0bc9de6f63b5" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" diff --git a/crates/services/p2p/src/discovery.rs b/crates/services/p2p/src/discovery.rs index e41d228883d..7cfe3ca25e7 100644 --- a/crates/services/p2p/src/discovery.rs +++ b/crates/services/p2p/src/discovery.rs @@ -368,7 +368,7 @@ mod tests { .add_address(&peer_id, unroutable_peer_addr.clone()); } SwarmEvent::ConnectionClosed { peer_id, .. } => { - dbg!(peer_id); + left_to_discover[swarm_index].insert(peer_id); } _ => {} } diff --git a/crates/services/upgradable-executor/Cargo.toml b/crates/services/upgradable-executor/Cargo.toml index 16eeb0fc391..4b340df0a25 100644 --- a/crates/services/upgradable-executor/Cargo.toml +++ b/crates/services/upgradable-executor/Cargo.toml @@ -34,6 +34,7 @@ wasmtime = { version = "23.0.2", default-features = false, features = [ anyhow = { workspace = true } fuel-core-storage = { workspace = true, features = ["test-helpers"] } fuel-core-types = { workspace = true, features = ["test-helpers"] } +ntest = "0.9.2" [build-dependencies] fuel-core-wasm-executor = { workspace = true, optional = true, default-features = false } diff --git a/crates/services/upgradable-executor/src/executor.rs b/crates/services/upgradable-executor/src/executor.rs index 39d778bc650..c94ec19488f 100644 --- a/crates/services/upgradable-executor/src/executor.rs +++ b/crates/services/upgradable-executor/src/executor.rs @@ -739,6 +739,9 @@ where #[allow(unexpected_cfgs)] // for cfg(coverage) #[cfg(test)] mod test { + #[cfg(coverage)] + use ntest as _; // Only used outside cdg(coverage) + use super::*; use fuel_core_storage::{ kv_store::Value, @@ -943,6 +946,7 @@ mod test { mod native { use super::*; use crate::executor::Executor; + use ntest as _; #[test] fn can_validate_block() { @@ -1107,6 +1111,7 @@ mod test { // If it doesn't cache the modules, the test will fail with a timeout. #[test] #[cfg(not(coverage))] // Too slow for coverage + #[ntest::timeout(90_000)] fn reuse_cached_compiled_module__native_strategy() { // Given let next_version = Executor::::VERSION + 1; @@ -1127,6 +1132,7 @@ mod test { // If it doesn't cache the modules, the test will fail with a timeout. #[test] #[cfg(not(coverage))] // Too slow for coverage + #[ntest::timeout(90_000)] fn reuse_cached_compiled_module__wasm_strategy() { // Given let next_version = Executor::::VERSION + 1; From d0b7b431ea09f81481281a1548e367fda472e141 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Wed, 13 Nov 2024 08:21:19 +0100 Subject: [PATCH 34/45] Fix big timeout filter --- .config/nextest.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 3e3c1dd8944..96b273f5fc5 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -2,5 +2,5 @@ slow-timeout = { period = "60s", terminate-after = 5 } [[profile.default.overrides]] -filter = 'test(too_many_transactions_are_split_in_blocks | block_producer_never_includes_more_than_max_tx_count_transactions)' +filter = 'test(too_many_transactions_are_split_in_blocks) + test(block_producer_never_includes_more_than_max_tx_count_transactions)' slow-timeout = { period = "60s", terminate-after = 20 } \ No newline at end of file From d140df4a77e8238530388b93b2f4e80d215a8454 Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Wed, 13 Nov 2024 09:16:46 +0100 Subject: [PATCH 35/45] Change timeout test execution --- Cargo.lock | 34 ------------------- .../services/upgradable-executor/Cargo.toml | 1 - .../upgradable-executor/src/executor.rs | 16 +++++---- 3 files changed, 10 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67a97f72c3e..c764158c37a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3852,7 +3852,6 @@ dependencies = [ "fuel-core-storage", "fuel-core-types 0.40.0", "fuel-core-wasm-executor", - "ntest", "parking_lot", "postcard", "tracing", @@ -6312,39 +6311,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "ntest" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb183f0a1da7a937f672e5ee7b7edb727bf52b8a52d531374ba8ebb9345c0330" -dependencies = [ - "ntest_test_cases", - "ntest_timeout", -] - -[[package]] -name = "ntest_test_cases" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d0d3f2a488592e5368ebbe996e7f1d44aa13156efad201f5b4d84e150eaa93" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "ntest_timeout" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc7c92f190c97f79b4a332f5e81dcf68c8420af2045c936c9be0bc9de6f63b5" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "nu-ansi-term" version = "0.46.0" diff --git a/crates/services/upgradable-executor/Cargo.toml b/crates/services/upgradable-executor/Cargo.toml index 4b340df0a25..16eeb0fc391 100644 --- a/crates/services/upgradable-executor/Cargo.toml +++ b/crates/services/upgradable-executor/Cargo.toml @@ -34,7 +34,6 @@ wasmtime = { version = "23.0.2", default-features = false, features = [ anyhow = { workspace = true } fuel-core-storage = { workspace = true, features = ["test-helpers"] } fuel-core-types = { workspace = true, features = ["test-helpers"] } -ntest = "0.9.2" [build-dependencies] fuel-core-wasm-executor = { workspace = true, optional = true, default-features = false } diff --git a/crates/services/upgradable-executor/src/executor.rs b/crates/services/upgradable-executor/src/executor.rs index c94ec19488f..c9140b7a63f 100644 --- a/crates/services/upgradable-executor/src/executor.rs +++ b/crates/services/upgradable-executor/src/executor.rs @@ -739,9 +739,6 @@ where #[allow(unexpected_cfgs)] // for cfg(coverage) #[cfg(test)] mod test { - #[cfg(coverage)] - use ntest as _; // Only used outside cdg(coverage) - use super::*; use fuel_core_storage::{ kv_store::Value, @@ -946,7 +943,6 @@ mod test { mod native { use super::*; use crate::executor::Executor; - use ntest as _; #[test] fn can_validate_block() { @@ -1111,7 +1107,6 @@ mod test { // If it doesn't cache the modules, the test will fail with a timeout. #[test] #[cfg(not(coverage))] // Too slow for coverage - #[ntest::timeout(90_000)] fn reuse_cached_compiled_module__native_strategy() { // Given let next_version = Executor::::VERSION + 1; @@ -1119,10 +1114,15 @@ mod test { let executor = Executor::native(storage, DisabledRelayer, Config::default()); let block = valid_block(next_version); + executor.validate(&block).map(|_| ()).unwrap(); + let start = std::time::Instant::now(); // When for _ in 0..1000 { let result = executor.validate(&block).map(|_| ()); + if start.elapsed().as_secs() > 60 { + panic!("The test is too slow"); + } // Then assert_eq!(Ok(()), result); } @@ -1132,7 +1132,6 @@ mod test { // If it doesn't cache the modules, the test will fail with a timeout. #[test] #[cfg(not(coverage))] // Too slow for coverage - #[ntest::timeout(90_000)] fn reuse_cached_compiled_module__wasm_strategy() { // Given let next_version = Executor::::VERSION + 1; @@ -1140,10 +1139,15 @@ mod test { let executor = Executor::wasm(storage, DisabledRelayer, Config::default()); let block = valid_block(next_version); + executor.validate(&block).map(|_| ()).unwrap(); + let start = std::time::Instant::now(); // When for _ in 0..1000 { let result = executor.validate(&block).map(|_| ()); + if start.elapsed().as_secs() > 60 { + panic!("The test is too slow"); + } // Then assert_eq!(Ok(()), result); } From f32a3e3964d1e117e28d0c8dd2882b7c56eb832f Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Thu, 14 Nov 2024 00:08:31 +0100 Subject: [PATCH 36/45] Update timeouts and remove new behavior discovery_works --- bin/e2e-test-client/src/config.rs | 2 +- bin/e2e-test-client/src/tests/transfers.rs | 2 +- crates/services/p2p/src/discovery.rs | 2 +- crates/services/upgradable-executor/src/executor.rs | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/e2e-test-client/src/config.rs b/bin/e2e-test-client/src/config.rs index 9d537043351..742cb3fd309 100644 --- a/bin/e2e-test-client/src/config.rs +++ b/bin/e2e-test-client/src/config.rs @@ -39,7 +39,7 @@ impl Default for SuiteConfig { fn default() -> Self { Self { endpoint: "http://localhost:4000".to_string(), - wallet_sync_timeout: SYNC_TIMEOUT, + wallet_sync_timeout: SYNC_TIMEOUT.checked_mul(2).unwrap(), full_test: false, coinbase_contract_id: ContractId::from_str( "0x7777777777777777777777777777777777777777777777777777777777777777", diff --git a/bin/e2e-test-client/src/tests/transfers.rs b/bin/e2e-test-client/src/tests/transfers.rs index 5bede8e0ace..b9619453649 100644 --- a/bin/e2e-test-client/src/tests/transfers.rs +++ b/bin/e2e-test-client/src/tests/transfers.rs @@ -49,7 +49,7 @@ pub async fn transfer_back(ctx: &TestContext) -> Result<(), Failed> { } // wait until alice sees the transaction timeout( - ctx.config.sync_timeout().checked_mul(2).unwrap(), + ctx.config.sync_timeout(), ctx.alice.client.await_transaction_commit(&result.tx_id), ) .await??; diff --git a/crates/services/p2p/src/discovery.rs b/crates/services/p2p/src/discovery.rs index 7cfe3ca25e7..e41d228883d 100644 --- a/crates/services/p2p/src/discovery.rs +++ b/crates/services/p2p/src/discovery.rs @@ -368,7 +368,7 @@ mod tests { .add_address(&peer_id, unroutable_peer_addr.clone()); } SwarmEvent::ConnectionClosed { peer_id, .. } => { - left_to_discover[swarm_index].insert(peer_id); + dbg!(peer_id); } _ => {} } diff --git a/crates/services/upgradable-executor/src/executor.rs b/crates/services/upgradable-executor/src/executor.rs index c9140b7a63f..2b8c72fa5af 100644 --- a/crates/services/upgradable-executor/src/executor.rs +++ b/crates/services/upgradable-executor/src/executor.rs @@ -1115,12 +1115,12 @@ mod test { let block = valid_block(next_version); executor.validate(&block).map(|_| ()).unwrap(); - let start = std::time::Instant::now(); // When for _ in 0..1000 { + let start = std::time::Instant::now(); let result = executor.validate(&block).map(|_| ()); - if start.elapsed().as_secs() > 60 { + if start.elapsed().as_secs() > 1 { panic!("The test is too slow"); } // Then @@ -1140,12 +1140,12 @@ mod test { let block = valid_block(next_version); executor.validate(&block).map(|_| ()).unwrap(); - let start = std::time::Instant::now(); // When for _ in 0..1000 { + let start = std::time::Instant::now(); let result = executor.validate(&block).map(|_| ()); - if start.elapsed().as_secs() > 60 { + if start.elapsed().as_secs() > 1 { panic!("The test is too slow"); } // Then From 596dc9036fe3782b491b556412cbff9c0c6ef06f Mon Sep 17 00:00:00 2001 From: AurelienFT Date: Thu, 14 Nov 2024 09:11:37 +0100 Subject: [PATCH 37/45] edit snapshot --- ...core_e2e_client__config__tests__default_config_snapshot.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/e2e-test-client/src/snapshots/fuel_core_e2e_client__config__tests__default_config_snapshot.snap b/bin/e2e-test-client/src/snapshots/fuel_core_e2e_client__config__tests__default_config_snapshot.snap index e56ef483f66..4a38c16f0c4 100644 --- a/bin/e2e-test-client/src/snapshots/fuel_core_e2e_client__config__tests__default_config_snapshot.snap +++ b/bin/e2e-test-client/src/snapshots/fuel_core_e2e_client__config__tests__default_config_snapshot.snap @@ -3,7 +3,7 @@ source: bin/e2e-test-client/src/config.rs expression: serialized --- endpoint = "http://localhost:4000" -wallet_sync_timeout = "10s" +wallet_sync_timeout = "20s" full_test = false coinbase_contract_id = "7777777777777777777777777777777777777777777777777777777777777777" From b3677ef9cb24d322c2bf82fdcb50a61b311e65fa Mon Sep 17 00:00:00 2001 From: green Date: Sat, 16 Nov 2024 21:52:50 -0500 Subject: [PATCH 38/45] Speed up tests related to maximum number of transactions allowed by executor(it uses 1024). Speed up state rewind test by using less blocks to re-run. Speedup gas price test by using less maximum block height. Speed up all tests by not creating all family colums in the RocksDB database. Related issue: https://github.com/facebook/rocksdb/issues/5117 Fixed deploy large contract e2e test(before it was not tested). Buffered dry runs in e2e tests to avoid resource exhaustion. Use interval in e2e tests, to avoid issues with block broadcasting. --- .config/nextest.toml | 4 -- bin/e2e-test-client/src/lib.rs | 2 +- bin/e2e-test-client/src/tests/script.rs | 9 ++-- .../test_data/large_state/state_config.json | 3 +- .../tests/integration_tests.rs | 11 +++- crates/fuel-core/src/executor.rs | 10 ++-- crates/fuel-core/src/state/rocks_db.rs | 50 +++++++++++++++---- crates/services/executor/src/executor.rs | 13 +++-- tests/Cargo.toml | 2 +- tests/tests/blocks.rs | 8 +-- tests/tests/coins.rs | 1 + tests/tests/recovery.rs | 20 +++++--- tests/tests/state_rewind.rs | 2 +- 13 files changed, 92 insertions(+), 43 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 96b273f5fc5..47e5bf53d0f 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,6 +1,2 @@ [profile.default] slow-timeout = { period = "60s", terminate-after = 5 } - -[[profile.default.overrides]] -filter = 'test(too_many_transactions_are_split_in_blocks) + test(block_producer_never_includes_more_than_max_tx_count_transactions)' -slow-timeout = { period = "60s", terminate-after = 20 } \ No newline at end of file diff --git a/bin/e2e-test-client/src/lib.rs b/bin/e2e-test-client/src/lib.rs index 36ee3ea5413..1dbec93fb1c 100644 --- a/bin/e2e-test-client/src/lib.rs +++ b/bin/e2e-test-client/src/lib.rs @@ -115,7 +115,7 @@ pub fn main_body(config: SuiteConfig, mut args: Arguments) { with_cloned(&config, |config| { async_execute(async { let ctx = TestContext::new(config).await; - tests::transfers::transfer_back(&ctx).await + tests::contracts::deploy_large_contract(&ctx).await }) }), ), diff --git a/bin/e2e-test-client/src/tests/script.rs b/bin/e2e-test-client/src/tests/script.rs index 3c5b8a26f83..d0bed59d178 100644 --- a/bin/e2e-test-client/src/tests/script.rs +++ b/bin/e2e-test-client/src/tests/script.rs @@ -23,6 +23,7 @@ use fuel_core_types::{ }, services::executor::TransactionExecutionResult, }; +use futures::StreamExt; use libtest_mimic::Failed; use std::{ path::Path, @@ -199,10 +200,12 @@ async fn _dry_runs( }); } + let stream = futures::stream::iter(queries.into_iter()) + .buffered(10) + .collect::>(); + // All queries should be resolved for 90 seconds. - let queries = - tokio::time::timeout(Duration::from_secs(90), futures::future::join_all(queries)) - .await?; + let queries = tokio::time::timeout(Duration::from_secs(90), stream).await?; let chain_info = ctx.alice.client.chain_info().await?; for query in queries { diff --git a/bin/e2e-test-client/src/tests/test_data/large_state/state_config.json b/bin/e2e-test-client/src/tests/test_data/large_state/state_config.json index 0be6bf6e091..1cf35d39097 100644 --- a/bin/e2e-test-client/src/tests/test_data/large_state/state_config.json +++ b/bin/e2e-test-client/src/tests/test_data/large_state/state_config.json @@ -118150,6 +118150,5 @@ } ] } - ], - "last_block": null + ] } diff --git a/bin/e2e-test-client/tests/integration_tests.rs b/bin/e2e-test-client/tests/integration_tests.rs index a6a13f75f2a..bb62d4e8dc3 100644 --- a/bin/e2e-test-client/tests/integration_tests.rs +++ b/bin/e2e-test-client/tests/integration_tests.rs @@ -4,15 +4,20 @@ use fuel_core::service::{ }; // Add methods on commands +use fuel_core::service::config::Trigger; use fuel_core_chain_config::{ SnapshotMetadata, SnapshotReader, }; use fuel_core_e2e_client::config::SuiteConfig; -use fuel_core_types::fuel_tx::ContractId; +use fuel_core_types::{ + blockchain::header::LATEST_STATE_TRANSITION_VERSION, + fuel_tx::ContractId, +}; use std::{ fs, str::FromStr, + time::Duration, }; use tempfile::TempDir; // Used for writing assertions // Run programs @@ -127,6 +132,7 @@ fn dev_config() -> Config { } chain_config.state_transition_bytecode = fuel_core::upgradable_executor::WASM_BYTECODE.to_vec(); + chain_config.genesis_state_transition_version = Some(LATEST_STATE_TRANSITION_VERSION); let reader = reader.with_chain_config(chain_config); let mut config = Config::local_node_with_reader(reader); @@ -137,6 +143,9 @@ fn dev_config() -> Config { ) .unwrap(), ); + config.block_production = Trigger::Interval { + block_time: Duration::from_secs(1), + }; config } diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index c951a711549..067c41cc486 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -7,8 +7,8 @@ mod tests { use fuel_core::database::Database; use fuel_core_executor::{ executor::{ + max_tx_count, OnceTransactionsSource, - MAX_TX_COUNT, }, ports::{ MaybeCheckedTransaction, @@ -3032,7 +3032,7 @@ mod tests { let mut consensus_parameters = ConsensusParameters::default(); // Given - let transactions_in_tx_source = (MAX_TX_COUNT as usize) + 10; + let transactions_in_tx_source = (max_tx_count() as usize) + 10; consensus_parameters.set_block_gas_limit(u64::MAX); let config = Config { consensus_parameters, @@ -3056,7 +3056,7 @@ mod tests { // Then assert_eq!( result.block.transactions().len(), - (MAX_TX_COUNT as usize + 1) + (max_tx_count() as usize + 1) ); } @@ -3069,7 +3069,7 @@ mod tests { let mut consensus_parameters = ConsensusParameters::default(); // Given - let transactions_in_tx_source = (MAX_TX_COUNT as usize) + 10; + let transactions_in_tx_source = (max_tx_count() as usize) + 10; consensus_parameters.set_block_gas_limit(u64::MAX); let config = Config { consensus_parameters, @@ -3101,7 +3101,7 @@ mod tests { // Then assert_eq!( result.block.transactions().len(), - (MAX_TX_COUNT as usize + 1) + (max_tx_count() as usize + 1) ); } diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index d2a9f7bfc9c..9d4b7058a40 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -57,7 +57,10 @@ use std::{ Path, PathBuf, }, - sync::Arc, + sync::{ + Arc, + Mutex, + }, }; use tempfile::TempDir; @@ -95,6 +98,7 @@ impl Drop for DropResources { pub struct RocksDb { read_options: ReadOptions, db: Arc, + create_family: Arc>>, snapshot: Option>, metrics: Arc, // used for RAII @@ -270,7 +274,6 @@ where block_opts.set_block_size(16 * 1024); let mut opts = Options::default(); - opts.create_if_missing(true); opts.set_compression_type(DBCompressionType::Lz4); // TODO: Make it customizable https://github.com/FuelLabs/fuel-core/issues/1666 opts.set_max_total_wal_size(64 * 1024 * 1024); @@ -304,6 +307,10 @@ where } } + if cf_descriptors_to_open.is_empty() { + opts.create_if_missing(true); + } + let unknown_columns_to_open: BTreeMap<_, _> = existing_column_families .iter() .filter(|column_name| { @@ -341,22 +348,19 @@ where } .map_err(|e| DatabaseError::Other(e.into()))?; - // Setup cfs - for (name, opt) in cf_descriptors_to_create { - db.create_cf(name, &opt) - .map_err(|e| DatabaseError::Other(e.into()))?; - } - let db = Arc::new(db); + let create_family = Arc::new(Mutex::new(cf_descriptors_to_create)); let rocks_db = RocksDb { read_options: Self::generate_read_options(&None), snapshot: None, db, metrics, + create_family, _drop: Default::default(), _marker: Default::default(), }; + Ok(rocks_db) } @@ -383,6 +387,7 @@ where &self, ) -> RocksDb { let db = self.db.clone(); + let create_family = self.create_family.clone(); let metrics = self.metrics.clone(); let _drop = self._drop.clone(); @@ -402,6 +407,7 @@ where read_options: Self::generate_read_options(&snapshot), snapshot, db, + create_family, metrics, _drop, _marker: Default::default(), @@ -413,9 +419,31 @@ where } fn cf_u32(&self, column: u32) -> Arc { - self.db - .cf_handle(&Self::col_name(column)) - .expect("invalid column state") + let family = self.db.cf_handle(&Self::col_name(column)); + + match family { + None => { + let mut lock = self + .create_family + .lock() + .expect("The create family lock should be available"); + + let name = Self::col_name(column); + let family = lock.remove(&name).expect("invalid column state"); + + self.db + .create_cf(&name, &family) + .expect("Couldn't create column family"); + + let family = self + .db + .cf_handle(&Self::col_name(column)) + .expect("invalid column state"); + + family + } + Some(family) => family, + } } fn col_name(column: u32) -> String { diff --git a/crates/services/executor/src/executor.rs b/crates/services/executor/src/executor.rs index 2e9b6ac6e0d..bc690c52d3d 100644 --- a/crates/services/executor/src/executor.rs +++ b/crates/services/executor/src/executor.rs @@ -164,7 +164,14 @@ use alloc::{ /// The maximum amount of transactions that can be included in a block, /// excluding the mint transaction. -pub const MAX_TX_COUNT: u16 = u16::MAX.saturating_sub(1); +#[cfg(not(feature = "test-helpers"))] +pub const fn max_tx_count() -> u16 { + u16::MAX.saturating_sub(1) +} +#[cfg(feature = "test-helpers")] +pub const fn max_tx_count() -> u16 { + 1024 +} pub struct OnceTransactionsSource { transactions: ParkingMutex>, @@ -593,7 +600,7 @@ where // When processing l2 transactions, we must take into account transactions from the l1 // that have been included in the block already (stored in `data.tx_count`), as well // as the final mint transaction. - let mut remaining_tx_count = MAX_TX_COUNT.saturating_sub(data.tx_count); + let mut remaining_tx_count = max_tx_count().saturating_sub(data.tx_count); let mut regular_tx_iter = l2_tx_source .next( @@ -636,7 +643,7 @@ where remaining_gas_limit = block_gas_limit.saturating_sub(data.used_gas); remaining_block_transaction_size_limit = block_transaction_size_limit.saturating_sub(data.used_size); - remaining_tx_count = MAX_TX_COUNT.saturating_sub(data.tx_count); + remaining_tx_count = max_tx_count().saturating_sub(data.tx_count); } regular_tx_iter = l2_tx_source diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 3c4ad33502e..aae69baf96e 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -35,7 +35,7 @@ fuel-core-benches = { path = "../benches" } fuel-core-bin = { path = "../bin/fuel-core", features = ["parquet", "p2p"] } fuel-core-client = { path = "../crates/client", features = ["test-helpers"] } fuel-core-compression = { path = "../crates/compression" } -fuel-core-executor = { workspace = true } +fuel-core-executor = { workspace = true, features = ["test-helpers"] } fuel-core-gas-price-service = { path = "../crates/services/gas_price_service" } fuel-core-p2p = { path = "../crates/services/p2p", features = [ "test-helpers", diff --git a/tests/tests/blocks.rs b/tests/tests/blocks.rs index 7a5ba4688d5..35e40130373 100644 --- a/tests/tests/blocks.rs +++ b/tests/tests/blocks.rs @@ -365,7 +365,7 @@ mod full_block { }, FuelClient, }; - use fuel_core_executor::executor; + use fuel_core_executor::executor::max_tx_count; use fuel_core_txpool::config::{ HeavyWorkConfig, PoolLimits, @@ -479,7 +479,7 @@ mod full_block { let srv = FuelService::new_node(patched_node_config).await.unwrap(); let client = FuelClient::from(srv.bound_address); - let tx_count: u64 = 66_000; + let tx_count: u64 = max_tx_count() as u64 + 100; let txs = (1..=tx_count) .map(|i| test_helpers::make_tx(&mut rng, i, max_gas_limit)) .collect_vec(); @@ -505,11 +505,11 @@ mod full_block { assert_eq!( second_last_block.transactions.len(), - executor::MAX_TX_COUNT as usize + 1 // Mint transaction for one block + max_tx_count() as usize + 1 // Mint transaction for one block ); assert_eq!( last_block.transactions.len(), - (tx_count as usize - (executor::MAX_TX_COUNT as usize)) + 1 /* Mint transaction for second block */ + (tx_count as usize - (max_tx_count() as usize)) + 1 /* Mint transaction for second block */ ); } } diff --git a/tests/tests/coins.rs b/tests/tests/coins.rs index 6a03bcd791d..60237b52d4a 100644 --- a/tests/tests/coins.rs +++ b/tests/tests/coins.rs @@ -157,6 +157,7 @@ mod coin { ) .await .unwrap(); + tracing::info!("coins_per_asset: {:?}", coins_per_asset); assert_eq!(coins_per_asset.len(), 2); assert_eq!(coins_per_asset[0].len(), 1); assert!(coins_per_asset[0].amount() >= 1); diff --git a/tests/tests/recovery.rs b/tests/tests/recovery.rs index 623c5b832f1..60806b58f05 100644 --- a/tests/tests/recovery.rs +++ b/tests/tests/recovery.rs @@ -1,5 +1,6 @@ #![allow(non_snake_case)] +use clap::Parser; use fuel_core_storage::transactional::HistoricalView; use fuel_core_types::fuel_types::BlockHeight; use proptest::{ @@ -66,7 +67,7 @@ async fn off_chain_worker_can_recover_on_start_up_when_is_behind() -> anyhow::Re } prop_compose! { - fn height_and_lower_height()(height in 2..100u32)(height in Just(height), lower_height in 1..height) -> (u32, u32) { + fn height_and_lower_height()(height in 2..15u32)(height in Just(height), lower_height in 1..height) -> (u32, u32) { (height, lower_height) } } @@ -91,13 +92,18 @@ async fn _gas_price_updater__can_recover_on_startup_when_gas_price_db_is_ahead( database.on_chain().latest_height(), Some(BlockHeight::new(height)) ); - let diff = height - lower_height; - for _ in 0..diff { - database.on_chain().rollback_last_block()?; - database.off_chain().rollback_last_block()?; - } - assert!(database.on_chain().latest_height() < database.gas_price().latest_height()); let temp_dir = driver.kill().await; + let target_block_height = lower_height.to_string(); + let args = [ + "_IGNORED_", + "--db-path", + temp_dir.path().to_str().unwrap(), + "--target-block-height", + target_block_height.as_str(), + ]; + let command = fuel_core_bin::cli::rollback::Command::parse_from(args); + tracing::info!("Rolling back to block {}", target_block_height); + fuel_core_bin::cli::rollback::exec(command).await?; // When let recovered_driver = FuelCoreDriver::spawn_with_directory( diff --git a/tests/tests/state_rewind.rs b/tests/tests/state_rewind.rs index f8553f351f5..3aea2950050 100644 --- a/tests/tests/state_rewind.rs +++ b/tests/tests/state_rewind.rs @@ -77,7 +77,7 @@ async fn validate_block_at_any_height__only_transfers() -> anyhow::Result<()> { let node = &driver.node; // Given - const TOTAL_BLOCKS: u64 = 5000; + const TOTAL_BLOCKS: u64 = 1000; const MIN_AMOUNT: u64 = 123456; let mut last_block_height = 0u32; let mut database_modifications = std::collections::HashMap::new(); From 4e7e8da3b9534d0d5350acc2b55ee82b1e5d68e3 Mon Sep 17 00:00:00 2001 From: green Date: Sat, 16 Nov 2024 22:00:25 -0500 Subject: [PATCH 39/45] Fixed race condition --- crates/fuel-core/src/state/rocks_db.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 9d4b7058a40..513a7feaf2c 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -429,16 +429,18 @@ where .expect("The create family lock should be available"); let name = Self::col_name(column); - let family = lock.remove(&name).expect("invalid column state"); + let Some(family) = lock.remove(&name) else { + return self + .db + .cf_handle(&Self::col_name(column)) + .expect("No column family found"); + }; self.db .create_cf(&name, &family) .expect("Couldn't create column family"); - let family = self - .db - .cf_handle(&Self::col_name(column)) - .expect("invalid column state"); + let family = self.db.cf_handle(&name).expect("invalid column state"); family } From d0fcad9f748c3a78943abe28833400c7d7f15c05 Mon Sep 17 00:00:00 2001 From: green Date: Sat, 16 Nov 2024 22:43:54 -0500 Subject: [PATCH 40/45] Fix tests in CI. Also run p2p only once. --- benches/src/db_lookup_times_utils/mod.rs | 12 ++-- crates/fuel-core/src/executor.rs | 1 + tests/Cargo.toml | 10 +-- tests/tests/aws_kms.rs | 80 ++++++++++++++++++++++++ tests/tests/lib.rs | 41 ++++++++++-- tests/tests/poa.rs | 77 ----------------------- 6 files changed, 126 insertions(+), 95 deletions(-) create mode 100644 tests/tests/aws_kms.rs diff --git a/benches/src/db_lookup_times_utils/mod.rs b/benches/src/db_lookup_times_utils/mod.rs index 0d001a3bc3e..b446a52d00f 100644 --- a/benches/src/db_lookup_times_utils/mod.rs +++ b/benches/src/db_lookup_times_utils/mod.rs @@ -14,20 +14,16 @@ mod tests { }; use fuel_core::state::rocks_db::RocksDb; - use crate::{ - db_lookup_times_utils::seed::{ - insert_compressed_block, - insert_full_block, - }, - utils::ShallowTempDir, + use crate::db_lookup_times_utils::seed::{ + insert_compressed_block, + insert_full_block, }; const TEST_HEIGHT: u32 = 1; const TEST_TX_COUNT: u32 = 10; fn setup_test_db() -> RocksDb { - let temp_dir = ShallowTempDir::new(); - RocksDb::default_open(temp_dir.path(), None, -1).unwrap() + RocksDb::default_open_temp(None).unwrap() } #[test] diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 067c41cc486..8ace3ccf318 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -3061,6 +3061,7 @@ mod tests { } #[test] + #[cfg(not(feature = "wasm-executor"))] fn block_producer_never_includes_more_than_max_tx_count_transactions_with_bad_tx_source( ) { let block_height = 1u32; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index aae69baf96e..291f41ab7d7 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -28,6 +28,8 @@ cynic = { workspace = true } ethers = "2" fuel-core = { path = "../crates/fuel-core", default-features = false, features = [ "smt", + "p2p", + "relayer", "wasm-executor", "test-helpers", ] } @@ -43,7 +45,7 @@ fuel-core-p2p = { path = "../crates/services/p2p", features = [ fuel-core-poa = { path = "../crates/services/consensus_module/poa" } fuel-core-relayer = { path = "../crates/services/relayer", features = [ "test-helpers", -], optional = true } +] } fuel-core-storage = { path = "../crates/storage", features = ["test-helpers"] } fuel-core-trace = { path = "../crates/trace" } fuel-core-txpool = { path = "../crates/services/txpool_v2", features = [ @@ -79,8 +81,6 @@ proptest = { workspace = true } tracing = { workspace = true } [features] -default = ["fuel-core/default", "relayer"] -p2p = ["fuel-core/p2p", "fuel-core-p2p"] -relayer = ["fuel-core/relayer", "fuel-core-relayer"] -wasm-executor = ["fuel-core/wasm-executor"] +default = ["fuel-core/default"] +only-p2p = ["fuel-core-p2p"] aws-kms = ["dep:aws-config", "dep:aws-sdk-kms", "fuel-core-bin/aws-kms"] diff --git a/tests/tests/aws_kms.rs b/tests/tests/aws_kms.rs new file mode 100644 index 00000000000..31dcb8e0448 --- /dev/null +++ b/tests/tests/aws_kms.rs @@ -0,0 +1,80 @@ +use fuel_core::combined_database::CombinedDatabase; +use fuel_core_storage::transactional::AtomicView; +use fuel_core_types::blockchain::consensus::Consensus; +use test_helpers::fuel_core_driver::FuelCoreDriver; + +#[tokio::test] +async fn can_get_sealed_block_from_poa_produced_block_when_signing_with_kms() { + use fuel_core_types::fuel_crypto::PublicKey; + use k256::pkcs8::DecodePublicKey; + + // This test is only enabled if the environment variable is set + let Some(kms_arn) = option_env!("FUEL_CORE_TEST_AWS_KMS_ARN") else { + return; + }; + + // Get the public key for the KMS key + let config = aws_config::load_from_env().await; + let kms_client = aws_sdk_kms::Client::new(&config); + let poa_public_der = kms_client + .get_public_key() + .key_id(kms_arn) + .send() + .await + .expect("Unable to fetch public key from KMS") + .public_key + .unwrap() + .into_inner(); + let poa_public = k256::PublicKey::from_public_key_der(&poa_public_der) + .expect("invalid DER public key from AWS KMS"); + let poa_public = PublicKey::from(poa_public); + + // start node with the kms enabled and produce some blocks + let num_blocks = 100; + let args = vec![ + "--debug", + "--poa-instant", + "true", + "--consensus-aws-kms", + kms_arn, + ]; + let driver = FuelCoreDriver::spawn(&args).await.unwrap(); + let _ = driver + .client + .produce_blocks(num_blocks, None) + .await + .unwrap(); + + // stop the node and just grab the database + let db_path = driver.kill().await; + let db = CombinedDatabase::open(db_path.path(), 1024 * 1024, Default::default(), 512) + .unwrap(); + + let view = db.on_chain().latest_view().unwrap(); + + // verify signatures and ensure that the block producer wont change + let mut block_producer = None; + for height in 1..=num_blocks { + let sealed_block = view + .get_sealed_block_by_height(&height.into()) + .unwrap() + .expect("expected sealed block to be available"); + let block_id = sealed_block.entity.id(); + let signature = match sealed_block.consensus { + Consensus::PoA(ref poa) => poa.signature, + _ => panic!("Not expected consensus"), + }; + signature + .verify(&poa_public, &block_id.into_message()) + .expect("failed to verify signature"); + let this_bp = sealed_block + .consensus + .block_producer(&block_id) + .expect("Block should have a block producer"); + if let Some(bp) = block_producer { + assert_eq!(bp, this_bp, "Block producer changed"); + } else { + block_producer = Some(this_bp); + } + } +} diff --git a/tests/tests/lib.rs b/tests/tests/lib.rs index 5337e134358..c42218b045f 100644 --- a/tests/tests/lib.rs +++ b/tests/tests/lib.rs @@ -1,38 +1,69 @@ #![deny(unused_must_use)] #![deny(warnings)] +#[cfg(not(feature = "only-p2p"))] mod balances; +#[cfg(not(feature = "only-p2p"))] mod blob; +#[cfg(not(feature = "only-p2p"))] mod blocks; +#[cfg(not(feature = "only-p2p"))] mod chain; +#[cfg(not(feature = "only-p2p"))] mod coin; +#[cfg(not(feature = "only-p2p"))] mod coins; +#[cfg(not(feature = "only-p2p"))] mod contract; +#[cfg(not(feature = "only-p2p"))] mod da_compression; +#[cfg(not(feature = "only-p2p"))] mod dap; +#[cfg(not(feature = "only-p2p"))] mod debugger; +#[cfg(not(feature = "only-p2p"))] mod dos; +#[cfg(not(feature = "only-p2p"))] mod fee_collection_contract; +#[cfg(not(feature = "only-p2p"))] mod gas_price; +#[cfg(not(feature = "only-p2p"))] mod health; +#[cfg(not(feature = "only-p2p"))] mod helpers; +#[cfg(not(feature = "only-p2p"))] mod local_node; +#[cfg(not(feature = "only-p2p"))] mod messages; +#[cfg(not(feature = "only-p2p"))] mod metrics; +#[cfg(not(feature = "only-p2p"))] mod node_info; +#[cfg(not(feature = "only-p2p"))] mod poa; +#[cfg(not(feature = "only-p2p"))] mod recovery; +#[cfg(not(feature = "only-p2p"))] mod regenesis; -#[cfg(feature = "relayer")] +#[cfg(not(feature = "only-p2p"))] mod relayer; +#[cfg(not(feature = "only-p2p"))] mod snapshot; +#[cfg(not(feature = "only-p2p"))] mod state_rewind; -#[cfg(feature = "p2p")] -mod sync; +#[cfg(not(feature = "only-p2p"))] mod trigger_integration; +#[cfg(not(feature = "only-p2p"))] mod tx; -#[cfg(feature = "p2p")] -mod tx_gossip; +#[cfg(not(feature = "only-p2p"))] mod vm_storage; +#[cfg(feature = "only-p2p")] +mod sync; +#[cfg(feature = "only-p2p")] +mod tx_gossip; + +#[cfg(feature = "aws-kms")] +mod aws_kms; + fuel_core_trace::enable_tracing!(); diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index 85334b005da..2de2ee4e82c 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -92,83 +92,6 @@ async fn can_get_sealed_block_from_poa_produced_block() { .expect("failed to verify signature"); } -#[tokio::test] -#[cfg(feature = "aws-kms")] -async fn can_get_sealed_block_from_poa_produced_block_when_signing_with_kms() { - use fuel_core_types::fuel_crypto::PublicKey; - use k256::pkcs8::DecodePublicKey; - - // This test is only enabled if the environment variable is set - let Some(kms_arn) = option_env!("FUEL_CORE_TEST_AWS_KMS_ARN") else { - return; - }; - - // Get the public key for the KMS key - let config = aws_config::load_from_env().await; - let kms_client = aws_sdk_kms::Client::new(&config); - let poa_public_der = kms_client - .get_public_key() - .key_id(kms_arn) - .send() - .await - .expect("Unable to fetch public key from KMS") - .public_key - .unwrap() - .into_inner(); - let poa_public = k256::PublicKey::from_public_key_der(&poa_public_der) - .expect("invalid DER public key from AWS KMS"); - let poa_public = PublicKey::from(poa_public); - - // start node with the kms enabled and produce some blocks - let num_blocks = 100; - let args = vec![ - "--debug", - "--poa-instant", - "true", - "--consensus-aws-kms", - kms_arn, - ]; - let driver = FuelCoreDriver::spawn(&args).await.unwrap(); - let _ = driver - .client - .produce_blocks(num_blocks, None) - .await - .unwrap(); - - // stop the node and just grab the database - let db_path = driver.kill().await; - let db = CombinedDatabase::open(db_path.path(), 1024 * 1024, Default::default(), 512) - .unwrap(); - - let view = db.on_chain().latest_view().unwrap(); - - // verify signatures and ensure that the block producer wont change - let mut block_producer = None; - for height in 1..=num_blocks { - let sealed_block = view - .get_sealed_block_by_height(&height.into()) - .unwrap() - .expect("expected sealed block to be available"); - let block_id = sealed_block.entity.id(); - let signature = match sealed_block.consensus { - Consensus::PoA(ref poa) => poa.signature, - _ => panic!("Not expected consensus"), - }; - signature - .verify(&poa_public, &block_id.into_message()) - .expect("failed to verify signature"); - let this_bp = sealed_block - .consensus - .block_producer(&block_id) - .expect("Block should have a block producer"); - if let Some(bp) = block_producer { - assert_eq!(bp, this_bp, "Block producer changed"); - } else { - block_producer = Some(this_bp); - } - } -} - #[tokio::test(flavor = "multi_thread")] async fn starting_node_with_predefined_nodes_produces_these_predefined_blocks( ) -> anyhow::Result<()> { From b3c9672ed868063a1c49f16142426a06204fb7f7 Mon Sep 17 00:00:00 2001 From: green Date: Sat, 16 Nov 2024 22:58:47 -0500 Subject: [PATCH 41/45] Make CI happy and update ci checks. --- Makefile.toml | 2 +- ci_checks.sh | 13 ++++---- crates/fuel-core/src/executor.rs | 56 ++++++++++++++++---------------- 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index 9494fce12af..4c7e5aa3ba2 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -35,4 +35,4 @@ env = { "CARGO_MAKE_WORKSPACE_SKIP_MEMBERS" = ["fuel-core-tests"] } description = "Run tests on each workspace member" category = "Test" command = "cargo" -args = ["test", "${@}"] +args = ["nextest", "run", "${@}"] diff --git a/ci_checks.sh b/ci_checks.sh index b948a32ce09..fa5454366dd 100755 --- a/ci_checks.sh +++ b/ci_checks.sh @@ -11,6 +11,7 @@ # - `cargo install cargo-sort` # - `cargo install cargo-make` # - `cargo install cargo-insta` +# - `cargo install cargo-nextest` # - `npm install prettier prettier-plugin-toml` npx prettier --write "**/Cargo.toml" && @@ -29,9 +30,9 @@ cargo check -p fuel-core-executor --target wasm32-unknown-unknown --no-default-f cargo make check --all-features --locked && cargo make check --locked && OVERRIDE_CHAIN_CONFIGS=true cargo test --test integration_tests local_node && -cargo test --workspace && -FUEL_ALWAYS_USE_WASM=true cargo test --all-features --workspace && -cargo test -p fuel-core --no-default-features && -cargo test -p fuel-core-client --no-default-features && -cargo test -p fuel-core-chain-config --no-default-features && -cargo test --manifest-path version-compatibility/Cargo.toml --workspace +cargo nextest run --workspace && +FUEL_ALWAYS_USE_WASM=true cargo nextest run --all-features --workspace && +cargo nextest run -p fuel-core --no-default-features && +cargo nextest run -p fuel-core-client --no-default-features && +cargo nextest run -p fuel-core-chain-config --no-default-features && +cargo nextest run --manifest-path version-compatibility/Cargo.toml --workspace diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 8ace3ccf318..a7f462416fa 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -13,7 +13,6 @@ mod tests { ports::{ MaybeCheckedTransaction, RelayerPort, - TransactionsSource, }, refs::ContractRef, }; @@ -151,7 +150,6 @@ mod tests { Rng, SeedableRng, }; - use std::sync::Mutex; #[derive(Clone, Debug, Default)] struct Config { @@ -186,32 +184,6 @@ mod tests { } } - /// Bad transaction source: ignores the limit of `u16::MAX -1` transactions - /// that should be returned by [`TransactionsSource::next()`]. - /// It is used only for testing purposes - pub struct BadTransactionsSource { - transactions: Mutex>, - } - - impl BadTransactionsSource { - pub fn new(transactions: Vec) -> Self { - Self { - transactions: Mutex::new( - transactions - .into_iter() - .map(MaybeCheckedTransaction::Transaction) - .collect(), - ), - } - } - } - - impl TransactionsSource for BadTransactionsSource { - fn next(&self, _: u64, _: u16, _: u32) -> Vec { - std::mem::take(&mut *self.transactions.lock().unwrap()) - } - } - fn add_consensus_parameters( mut database: Database, consensus_parameters: &ConsensusParameters, @@ -3064,6 +3036,34 @@ mod tests { #[cfg(not(feature = "wasm-executor"))] fn block_producer_never_includes_more_than_max_tx_count_transactions_with_bad_tx_source( ) { + use std::sync::Mutex; + + /// Bad transaction source: ignores the limit of `u16::MAX -1` transactions + /// that should be returned by [`TransactionsSource::next()`]. + /// It is used only for testing purposes + pub struct BadTransactionsSource { + transactions: Mutex>, + } + + impl BadTransactionsSource { + pub fn new(transactions: Vec) -> Self { + Self { + transactions: Mutex::new( + transactions + .into_iter() + .map(MaybeCheckedTransaction::Transaction) + .collect(), + ), + } + } + } + + impl fuel_core_executor::ports::TransactionsSource for BadTransactionsSource { + fn next(&self, _: u64, _: u16, _: u32) -> Vec { + std::mem::take(&mut *self.transactions.lock().unwrap()) + } + } + let block_height = 1u32; let block_da_height = 2u64; From b1dbbe712c95912120edb0777d0b4ff751d2794e Mon Sep 17 00:00:00 2001 From: green Date: Sat, 16 Nov 2024 23:11:30 -0500 Subject: [PATCH 42/45] Increase channel size for da source --- .../gas_price_service/src/v1/da_source_service/service.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/src/v1/da_source_service/service.rs b/crates/services/gas_price_service/src/v1/da_source_service/service.rs index e20a506a097..74f1fcafb42 100644 --- a/crates/services/gas_price_service/src/v1/da_source_service/service.rs +++ b/crates/services/gas_price_service/src/v1/da_source_service/service.rs @@ -24,6 +24,7 @@ impl SharedState { fn new(sender: Sender) -> Self { Self(sender) } + pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver { self.0.subscribe() } @@ -40,7 +41,7 @@ where shared_state: SharedState, } -const DA_BLOCK_COSTS_CHANNEL_SIZE: usize = 10; +const DA_BLOCK_COSTS_CHANNEL_SIZE: usize = 16 * 1024; const POLLING_INTERVAL_MS: u64 = 10_000; impl DaSourceService From ab65d20b6932a9685b301e5ee57cc9449e31aa2b Mon Sep 17 00:00:00 2001 From: green Date: Sat, 16 Nov 2024 23:12:30 -0500 Subject: [PATCH 43/45] Make executor tests happy --- crates/fuel-core/src/executor.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index a7f462416fa..14735b6cebc 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -2997,6 +2997,7 @@ mod tests { } #[test] + #[cfg(not(feature = "wasm-executor"))] fn block_producer_never_includes_more_than_max_tx_count_transactions() { let block_height = 1u32; let block_da_height = 2u64; From e5829d270fe45272dc097beddf113a33f3ab49cc Mon Sep 17 00:00:00 2001 From: green Date: Sat, 16 Nov 2024 23:17:13 -0500 Subject: [PATCH 44/45] Make CI happy --- crates/fuel-core/src/executor.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 14735b6cebc..186b9fd44b7 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -6,10 +6,7 @@ mod tests { use crate as fuel_core; use fuel_core::database::Database; use fuel_core_executor::{ - executor::{ - max_tx_count, - OnceTransactionsSource, - }, + executor::OnceTransactionsSource, ports::{ MaybeCheckedTransaction, RelayerPort, @@ -2999,6 +2996,8 @@ mod tests { #[test] #[cfg(not(feature = "wasm-executor"))] fn block_producer_never_includes_more_than_max_tx_count_transactions() { + use fuel_core_executor::executor::max_tx_count; + let block_height = 1u32; let block_da_height = 2u64; @@ -3037,6 +3036,7 @@ mod tests { #[cfg(not(feature = "wasm-executor"))] fn block_producer_never_includes_more_than_max_tx_count_transactions_with_bad_tx_source( ) { + use fuel_core_executor::executor::max_tx_count; use std::sync::Mutex; /// Bad transaction source: ignores the limit of `u16::MAX -1` transactions From 0c93efefd5ddb57bc33e73d5e5a65a3e09664599 Mon Sep 17 00:00:00 2001 From: green Date: Sat, 16 Nov 2024 23:38:17 -0500 Subject: [PATCH 45/45] Clean up small things --- bin/e2e-test-client/src/config.rs | 2 +- bin/e2e-test-client/src/lib.rs | 2 +- tests/tests/coins.rs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/e2e-test-client/src/config.rs b/bin/e2e-test-client/src/config.rs index 742cb3fd309..9d537043351 100644 --- a/bin/e2e-test-client/src/config.rs +++ b/bin/e2e-test-client/src/config.rs @@ -39,7 +39,7 @@ impl Default for SuiteConfig { fn default() -> Self { Self { endpoint: "http://localhost:4000".to_string(), - wallet_sync_timeout: SYNC_TIMEOUT.checked_mul(2).unwrap(), + wallet_sync_timeout: SYNC_TIMEOUT, full_test: false, coinbase_contract_id: ContractId::from_str( "0x7777777777777777777777777777777777777777777777777777777777777777", diff --git a/bin/e2e-test-client/src/lib.rs b/bin/e2e-test-client/src/lib.rs index 1dbec93fb1c..b8bf49b92ac 100644 --- a/bin/e2e-test-client/src/lib.rs +++ b/bin/e2e-test-client/src/lib.rs @@ -15,7 +15,7 @@ use std::{ }; pub const CONFIG_FILE_KEY: &str = "FUEL_CORE_E2E_CONFIG"; -pub const SYNC_TIMEOUT: Duration = Duration::from_secs(10); +pub const SYNC_TIMEOUT: Duration = Duration::from_secs(20); pub mod config; pub mod test_context; diff --git a/tests/tests/coins.rs b/tests/tests/coins.rs index 60237b52d4a..6a03bcd791d 100644 --- a/tests/tests/coins.rs +++ b/tests/tests/coins.rs @@ -157,7 +157,6 @@ mod coin { ) .await .unwrap(); - tracing::info!("coins_per_asset: {:?}", coins_per_asset); assert_eq!(coins_per_asset.len(), 2); assert_eq!(coins_per_asset[0].len(), 1); assert!(coins_per_asset[0].amount() >= 1);