Merge move marketplace version into weekly hotshot update

.config/nextest.toml

@@ -1,8 +1,17 @@
 [profile.default]
-slow-timeout = "2m"
+# Kill tests after 3 periods of 2m, because they are probably hanging
+slow-timeout = { period = "2m", terminate-after = 3 }
 default-filter = 'not (test(slow_) | package(tests))'
 retries = 2
 
+[[profile.default.overrides]]
+# This test is fast if it works (usually about 150ms) but sometimes hangs,
+# especially when running concurrently with other tests. Kill it after a while
+# and retry up to 10 times.
+filter = 'test(test_process_client_handling_stream_subscribe_node_identity)'
+slow-timeout = { period = "2s", terminate-after = 1 }
+retries  = 10
+
 # The restart tests run an entire sequencing network, and so are quite resource intensive.
 [[profile.default.overrides]]
 filter = 'test(slow_test_restart)'

.env

@@ -148,3 +148,6 @@ INTEGRATION_TEST_PROTO=http
 # Version of sequencer protocol we want to test. If this is set to
 # `03`, marketplace upgrade will be tested.
 INTEGRATION_TEST_SEQUENCER_VERSION=02
+
+# max database connections 
+ESPRESSO_SEQUENCER_DATABASE_MAX_CONNECTIONS=25

.github/workflows/benchmark-build.yaml

@@ -38,7 +38,7 @@ jobs:
       - name: Build Espresso Dev Node
         # Espresso Dev Node currently requires testing feature, so it is built separately.
         run: |
-          cargo build --locked --release --features benchmarking,testing --bin espresso-dev-node
+          cargo build --locked --release --features benchmarking,testing,embedded-db --bin espresso-dev-node
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -80,7 +80,7 @@ jobs:
       - name: Build Espresso Dev Node
         # Espresso Dev Node currently requires testing feature, so it is built separately.
         run: |
-          cargo build --locked --release --features benchmarking,testing --bin espresso-dev-node
+          cargo build --locked --release --features benchmarking,testing,embedded-db --bin espresso-dev-node
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4

.github/workflows/build.yml

@@ -46,11 +46,14 @@ jobs:
         # Build in release without `testing` feature, this should work without `hotshot_example` config.
         run: |
           cargo build --locked --release --workspace
+
+      - name: Build sequencer-sqlite 
+        run: cargo build --locked --release --manifest-path ./sequencer-sqlite/Cargo.toml --target-dir ./target
 
       - name: Build Espresso Dev Node
         # Espresso Dev Node currently requires testing feature, so it is built separately.
         run: |
-          cargo build --locked --release --features testing --bin espresso-dev-node
+          cargo build --locked --release --features "testing embedded-db" --bin espresso-dev-node
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -79,6 +82,7 @@ jobs:
             target/release/marketplace-builder
             target/release/node-metrics
             target/release/dev-rollup
+            target/release/sequencer-sqlite
 
   build-arm:
     if: github.event_name != 'pull_request'
@@ -100,6 +104,9 @@ jobs:
         run: |
           cargo build --locked --release --workspace
 
+      - name: Build sequencer-sqlite 
+        run: cargo build --locked --release --manifest-path ./sequencer-sqlite/Cargo.toml --target-dir ./target
+
       - name: Build Espresso Dev Node
         # Espresso Dev Node currently requires testing feature, so it is built separately.
         run: |
@@ -132,6 +139,7 @@ jobs:
             target/release/marketplace-builder
             target/release/node-metrics
             target/release/dev-rollup
+            target/release/sequencer-sqlite
 
   build-dockers:
     runs-on: ubuntu-latest
@@ -286,7 +294,6 @@ jobs:
         with:
           images: ghcr.io/espressosystems/espresso-sequencer/dev-rollup
 
-
       - name: Build and push sequencer docker
         uses: docker/build-push-action@v6
         with:

.github/workflows/build_static.yml

@@ -68,7 +68,8 @@ jobs:
       - name: Compile all executables
         # timeout-minutes: 120
         run: |
-          nix develop $DEVSHELL --accept-flake-config --option sandbox relaxed -c cargo build --locked --release
+          nix develop $DEVSHELL --accept-flake-config --option sandbox relaxed -c cargo build --locked --release 
+          -c cargo build --locked --release --manifest-path ./sequencer-sqlite/Cargo.toml --target-dir 'echo $CARGO_TARGET_DIR'
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -95,6 +96,7 @@ jobs:
             ${{ env.CARGO_TARGET_DIR }}/${{ env.TARGET_TRIPLET }}/release/marketplace-builder
             ${{ env.CARGO_TARGET_DIR }}/${{ env.TARGET_TRIPLET }}/release/node-metrics
             ${{ env.CARGO_TARGET_DIR }}/${{ env.TARGET_TRIPLET }}/release/dev-rollup 
+            ${{ env.CARGO_TARGET_DIR }}/${{ env.TARGET_TRIPLET }}/release/sequencer-sqlite
   static-dockers:
     runs-on: ubuntu-latest
     needs: static-build
@@ -377,4 +379,4 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.dev-rollup.outputs.tags }}
-          labels: ${{ steps.dev-rollup.outputs.labels }}
+          labels: ${{ steps.dev-rollup.outputs.labels }}

.github/workflows/cargo-features.yml

@@ -13,7 +13,7 @@ on:
   pull_request:
   workflow_dispatch:
 
-concurrency:
+concurrency: 
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 

.github/workflows/lint.yml

@@ -32,5 +32,8 @@ jobs:
       - name: Format Check
         run: cargo fmt -- --check
 
-      - name: Check
-        run: cargo clippy --workspace --all-features --all-targets -- -D warnings
+      - name: Check (embedded-db)
+        run: cargo clippy --workspace --features "embedded-db testing" --all-targets -- -D warnings
+
+      - name: Check (postgres)
+        run: cargo clippy --workspace --features testing --all-targets -- -D warnings

.github/workflows/slowtest.yaml

@@ -23,7 +23,7 @@ env:
   RUST_LOG: info,libp2p=off,node=error
 
 jobs:
-  slow-tests:
+  slow-tests-sqlite:
     runs-on: ubuntu-latest
     steps:
       # These tests need the `anvil` binary provided by foundry
@@ -46,11 +46,49 @@ jobs:
       - name: Build
         run: |
           cargo build --locked --bin diff-test --release
-          cargo nextest run --locked --release --workspace --all-features --no-run
+          cargo nextest run --locked --release --workspace --features embedded-db --no-run
         timeout-minutes: 90
 
       - name: Slow Test
         env:
           NEXTEST_PROFILE: slow
-        run: cargo nextest run --locked --release --workspace --all-features --verbose --no-fail-fast --nocapture
+        run: cargo nextest run --locked --release --workspace --features embedded-db --verbose --no-fail-fast --nocapture
         timeout-minutes: 40
+  slow-tests-postgres:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fix submodule permissions check
+        run: |
+          git config --global --add safe.directory '*'
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+        with:
+          version: nightly
+
+      - uses: taiki-e/install-action@nextest
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Configure Environment
+        run: PATH="$PWD/target/release:$PATH"
+
+      - name: Enable Rust Caching
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-all-crates: true
+
+      - name: Build
+        run: |
+          cargo build --locked --bin diff-test --release
+          cargo nextest run --locked --release --workspace --no-run
+        timeout-minutes: 90
+
+      - name: Slow Test
+        env:
+          NEXTEST_PROFILE: slow
+        run: cargo nextest run --locked --release --workspace --verbose --no-fail-fast --nocapture
+        timeout-minutes: 40

.github/workflows/test.yml

@@ -20,10 +20,12 @@ concurrency:
 env:
   RUST_LOG: info,libp2p=off,node=error
   CARGO_TERM_COLOR: always
+  # Save the process compose logs
+  PC_LOGS: /tmp/pc.log
 
 jobs:
-  build-test-artifacts:
-    name: Build test artifacts
+  build-test-artifacts-postgres:
+    name: Build test artifacts (postgres)
     runs-on: buildjet-8vcpu-ubuntu-2204
     steps:
       - uses: rui314/setup-mold@v1
@@ -40,13 +42,40 @@ jobs:
           cache-provider: buildjet
 
       - name: Build and archive tests
-        run: cargo nextest archive --locked --workspace --all-features --archive-file nextest-archive.tar.zst
+        run: cargo nextest archive --locked --workspace --archive-file nextest-archive-postgres.tar.zst
 
       - name: Upload archive to workflow
         uses: actions/upload-artifact@v4
         with:
-          name: nextest-archive
-          path: nextest-archive.tar.zst
+          name: nextest-archive-postgres
+          path: nextest-archive-postgres.tar.zst
+
+  build-test-artifacts-sqlite:
+    name: Build test artifacts (sqlite)
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - uses: rui314/setup-mold@v1
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install nextest
+        uses: taiki-e/install-action@nextest
+
+      - name: Enable Rust Caching
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-all-crates: true
+          cache-provider: buildjet
+
+      - name: Build and archive tests
+        run: cargo nextest archive --locked --features "embedded-db testing" --workspace --archive-file nextest-archive-sqlite.tar.zst
+
+      - name: Upload archive to workflow
+        uses: actions/upload-artifact@v4
+        with:
+          name: nextest-archive-sqlite
+          path: nextest-archive-sqlite.tar.zst
 
   build-test-bins:
     name: Build test binaries
@@ -63,7 +92,9 @@ jobs:
           cache-provider: buildjet
 
       - name: Build Bins
-        run: cargo build --locked --profile test --bins
+        run: |
+          cargo build --locked --profile test --bins
+          cargo build --manifest-path ./sequencer-sqlite/Cargo.toml --target-dir ./target
         timeout-minutes: 30
 
       - name: Upload archive to workflow
@@ -93,39 +124,70 @@ jobs:
             target/debug/marketplace-builder
             target/debug/node-metrics
             target/debug/dev-rollup
+            target/debug/sequencer-sqlite
+
+  test-postgres:
+    needs: build-test-artifacts-postgres
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      - uses: actions/checkout@v4
+
+      - name: Install nextest
+        uses: taiki-e/install-action@nextest
+      - name: Download archive
+        uses: actions/download-artifact@v4
+        with:
+          name: nextest-archive-postgres
+
+      - name: Test
+        run: |
+          cargo nextest run --archive-file nextest-archive-postgres.tar.zst  --verbose --no-fail-fast \
+          --workspace-remap $PWD
+        timeout-minutes: 20
 
-  test:
-    needs: build-test-artifacts
+  test-sqlite:
+    needs: build-test-artifacts-sqlite
     runs-on: ubuntu-latest
     steps:
+      - name: Fix submodule permissions check
+        run: |
+          git config --global --add safe.directory '*'
+
       - name: Install Foundry
         uses: foundry-rs/foundry-toolchain@v1
+        with:
+          version: nightly
 
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Install nextest
         uses: taiki-e/install-action@nextest
       - name: Download archive
         uses: actions/download-artifact@v4
         with:
-          name: nextest-archive
+          name: nextest-archive-sqlite
 
       - name: Test
         run: |
-          cargo nextest run --archive-file nextest-archive.tar.zst  --verbose --no-fail-fast \
+          cargo nextest run --archive-file nextest-archive-sqlite.tar.zst  --verbose --no-fail-fast \
           --workspace-remap $PWD
         timeout-minutes: 20
 
   test-integration:
-    needs: [build-test-artifacts, build-test-bins]
+    needs: [build-test-bins, build-test-artifacts-postgres]
     strategy:
       matrix:
-        version: [02,03]
+        version: [02,99]
         include:
           - version: 02
             compose: "-f process-compose.yaml -D"
 
-          - version: 03
+          - version: 99
             compose: "-f process-compose.yaml -f process-compose-mp.yml -D"
     runs-on: ubuntu-latest
     steps:
@@ -144,31 +206,33 @@ jobs:
 
       - name: Move files
         run: |
-          mv nextest-archive/* .
+          mv nextest-archive-postgres/* .
           mkdir -p target/debug
           mv test-binaries/* target/debug
           chmod -c +x target/debug/*
 
       - name: Install process-compose
         run: nix profile install nixpkgs#process-compose
 
-      - name: Pull Docker Images
-        run: docker compose pull || docker-compose pull
-
       - name: Run Demo-Native ${{matrix.version}}
-        run: bash -x scripts/demo-native ${{matrix.compose}}
+        run: bash -x scripts/demo-native ${{matrix.compose}} --tui=false > ${{ env.PC_LOGS }} 2>&1 &
 
       - name: Test Integration
         env:
+          RUST_LOG: debug
           NEXTEST_PROFILE: integration
           INTEGRATION_TEST_SEQUENCER_VERSION: ${{ matrix.version }}
         run: |
-          cargo nextest run --archive-file nextest-archive.tar.zst --verbose --no-fail-fast \
+          cargo nextest run --archive-file nextest-archive-postgres.tar.zst --verbose --no-fail-fast --nocapture \
           --workspace-remap $PWD $(if [ "${{ matrix.version }}" == "2" ]; then echo " smoke"; fi)
-        timeout-minutes: 40
+        timeout-minutes: 10
 
-      - name: Process Compose Down
-        run: process-compose down
+      - name: Upload process compose logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: process-compose-logs-integration-v${{ matrix.version  }}
+          path: ${{ env.PC_LOGS }}
 
   demo-native:
     needs: build-test-bins
@@ -197,6 +261,3 @@ jobs:
           set -o pipefail
           scripts/demo-native --tui=false &
           timeout -v 600 scripts/smoke-test-demo | sed -e 's/^/smoke-test: /;'
-
-      - name: Process Compose Down
-        run: process-compose down