Merge remote-tracking branch 'origin/main' into rm/cdn-gcra

EspressoSystems · Nov 13, 2024 · 8cd96d9 · 8cd96d9
2 parents abfb9da + 32e6970
commit 8cd96d9
Show file tree

Hide file tree

Showing 177 changed files with 3,800 additions and 5,491 deletions.
diff --git a/.github/ISSUE_TEMPLATE/EXTERNAL_ISSUE_FORM.yml b/.github/ISSUE_TEMPLATE/EXTERNAL_ISSUE_FORM.yml
@@ -24,7 +24,7 @@ body:
       placeholder: | 
         Include other details here such as: open questions, directions to reproduce a bug, relevant error logs, etc. 
 
-        E.g. To reproduce this bug run `just async_std test_basic`. You should see logs similar to the ones below: 
+        E.g. To reproduce this bug run `just test_basic`. You should see logs similar to the ones below: 
         `ERROR: This is an important error!`
     validations:
       required: false

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -15,6 +15,10 @@ updates:
         - "*"
       exclude-patterns:
         - "cdn-*"
+        - "ark-*"
     cdn:
       patterns:
         - "cdn-*"
+    ark:
+      patterns:
+        - "ark-*"
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -18,9 +18,6 @@ jobs:
   test:
     strategy:
       matrix:
-        just_variants:
-          - async-std
-          - tokio
         test_suites:
           - test-ci-1
           - test-ci-2
@@ -41,31 +38,22 @@ jobs:
         name: Enable Rust Caching
         with:
           shared-key: "test"
-          prefix-key: ${{ matrix.just_variants }}
           cache-on-failure: "true"
           save-if: ${{ github.ref == 'refs/heads/main' && matrix.test_suites == 'test-ci-rest' }}
 
-      - name: Install Just
-        run: |
-          wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz
-          tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just
-          sudo cp just /usr/bin/just
+      - uses: taiki-e/install-action@just
 
       - uses: taiki-e/install-action@nextest
 
       - name: Unit and integration tests for all crates in workspace
         run: |
-          just ${{ matrix.just_variants }} ${{ matrix.test_suites }}
+          just ${{ matrix.test_suites }}
         timeout-minutes: 60
         env:
           RUST_BACKTRACE: full
 
   test-examples:
     strategy:
-      matrix:
-        just_variants:
-          - async-std
-          - tokio
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
@@ -79,27 +67,18 @@ jobs:
         name: Enable Rust Caching
         with:
           shared-key: "examples"
-          prefix-key: ${{ matrix.just_variants }}
           cache-on-failure: "true"
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - name: Install Just
-        run: |
-          wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz
-          tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just
-          sudo cp just /usr/bin/just
+      - uses: taiki-e/install-action@just
 
       - name: Test examples
         run: |
-          just ${{ matrix.just_variants }} example all-push-cdn -- --config_file ./crates/orchestrator/run-config.toml
+          just example all-push-cdn -- --config_file ./crates/orchestrator/run-config.toml
         timeout-minutes: 20
 
   build-release:
     strategy:
-      matrix:
-        just_variants:
-          - async-std
-          - tokio
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
@@ -113,39 +92,30 @@ jobs:
         name: Enable Rust Caching
         with:
           shared-key: "build-release"
-          prefix-key: ${{ matrix.just_variants }}
           cache-on-failure: "true"
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - name: Install Just
-        run: |
-          wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz
-          tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just
-          sudo cp just /usr/bin/just
+      - uses: taiki-e/install-action@just
 
       - name: Build examples in release mode
-        run: just ${{ matrix.just_variants }} build_release --examples --package hotshot-examples --no-default-features
+        run: just build_release --examples --package hotshot-examples --no-default-features
 
       - name: Upload Binaries
         uses: actions/upload-artifact@v4
         with:
-          name: binaries-amd64-${{ matrix.just_variants }}
+          name: binaries-amd64
           path: |
-            target/${{ matrix.just_variants }}/release/examples/counter
-            target/${{ matrix.just_variants }}/release/examples/multi-validator-libp2p
-            target/${{ matrix.just_variants }}/release/examples/validator-libp2p
-            target/${{ matrix.just_variants }}/release/examples/validator-combined
-            target/${{ matrix.just_variants }}/release/examples/validator-push-cdn
-            target/${{ matrix.just_variants }}/release/examples/orchestrator
-            target/${{ matrix.just_variants }}/release/examples/cdn-broker
-            target/${{ matrix.just_variants }}/release/examples/cdn-marshal
+            target/release/examples/counter
+            target/release/examples/multi-validator-libp2p
+            target/release/examples/validator-libp2p
+            target/release/examples/validator-combined
+            target/release/examples/validator-push-cdn
+            target/release/examples/orchestrator
+            target/release/examples/cdn-broker
+            target/release/examples/cdn-marshal
 
   build-arm-release:
     strategy:
-      matrix:
-        just_variants:
-          - async-std
-          - tokio
       fail-fast: false
     runs-on: buildjet-4vcpu-ubuntu-2204-arm
     if: ${{ github.ref == 'refs/heads/main' }}
@@ -158,33 +128,28 @@ jobs:
         name: Enable Rust Caching
         with:
           shared-key: "build-arm-release"
-          prefix-key: ${{ matrix.just_variants }}
           cache-on-failure: "true"
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Build examples in release mode
-        run: just ${{ matrix.just_variants }} build_release --examples --package hotshot-examples --no-default-features
+        run: just build_release --examples --package hotshot-examples --no-default-features
 
       - name: Upload Binaries
         uses: actions/upload-artifact@v4
         with:
-          name: binaries-aarch64-${{ matrix.just_variants }}
+          name: binaries-aarch64
           path: |
-            target/${{ matrix.just_variants }}/release/examples/counter
-            target/${{ matrix.just_variants }}/release/examples/multi-validator-libp2p
-            target/${{ matrix.just_variants }}/release/examples/validator-libp2p
-            target/${{ matrix.just_variants }}/release/examples/validator-combined
-            target/${{ matrix.just_variants }}/release/examples/validator-push-cdn
-            target/${{ matrix.just_variants }}/release/examples/orchestrator
-            target/${{ matrix.just_variants }}/release/examples/cdn-broker
-            target/${{ matrix.just_variants }}/release/examples/cdn-marshal
+            target/release/examples/counter
+            target/release/examples/multi-validator-libp2p
+            target/release/examples/validator-libp2p
+            target/release/examples/validator-combined
+            target/release/examples/validator-push-cdn
+            target/release/examples/orchestrator
+            target/release/examples/cdn-broker
+            target/release/examples/cdn-marshal
 
   build-dockers:
     strategy:
-      matrix:
-        just_variants:
-          - async-std
-          - tokio
       fail-fast: false
     runs-on: ubuntu-latest
     if: ${{ github.ref == 'refs/heads/main' }}
@@ -207,56 +172,50 @@ jobs:
       - name: Download AMD executables
         uses: actions/download-artifact@v4
         with:
-          name: binaries-amd64-${{ matrix.just_variants }}
-          path: target/${{ matrix.just_variants }}/amd64/release/examples
+          name: binaries-amd64
+          path: target/amd64/release/examples
 
       - name: Download ARM executables
         uses: actions/download-artifact@v4
         with:
-          name: binaries-aarch64-${{ matrix.just_variants }}
-          path: target/${{ matrix.just_variants }}/arm64/release/examples
+          name: binaries-aarch64
+          path: target/arm64/release/examples
 
       - name: Generate validator-libp2p docker metadata
         uses: docker/metadata-action@v5
         id: validator-libp2p
         with:
           images: ghcr.io/espressosystems/hotshot/validator-libp2p
-          flavor: suffix=-${{ matrix.just_variants }}
 
       - name: Generate validator-combined docker metadata
         uses: docker/metadata-action@v5
         id: validator-combined
         with:
           images: ghcr.io/espressosystems/hotshot/validator-combined
-          flavor: suffix=-${{ matrix.just_variants }}
 
       - name: Generate validator-push-cdn docker metadata
         uses: docker/metadata-action@v5
         id: validator-push-cdn
         with:
           images: ghcr.io/espressosystems/hotshot/validator-push-cdn
-          flavor: suffix=-${{ matrix.just_variants }}
 
       - name: Generate orchestrator docker metadata
         uses: docker/metadata-action@v5
         id: orchestrator
         with:
           images: ghcr.io/espressosystems/hotshot/orchestrator
-          flavor: suffix=-${{ matrix.just_variants }}
 
       - name: Generate cdn-broker docker metadata
         uses: docker/metadata-action@v5
         id: cdn-broker
         with:
           images: ghcr.io/espressosystems/hotshot/cdn-broker
-          flavor: suffix=-${{ matrix.just_variants }}
 
       - name: Generate cdn-marshal docker metadata
         uses: docker/metadata-action@v5
         id: cdn-marshal
         with:
           images: ghcr.io/espressosystems/hotshot/cdn-marshal
-          flavor: suffix=-${{ matrix.just_variants }}
 
       - name: Build and push validator-libp2p docker
         uses: docker/build-push-action@v6
@@ -269,8 +228,6 @@ jobs:
           labels: ${{ steps.validator-libp2p.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          build-args: |
-            ASYNC_EXECUTOR=${{ matrix.just_variants }}
 
       - name: Build and push validator-combined docker
         uses: docker/build-push-action@v6
@@ -283,8 +240,6 @@ jobs:
           labels: ${{ steps.validator-combined.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          build-args: |
-            ASYNC_EXECUTOR=${{ matrix.just_variants }}
 
       - name: Build and push validator-push-cdn docker
         uses: docker/build-push-action@v6
@@ -297,8 +252,6 @@ jobs:
           labels: ${{ steps.validator-push-cdn.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          build-args: |
-            ASYNC_EXECUTOR=${{ matrix.just_variants }}
 
       - name: Build and push orchestrator docker
         uses: docker/build-push-action@v6
@@ -311,8 +264,6 @@ jobs:
           labels: ${{ steps.orchestrator.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          build-args: |
-            ASYNC_EXECUTOR=${{ matrix.just_variants }}
 
       - name: Build and push cdn-broker docker
         uses: docker/build-push-action@v6
@@ -325,8 +276,6 @@ jobs:
           labels: ${{ steps.cdn-broker.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          build-args: |
-            ASYNC_EXECUTOR=${{ matrix.just_variants }}
 
       - name: Build and push cdn-marshal docker
         uses: docker/build-push-action@v6
@@ -339,5 +288,3 @@ jobs:
           labels: ${{ steps.cdn-marshal.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          build-args: |
-            ASYNC_EXECUTOR=${{ matrix.just_variants }}
diff --git a/.github/workflows/build-without-lockfile.yml b/.github/workflows/build-without-lockfile.yml
@@ -0,0 +1,37 @@
+name: Build without committed Cargo.lock
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+    tags:
+      # YYYYMMDD
+      - "20[0-9][0-9][0-1][0-9][0-3][0-9]*"
+  schedule:
+    - cron: "0 0 * * 1"
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-ignore-lockfile:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Enable Rust Caching
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: v1-rust
+
+      - name: Build without committed Cargo.lock
+        run: |
+          cargo generate-lockfile
+          cargo check --all-targets
diff --git a/.github/workflows/build_nix.yml b/.github/workflows/build_nix.yml
@@ -45,4 +45,4 @@ jobs:
       # sanity check that repository builds with nix
       - name: Build
         run: |
-          nix develop -c just async-std build
+          nix develop -c just build
diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
@@ -21,11 +21,7 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
 
-      - name: Install Just
-        run: |
-          wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz
-          tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just
-          sudo cp just /usr/bin/just
+      - uses: taiki-e/install-action@just
 
       - uses: Swatinem/rust-cache@v2
         name: Enable Rust Caching
@@ -35,16 +31,16 @@ jobs:
 
       - name: Test Docs
         run: |
-          just async-std doc_test
+          just doc_test
 
       - name: Build Docs
         run: |
-          just async-std doc
+          just doc
 
       - name: Create documentation
         if: ${{ github.ref == 'refs/heads/main' }}
         run: |
-          cp -R target/async-std/doc public
+          cp -R target/doc public
           echo '<meta http-equiv="refresh" content="0; url=hotshot">' > public/index.html
 
       - name: Deploy