Testing Builder Core logic

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "daily"
+

.github/workflows/build.yml

@@ -0,0 +1,81 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+  pull_request:
+    branches:
+      - main
+      - release-*
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+        RUSTFLAGS: "--cfg async_executor_impl=\"async-std\" --cfg async_channel_impl=\"async-std\""
+        RUST_LOG: info
+    steps:
+      - uses: styfle/[email protected]
+        name: Cancel Outdated Builds
+        with:
+          all_but_latest: true
+          access_token: ${{ github.token }}
+
+      - uses: actions/checkout@v4
+        name: Checkout Repository
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Configure Git
+        run: |
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com".insteadOf git://github.com
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com".insteadOf ssh://[email protected]
+
+      - uses: Swatinem/rust-cache@v2
+        name: Enable Rust Caching
+
+      - name: Format Check
+        run: cargo fmt -- --check
+
+      # Run Clippy on all targets. The lint workflow doesn't run Clippy on tests, because the tests
+      # don't compile with all combinations of features.
+      - uses: actions-rs/clippy-check@v1
+        name: Clippy
+        with:
+          token: ${{ github.token }}
+          args: --workspace --all-features --all-targets -- -D warnings
+
+      - name: Audit
+        run: cargo audit --ignore RUSTSEC-2023-0018 --ignore RUSTSEC-2023-0052 --ignore RUSTSEC-2023-0065
+
+      - name: Build
+        # Build in release without `testing` feature, this should work without `hotshot_example` config.
+        run: |
+          cargo build --workspace --release
+
+      - name: Test
+        # Build test binary with `testing` feature, which requires `hotshot_example` config
+        run: |
+          export RUSTFLAGS="$RUSTFLAGS --cfg hotshot_example"
+          cargo test --workspace --release --all-features --no-run
+          cargo test --workspace --release --all-features --verbose -- --test-threads 2
+        timeout-minutes: 60
+
+      - name: Generate Documentation
+        run: |
+          cargo doc --no-deps --lib --release
+          echo '<meta http-equiv="refresh" content="0; url=hotshot_query_service">' > target/doc/index.html
+
+      - name: Deploy Documentation
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.ref == 'refs/heads/main' }}
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./target/doc
+          cname: tide-disco.docs.espressosys.com

.github/workflows/build_nix.yml

@@ -0,0 +1,47 @@
+name: Nix
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+  schedule:
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+
+jobs:
+  nix:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Configure Git
+        run: |
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com/".insteadOf git://github.com/
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com/".insteadOf ssh://[email protected]/
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v24
+
+      # - uses: cachix/cachix-action@v12
+      #   with:
+      #     name: espresso-systems-private
+      #     authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: Cache cargo
+        uses: actions/[email protected]
+        with:
+          path: |
+            ~/.cargo-nix/registry/index
+            ~/.cargo-nix/registry/cache
+            ~/.cargo-nix/git
+            target
+          key: espresso-nix-v2-${{ hashFiles('Cargo.lock') }}
+
+      - name: "Sanity Check: nix environment loads"
+        run: nix-shell --run "echo Success"
+
+      - name: "Sanity Check: nix environment builds all targets"
+        run: nix-shell --run "cargo build --all-targets --all-features --release --workspace"

.github/workflows/build_windows.yml

@@ -0,0 +1,57 @@
+# Copyright (c) 2022 Espresso Systems (espressosys.com)
+# This file is part of the Tide Disco library.
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+# You should have received a copy of the GNU General Public License along with this program. If not,
+# see <https://www.gnu.org/licenses/>.
+
+name: Windows build
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+  pull_request:
+    branches:
+      - main
+      - release-*
+  workflow_dispatch:
+
+jobs:
+  windows:
+    runs-on: windows-latest
+    env:
+        RUSTFLAGS: "--cfg async_executor_impl=\"async-std\" --cfg async_channel_impl=\"async-std\" --cfg hotshot_example"
+        RUST_LOG: info
+    steps:
+      - name: Configure Git
+        run: |
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com/".insteadOf git://github.com/
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com/".insteadOf ssh://[email protected]/
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@v2
+        name: Enable Rust Caching
+
+      - name: Build
+        run: |
+          cargo build --workspace --release
+
+      - name: Test
+        run: |
+          cargo test --workspace --release --all-features --no-run
+          cargo test --workspace --release --all-features --verbose -- --test-threads 2
+        timeout-minutes: 60

.github/workflows/combine-prs.yml

@@ -0,0 +1,24 @@
+name: Combine PRs
+
+on:
+  schedule:
+    - cron: "0 1 * * MON"
+  workflow_dispatch: # allows to manually trigger the workflow
+
+# The minimum permissions required to run this Action
+permissions:
+  contents: write
+  pull-requests: write
+  checks: read
+
+jobs:
+  combine-prs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: combine-prs
+        id: combine-prs
+        uses: github/[email protected]
+        with:
+          github_token: ${{ secrets.ORG_GITHUB_PAT }}
+          labels: "dependabot,combined-pr"

.github/workflows/coverage.yml

@@ -0,0 +1,58 @@
+name: Code Coverage Workflow
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+  schedule:
+    - cron: "0 1 * * 1"
+  workflow_dispatch:
+
+jobs:
+  code-coverage:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - name: Configure Git
+        run: |
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com/".insteadOf git://github.com/
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com/".insteadOf ssh://[email protected]/
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v24
+
+      - name: Enable Cachix
+        uses: cachix/cachix-action@v13
+        # If PR is from a non-collaborator (e.g. dependabot) the secrets are missing and the login to cachix fails.
+        continue-on-error: true
+        with:
+          name: espresso-systems-private
+          authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
+          extraPullNames: nix-community
+          skipPush: ${{ github.actor == 'dependabot[bot]' }}
+
+      - name: Cache cargo
+        uses: actions/[email protected]
+        with:
+          path: |
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git
+            target
+          key: hotshot-query-service-codecov-v1-${{ hashFiles('Cargo.lock') }}
+
+      - name: Generate coverage reports
+        run: |
+          git config --global --add safe.directory "$PWD"
+          nix run "github:NixOS/nix?ref=1849e6a1f64734c488c2b1469249d65ce08cef93" -- develop .#perfShell -c cargo llvm-cov --profile=release --all-features --all-targets --lcov --output-path lcov.info -- --test-threads 1
+
+      - name: Coveralls upload
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          path-to-lcov: lcov.info
+          fail_ci_if_error: true

.github/workflows/debug_build.yml

@@ -0,0 +1,45 @@
+name: Debug Build
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+        RUSTFLAGS: "--cfg async_executor_impl=\"async-std\" --cfg async_channel_impl=\"async-std\""
+        RUST_LOG: info
+        RUST_MIN_STACK: '3145728'
+    steps:
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: styfle/[email protected]
+        name: Cancel Outdated Builds
+        with:
+          all_but_latest: true
+          access_token: ${{ github.token }}
+
+      - uses: actions/checkout@v4
+        name: Checkout Repository
+
+      - name: Configure Git
+        run: |
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com".insteadOf git://github.com
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com".insteadOf ssh://[email protected]
+
+      - uses: Swatinem/rust-cache@v2
+        name: Enable Rust Caching
+
+      - name: Build
+        run: |
+          cargo build --workspace --all-features
+
+      - name: Test
+        run: |
+          cargo test --workspace --all-features --no-run
+          cargo test --workspace --all-features --verbose -- --test-threads 2
+        timeout-minutes: 60

.github/workflows/lint.yml

@@ -0,0 +1,59 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+  pull_request:
+    branches:
+      - main
+      - release-*
+  workflow_dispatch:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # Lint with many combinations of feature flags
+        features:
+          # No optional features
+          - ''
+          # Each optional feature on its own
+          - sql-data-source
+          - file-system-data-source
+          - metrics-data-source
+          # All optional features together
+          - sql-data-source,file-system-data-source,metrics-data-source
+    env:
+        RUSTFLAGS: "--cfg async_executor_impl=\"async-std\" --cfg async_channel_impl=\"async-std\""
+        RUST_LOG: info
+    steps:
+      - uses: styfle/[email protected]
+        name: Cancel Outdated Builds
+        with:
+          all_but_latest: true
+          access_token: ${{ github.token }}
+
+      - uses: actions/checkout@v4
+        name: Checkout Repository
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Configure Git
+        run: |
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com".insteadOf git://github.com
+          git config --global url."https://ancient123:${{ secrets.ORG_GITHUB_PAT }}@github.com".insteadOf ssh://[email protected]
+
+      - uses: Swatinem/rust-cache@v2
+        name: Enable Rust Caching
+
+      - uses: actions-rs/clippy-check@v1
+        name: Clippy
+        with:
+          token: ${{ github.token }}
+          args: --workspace --no-default-features --features "${{ matrix.features }}" -- -D warnings