diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 9a40d2e405..451e762078 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -40,12 +40,6 @@ jobs: tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just sudo cp just /usr/bin/just - - name: Build all crates in workspace - run: just ${{ matrix.just_variants }} build - - - name: Build HotShot in release mode - run: just ${{ matrix.just_variants }} build_release - - name: Unit and integration tests for all crates in workspace run: | just ${{ matrix.just_variants }} test-ci @@ -53,20 +47,25 @@ jobs: env: RUST_BACKTRACE: full + - name: Build examples in release mode + run: just ${{ matrix.just_variants }} build_release --examples --package hotshot-examples --no-default-features --features="docs,doc-images" + - name: Upload Binaries uses: actions/upload-artifact@v4 with: name: binaries-amd64-${{ matrix.just_variants }} path: | - target/${{ matrix.just_variants }}/debug/examples/counter - target/${{ matrix.just_variants }}/debug/examples/multi-validator-libp2p - target/${{ matrix.just_variants }}/debug/examples/orchestrator-libp2p - target/${{ matrix.just_variants }}/debug/examples/validator-libp2p - target/${{ matrix.just_variants }}/debug/examples/multi-validator-webserver - target/${{ matrix.just_variants }}/debug/examples/multi-webserver - target/${{ matrix.just_variants }}/debug/examples/webserver - target/${{ matrix.just_variants }}/debug/examples/orchestrator-webserver - target/${{ matrix.just_variants }}/debug/examples/validator-webserver + target/${{ matrix.just_variants }}/release/examples/counter + target/${{ matrix.just_variants }}/release/examples/multi-validator-libp2p + target/${{ matrix.just_variants }}/release/examples/orchestrator-libp2p + target/${{ matrix.just_variants }}/release/examples/validator-libp2p + target/${{ matrix.just_variants }}/release/examples/multi-validator-webserver + target/${{ matrix.just_variants }}/release/examples/multi-webserver + target/${{ matrix.just_variants }}/release/examples/webserver + target/${{ matrix.just_variants }}/release/examples/orchestrator-webserver + target/${{ matrix.just_variants }}/release/examples/validator-webserver + target/${{ matrix.just_variants }}/release/examples/cdn-broker + target/${{ matrix.just_variants }}/release/examples/cdn-marshal build-arm: strategy: @@ -86,23 +85,25 @@ jobs: shared-key: "" prefix-key: arm-${{ matrix.just_variants }} - - name: Build all crates in workspace - run: just ${{ matrix.just_variants }} build + - name: Build examples in release mode + run: just ${{ matrix.just_variants }} build_release --examples --package hotshot-examples --no-default-features --features="docs,doc-images" - name: Upload Binaries uses: actions/upload-artifact@v4 with: name: binaries-aarch64-${{ matrix.just_variants }} path: | - target/${{ matrix.just_variants }}/debug/examples/counter - target/${{ matrix.just_variants }}/debug/examples/multi-validator-libp2p - target/${{ matrix.just_variants }}/debug/examples/orchestrator-libp2p - target/${{ matrix.just_variants }}/debug/examples/validator-libp2p - target/${{ matrix.just_variants }}/debug/examples/multi-validator-webserver - target/${{ matrix.just_variants }}/debug/examples/multi-webserver - target/${{ matrix.just_variants }}/debug/examples/webserver - target/${{ matrix.just_variants }}/debug/examples/orchestrator-webserver - target/${{ matrix.just_variants }}/debug/examples/validator-webserver + target/${{ matrix.just_variants }}/release/examples/counter + target/${{ matrix.just_variants }}/release/examples/multi-validator-libp2p + target/${{ matrix.just_variants }}/release/examples/orchestrator-libp2p + target/${{ matrix.just_variants }}/release/examples/validator-libp2p + target/${{ matrix.just_variants }}/release/examples/multi-validator-webserver + target/${{ matrix.just_variants }}/release/examples/multi-webserver + target/${{ matrix.just_variants }}/release/examples/webserver + target/${{ matrix.just_variants }}/release/examples/orchestrator-webserver + target/${{ matrix.just_variants }}/release/examples/validator-webserver + target/${{ matrix.just_variants }}/release/examples/cdn-broker + target/${{ matrix.just_variants }}/release/examples/cdn-marshal build-dockers: strategy: @@ -174,6 +175,20 @@ jobs: images: ghcr.io/espressosystems/hotshot/validator-webserver flavor: suffix=-${{ matrix.just_variants }} + - name: Generate cdn-broker docker metadata + uses: docker/metadata-action@v5 + id: cdn-broker + with: + images: ghcr.io/espressosystems/hotshot/cdn-broker + flavor: suffix=-${{ matrix.just_variants }} + + - name: Generate cdn-marshal docker metadata + uses: docker/metadata-action@v5 + id: cdn-marshal + with: + images: ghcr.io/espressosystems/hotshot/cdn-marshal + flavor: suffix=-${{ matrix.just_variants }} + - name: Build and push orchestrator-libp2p docker uses: docker/build-push-action@v5 with: @@ -234,6 +249,30 @@ jobs: build-args: | ASYNC_EXECUTOR=${{ matrix.just_variants }} + - name: Build and push cdn-broker docker + uses: docker/build-push-action@v5 + with: + context: ./ + file: ./docker/cdn-broker.Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.cdn-broker.outputs.tags }} + labels: ${{ steps.cdn-broker.outputs.labels }} + build-args: | + ASYNC_EXECUTOR=${{ matrix.just_variants }} + + - name: Build and push cdn-marshal docker + uses: docker/build-push-action@v5 + with: + context: ./ + file: ./docker/cdn-marshal.Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.cdn-marshal.outputs.tags }} + labels: ${{ steps.cdn-marshal.outputs.labels }} + build-args: | + ASYNC_EXECUTOR=${{ matrix.just_variants }} + test-crypto: strategy: matrix: diff --git a/Cargo.lock b/Cargo.lock index 88bfb557cd..e58db90eb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1286,17 +1286,17 @@ checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" [[package]] name = "cdn-broker" version = "0.1.0" -source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.1.12#3082083b1e67dd232e0cbb6519c643b9cfa93b2b" +source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.1.14#b7aed2568eba6e88c3399936e4bbedd22c44f076" dependencies = [ "async-std", "cdn-proto", "clap", "dashmap", "derivative", - "derive_builder 0.13.1", + "derive_builder", "jf-primitives", "lazy_static", - "local-ip-address 0.5.7", + "local-ip-address", "parking_lot", "prometheus", "rand 0.8.5", @@ -1309,12 +1309,12 @@ dependencies = [ [[package]] name = "cdn-client" version = "0.1.0" -source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.1.12#3082083b1e67dd232e0cbb6519c643b9cfa93b2b" +source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.1.14#b7aed2568eba6e88c3399936e4bbedd22c44f076" dependencies = [ "async-std", "cdn-proto", "clap", - "derive_builder 0.13.1", + "derive_builder", "jf-primitives", "rand 0.8.5", "tokio", @@ -1325,12 +1325,12 @@ dependencies = [ [[package]] name = "cdn-marshal" version = "0.1.0" -source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.1.12#3082083b1e67dd232e0cbb6519c643b9cfa93b2b" +source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.1.14#b7aed2568eba6e88c3399936e4bbedd22c44f076" dependencies = [ "async-std", "cdn-proto", "clap", - "derive_builder 0.13.1", + "derive_builder", "jf-primitives", "tokio", "tracing", @@ -1340,7 +1340,7 @@ dependencies = [ [[package]] name = "cdn-proto" version = "0.1.0" -source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.1.12#3082083b1e67dd232e0cbb6519c643b9cfa93b2b" +source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.1.14#b7aed2568eba6e88c3399936e4bbedd22c44f076" dependencies = [ "anyhow", "ark-serialize", @@ -1995,38 +1995,14 @@ dependencies = [ "synstructure", ] -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - [[package]] name = "darling" version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" dependencies = [ - "darling_core 0.20.8", - "darling_macro 0.20.8", -] - -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", + "darling_core", + "darling_macro", ] [[package]] @@ -2043,24 +2019,13 @@ dependencies = [ "syn 2.0.55", ] -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core 0.14.4", - "quote", - "syn 1.0.109", -] - [[package]] name = "darling_macro" version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ - "darling_core 0.20.8", + "darling_core", "quote", "syn 2.0.55", ] @@ -2161,34 +2126,13 @@ dependencies = [ "syn 2.0.55", ] -[[package]] -name = "derive_builder" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f59169f400d8087f238c5c0c7db6a28af18681717f3b623227d92f397e938c7" -dependencies = [ - "derive_builder_macro 0.13.1", -] - [[package]] name = "derive_builder" version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" dependencies = [ - "derive_builder_macro 0.20.0", -] - -[[package]] -name = "derive_builder_core" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ec317cc3e7ef0928b0ca6e4a634a4d6c001672ae210438cf114a83e56b018d" -dependencies = [ - "darling 0.14.4", - "proc-macro2", - "quote", - "syn 1.0.109", + "derive_builder_macro", ] [[package]] @@ -2197,29 +2141,19 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ - "darling 0.20.8", + "darling", "proc-macro2", "quote", "syn 2.0.55", ] -[[package]] -name = "derive_builder_macro" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870368c3fb35b8031abb378861d4460f573b92238ec2152c927a21f77e3e0127" -dependencies = [ - "derive_builder_core 0.13.1", - "syn 1.0.109", -] - [[package]] name = "derive_builder_macro" version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ - "derive_builder_core 0.20.0", + "derive_builder_core", "syn 2.0.55", ] @@ -3286,7 +3220,7 @@ dependencies = [ "hotshot-web-server", "libp2p-identity", "libp2p-networking", - "local-ip-address 0.6.1", + "local-ip-address", "lru", "portpicker", "rand 0.8.5", @@ -3444,7 +3378,7 @@ dependencies = [ "async-trait", "blake3", "commit", - "derive_builder 0.20.0", + "derive_builder", "either", "futures", "hotshot", @@ -4525,7 +4459,7 @@ dependencies = [ "async-trait", "blake3", "custom_debug", - "derive_builder 0.20.0", + "derive_builder", "either", "futures", "hotshot-types", @@ -4951,18 +4885,6 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" -[[package]] -name = "local-ip-address" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612ed4ea9ce5acfb5d26339302528a5e1e59dfed95e9e11af3c083236ff1d15d" -dependencies = [ - "libc", - "neli", - "thiserror", - "windows-sys 0.48.0", -] - [[package]] name = "local-ip-address" version = "0.6.1" @@ -6928,7 +6850,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" dependencies = [ - "darling 0.20.8", + "darling", "proc-macro2", "quote", "syn 2.0.55", @@ -7312,6 +7234,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" dependencies = [ "ahash 0.8.11", + "async-io 1.13.0", + "async-std", "atoi", "byteorder", "bytes 1.5.0", @@ -7364,6 +7288,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" dependencies = [ + "async-std", "dotenvy", "either", "heck 0.4.1", diff --git a/Cargo.toml b/Cargo.toml index 9aa0cea124..a995d48bf3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,9 +132,9 @@ anyhow = "1.0.81" # Push CDN imports -cdn-client = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.12" } -cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.12" } -cdn-marshal = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.12" } +cdn-client = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.14" } +cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.14" } +cdn-marshal = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.14" } ### Profiles ### @@ -153,13 +153,6 @@ opt-level = 1 strip = true debug = 0 -# The default release profile without LTO. -[profile.release] -debug = 1 -# Disable LTO to decrease the linking time. -lto = "off" -incremental = true - # The release profile with LTO. # CI uses this profile. To run it locally, add `--profile=release-lto` to `cargo build` or `cargo # test`. diff --git a/crates/examples/infra/mod.rs b/crates/examples/infra/mod.rs index 405fe9642b..2af3c172f3 100644 --- a/crates/examples/infra/mod.rs +++ b/crates/examples/infra/mod.rs @@ -444,6 +444,7 @@ async fn libp2p_network_from_config( // NOTE: this introduces an invariant that the keys are assigned using this indexed // function all_keys, + #[cfg(feature = "hotshot-testing")] None, da_keys.clone(), da_keys.contains(&pub_key), diff --git a/docker/cdn-broker.Dockerfile b/docker/cdn-broker.Dockerfile index 08a8605190..bfcef89e92 100644 --- a/docker/cdn-broker.Dockerfile +++ b/docker/cdn-broker.Dockerfile @@ -1,24 +1,14 @@ -# Use a temporary builder image -FROM docker.io/rust:1.76-bookworm as BUILDER +FROM ubuntu:jammy -# Set up the working directory -WORKDIR /build -COPY . . +ARG TARGETARCH +ARG ASYNC_EXECUTOR -# Build our example -RUN RUSTFLAGS='--cfg async_executor_impl="tokio" --cfg async_channel_impl="tokio"' cargo build --profile release-lto --example broker-push-cdn +COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/cdn-broker /usr/local/bin/cdn-broker -# Use a minimal image for the final build -FROM debian:bookworm as RUNNER +# logging +ENV RUST_LOG="warn" -# Install necessary dependencies -RUN apt-get update && apt-get install libcurl4 -y +# log format. JSON no ansi +ENV RUST_LOG_FORMAT="json" -# Set the Rust log level -ENV RUST_LOG=info - -# Copy the built binary from the builder image -COPY --from=BUILDER ./build/target/release-lto/examples/broker-push-cdn /bin/broker-push-cdn - -# Set the entrypoint -ENTRYPOINT ["/bin/broker-push-cdn"] \ No newline at end of file +ENTRYPOINT ["cdn-broker"] diff --git a/docker/cdn-marshal.Dockerfile b/docker/cdn-marshal.Dockerfile index e7daca51fe..aecb0fb30c 100644 --- a/docker/cdn-marshal.Dockerfile +++ b/docker/cdn-marshal.Dockerfile @@ -1,24 +1,14 @@ -# Use a temporary builder image -FROM docker.io/rust:1.76-bookworm as BUILDER +FROM ubuntu:jammy -# Set up the working directory -WORKDIR /build -COPY . . +ARG TARGETARCH +ARG ASYNC_EXECUTOR -# Build our example -RUN RUSTFLAGS='--cfg async_executor_impl="tokio" --cfg async_channel_impl="tokio"' cargo build --profile release-lto --example marshal-push-cdn +COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/cdn-marshal /usr/local/bin/cdn-marshal -# Use a minimal image for the final build -FROM debian:bookworm as RUNNER +# logging +ENV RUST_LOG="warn" -# Install necessary dependencies -RUN apt-get update && apt-get install libcurl4 -y +# log format. JSON no ansi +ENV RUST_LOG_FORMAT="json" -# Set the Rust log level -ENV RUST_LOG=info - -# Copy the built binary from the builder image -COPY --from=BUILDER ./build/target/release-lto/examples/marshal-push-cdn /bin/marshal-push-cdn - -# Set the entrypoint -ENTRYPOINT ["/bin/marshal-push-cdn"] \ No newline at end of file +ENTRYPOINT ["cdn-marshal"] diff --git a/docker/orchestrator-libp2p.Dockerfile b/docker/orchestrator-libp2p.Dockerfile index 95876a6a1e..7a0cd1f853 100644 --- a/docker/orchestrator-libp2p.Dockerfile +++ b/docker/orchestrator-libp2p.Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update \ ARG TARGETARCH ARG ASYNC_EXECUTOR -COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/debug/examples/orchestrator-libp2p /usr/local/bin/orchestrator-libp2p +COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/orchestrator-libp2p /usr/local/bin/orchestrator-libp2p # logging ENV RUST_LOG="warn" diff --git a/docker/orchestrator-webserver.Dockerfile b/docker/orchestrator-webserver.Dockerfile index b5961c7c94..6d769b7b31 100644 --- a/docker/orchestrator-webserver.Dockerfile +++ b/docker/orchestrator-webserver.Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update \ ARG TARGETARCH ARG ASYNC_EXECUTOR -COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/debug/examples/orchestrator-webserver /usr/local/bin/orchestrator-webserver +COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/orchestrator-webserver /usr/local/bin/orchestrator-webserver # logging ENV RUST_LOG="warn" diff --git a/docker/validator-libp2p.Dockerfile b/docker/validator-libp2p.Dockerfile index 94082d1177..ece5e97d6a 100644 --- a/docker/validator-libp2p.Dockerfile +++ b/docker/validator-libp2p.Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update \ ARG TARGETARCH ARG ASYNC_EXECUTOR -COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/debug/examples/validator-libp2p /usr/local/bin/validator-libp2p +COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-libp2p /usr/local/bin/validator-libp2p # logging ENV RUST_LOG="warn" diff --git a/docker/validator-webserver-local.Dockerfile b/docker/validator-webserver-local.Dockerfile index 02a51aad1a..482723e344 100644 --- a/docker/validator-webserver-local.Dockerfile +++ b/docker/validator-webserver-local.Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update \ ARG ASYNC_EXECUTOR=async-std -COPY ./target/${ASYNC_EXECUTOR}/debug/examples/validator-webserver /usr/local/bin/validator-webserver +COPY ./target/${ASYNC_EXECUTOR}/release/examples/validator-webserver /usr/local/bin/validator-webserver # logging ENV RUST_LOG="warn" diff --git a/docker/validator-webserver.Dockerfile b/docker/validator-webserver.Dockerfile index 88706bc568..a13aa5058d 100644 --- a/docker/validator-webserver.Dockerfile +++ b/docker/validator-webserver.Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update \ ARG TARGETARCH ARG ASYNC_EXECUTOR -COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/debug/examples/validator-webserver /usr/local/bin/validator-webserver +COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-webserver /usr/local/bin/validator-webserver # logging ENV RUST_LOG="warn" diff --git a/docker/webserver.Dockerfile b/docker/webserver.Dockerfile index cf6c275e46..6415ddd9a1 100644 --- a/docker/webserver.Dockerfile +++ b/docker/webserver.Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update \ ARG TARGETARCH ARG ASYNC_EXECUTOR -COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/debug/examples/webserver /usr/local/bin/webserver +COPY ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/webserver /usr/local/bin/webserver # logging ENV RUST_LOG="warn" diff --git a/justfile b/justfile index 4270f8cfcb..f1c9f665c4 100644 --- a/justfile +++ b/justfile @@ -35,8 +35,8 @@ async := "async-std" build: cargo build --workspace --examples --bins --tests --lib --benches -build_release: - cargo build --package hotshot --profile=release --no-default-features --features="docs, doc-images" +build_release *ARGS: + cargo build --profile=release {{ARGS}} example *ARGS: cargo run --profile=release-lto --example {{ARGS}}