From 36655197e83dee68ba76e1a1a62e9f81000152e4 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 3 Dec 2024 14:53:29 +0500 Subject: [PATCH 01/25] Add Dockerfiles --- docker-compose.yaml | 19 ++++++++ packages/ciphernode/Dockerfile | 49 ++++++++++++++++++++ packages/ciphernode/ciphernode-entrypoint.sh | 40 ++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 docker-compose.yaml create mode 100644 packages/ciphernode/Dockerfile create mode 100644 packages/ciphernode/ciphernode-entrypoint.sh diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 00000000..e8635bb3 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,19 @@ +services: + ciphernode: + build: + context: . + dockerfile: ./packages/ciphernode/Dockerfile + image: ciphernode:latest + container_name: ciphernode + volumes: + - ${PWD}/config/config.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory + - ${PWD}/secrets/secrets.json:/home/ciphernode/secrets/secrets.json:ro # Read-only secrets directory + - ciphernode-data:/home/ciphernode/.local/share/enclave # Persistent data + environment: + RUST_LOG: "info" + ports: + - "8080:8080" + restart: unless-stopped + +volumes: + ciphernode-data: diff --git a/packages/ciphernode/Dockerfile b/packages/ciphernode/Dockerfile new file mode 100644 index 00000000..db82847e --- /dev/null +++ b/packages/ciphernode/Dockerfile @@ -0,0 +1,49 @@ +FROM node:20 AS evm-builder + +WORKDIR /build/packages/evm +COPY ./packages/evm ./ +RUN yarn install && yarn compile + +# Build stage +FROM rust:1.81 AS ciphernode-builder + +# Create build directory +WORKDIR /build/packages/ciphernode +COPY ./packages/ciphernode ./ +COPY --from=evm-builder /build/packages/evm/artifacts ../evm/artifacts +RUN cargo build --release + +# Runtime stage +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends iptables ca-certificates jq && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Create non-root user +RUN useradd -m -u 1000 -s /bin/bash ciphernode + +# Create necessary directories with proper permissions +RUN mkdir -p /home/ciphernode/.config/enclave \ + /home/ciphernode/.local/share/enclave \ + /home/ciphernode/secrets && \ + chown -R ciphernode:ciphernode /home/ciphernode && \ + chmod 700 /home/ciphernode/secrets + +# Switch to non-root user +USER ciphernode +WORKDIR /home/ciphernode + +# Copy binary from builder +COPY --from=ciphernode-builder --chown=ciphernode:ciphernode /build/packages/ciphernode/target/release/enclave /usr/local/bin/ + +# Environment variables for configuration +ENV CONFIG_DIR=/home/ciphernode/.config/enclave +ENV SECRETS_DIR=/home/ciphernode/secrets +ENV DATA_DIR=/home/ciphernode/.local/share/enclave +ENV RUST_LOG=info + +# Add entrypoint script +COPY --from=ciphernode-builder --chmod=755 --chown=ciphernode:ciphernode /build/packages/ciphernode/ciphernode-entrypoint.sh /usr/local/bin/ + +ENTRYPOINT ["ciphernode-entrypoint.sh"] \ No newline at end of file diff --git a/packages/ciphernode/ciphernode-entrypoint.sh b/packages/ciphernode/ciphernode-entrypoint.sh new file mode 100644 index 00000000..d627e494 --- /dev/null +++ b/packages/ciphernode/ciphernode-entrypoint.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e + +# Paths to config and secrets +CONFIG_FILE="$CONFIG_DIR/config.yaml" +SECRETS_FILE="$SECRETS_DIR/secrets.json" +KEYFILE="$CONFIG_DIR/key" + +# Ensure required files exist +if [ ! -f "$CONFIG_FILE" ]; then + echo "Error: Config file $CONFIG_FILE not found!" + exit 1 +fi + +if [ ! -f "$SECRETS_FILE" ]; then + echo "Error: Secrets file $SECRETS_FILE not found!" + exit 1 +fi + +# Read secrets from the JSON file +PRIVATE_KEY=$(jq -r '.private_key' "$SECRETS_FILE") +PASSWORD=$(jq -r '.password' "$SECRETS_FILE") + +if [ -z "$PRIVATE_KEY" ] || [ -z "$PASSWORD" ]; then + echo "Error: Missing 'private_key' or 'password' in secrets file!" + exit 1 +fi + +# Set password and private key +echo "Setting password" +enclave password create --config "$CONFIG_FILE" --password "$PASSWORD" + +if [ -f "$KEYFILE" ]; then + echo "Setting private key" + enclave wallet set --config "$CONFIG_FILE" --private-key "$PRIVATE_KEY" +fi + +echo "Starting aggregator" +# Start the aggregator +exec enclave aggregator start --config "$CONFIG_FILE" From 2486ce18d12f205099f62178ddd1f957789cf8ce Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 3 Dec 2024 15:15:41 +0500 Subject: [PATCH 02/25] Expose Udp port --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index e8635bb3..a757dcc8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,6 +13,7 @@ services: RUST_LOG: "info" ports: - "8080:8080" + - "47678:47678/udp" restart: unless-stopped volumes: From c38c5247305a121057e0494f2931993a72d45c22 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 3 Dec 2024 15:54:52 +0500 Subject: [PATCH 03/25] Update Entry point for CN --- docker-compose.yaml | 5 +++-- packages/ciphernode/ciphernode-entrypoint.sh | 20 +++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index a757dcc8..1d3d64e1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,11 +6,12 @@ services: image: ciphernode:latest container_name: ciphernode volumes: - - ${PWD}/config/config.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory - - ${PWD}/secrets/secrets.json:/home/ciphernode/secrets/secrets.json:ro # Read-only secrets directory + - ${PWD}/config.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory + - ${PWD}/secrets.json:/home/ciphernode/secrets/secrets.json:ro # Read-only secrets directory - ciphernode-data:/home/ciphernode/.local/share/enclave # Persistent data environment: RUST_LOG: "info" + AGGREGATOR: "false" ports: - "8080:8080" - "47678:47678/udp" diff --git a/packages/ciphernode/ciphernode-entrypoint.sh b/packages/ciphernode/ciphernode-entrypoint.sh index d627e494..e31a1193 100644 --- a/packages/ciphernode/ciphernode-entrypoint.sh +++ b/packages/ciphernode/ciphernode-entrypoint.sh @@ -5,6 +5,7 @@ set -e CONFIG_FILE="$CONFIG_DIR/config.yaml" SECRETS_FILE="$SECRETS_DIR/secrets.json" KEYFILE="$CONFIG_DIR/key" +AGGREGATOR="$AGGREGATOR" # Ensure required files exist if [ ! -f "$CONFIG_FILE" ]; then @@ -29,12 +30,17 @@ fi # Set password and private key echo "Setting password" enclave password create --config "$CONFIG_FILE" --password "$PASSWORD" - -if [ -f "$KEYFILE" ]; then - echo "Setting private key" - enclave wallet set --config "$CONFIG_FILE" --private-key "$PRIVATE_KEY" +if [ "$AGGREGATOR" = "true" ]; then + if [ -f "$KEYFILE" ]; then + echo "Setting private key" + enclave wallet set --config "$CONFIG_FILE" --private-key "$PRIVATE_KEY" + fi + echo "Starting aggregator" + # Start the aggregator + exec enclave aggregator start --config "$CONFIG_FILE" +else + echo "Starting Ciphernode" + exec enclave start --config "$CONFIG_FILE" fi -echo "Starting aggregator" -# Start the aggregator -exec enclave aggregator start --config "$CONFIG_FILE" + From 5beb9a3fe7cc5ade8d560675fdc50f6b73b58343 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 3 Dec 2024 22:03:39 +0500 Subject: [PATCH 04/25] Dev compose --- docker-compose.dev.yml | 3 +++ docker-compose.yml | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.yml diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..bf11b7e0 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,3 @@ +services: + ciphernode: + network_mode: "host" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..e61e11be --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +services: + ciphernode: + build: + context: . + dockerfile: ./packages/ciphernode/Dockerfile + image: ciphernode:latest + volumes: + - ${CONFIG_FILE}:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory + - ${SECRETS_FILE}:/home/ciphernode/secrets/secrets.json:ro # Read-only secrets directory + - ciphernode-data:/home/ciphernode/.local/share/enclave # Persistent data + environment: + RUST_LOG: "info" + AGGREGATOR: "false" + restart: unless-stopped + +volumes: + ciphernode-data: From 7150316aa369befe0122ac248e0df4b4a73598f3 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 3 Dec 2024 22:03:53 +0500 Subject: [PATCH 05/25] delete old compose --- docker-compose.yaml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 1d3d64e1..00000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,21 +0,0 @@ -services: - ciphernode: - build: - context: . - dockerfile: ./packages/ciphernode/Dockerfile - image: ciphernode:latest - container_name: ciphernode - volumes: - - ${PWD}/config.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory - - ${PWD}/secrets.json:/home/ciphernode/secrets/secrets.json:ro # Read-only secrets directory - - ciphernode-data:/home/ciphernode/.local/share/enclave # Persistent data - environment: - RUST_LOG: "info" - AGGREGATOR: "false" - ports: - - "8080:8080" - - "47678:47678/udp" - restart: unless-stopped - -volumes: - ciphernode-data: From 561e6e7d2a4768d5d0257e7abebd6231a28aceb3 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 3 Dec 2024 22:04:16 +0500 Subject: [PATCH 06/25] push image to ecr --- .github/workflows/ecs-deployment.yml | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/ecs-deployment.yml diff --git a/.github/workflows/ecs-deployment.yml b/.github/workflows/ecs-deployment.yml new file mode 100644 index 00000000..7adb9e62 --- /dev/null +++ b/.github/workflows/ecs-deployment.yml @@ -0,0 +1,55 @@ +name: Build and Deploy Ciphernode to ECS + +on: + push: + branches: + - main + paths: + - 'packages/ciphernode/**' + - 'packages/evm/contracts/**' + pull_request: + branches: + - main + paths: + - 'packages/ciphernode/**' + - 'packages/evm/contracts/**' + +env: + AWS_REGION: us-east-1 + ECR_REPOSITORY: ciphernode + DOCKERFILE_PATH: packages/ciphernode/Dockerfile + +permissions: + contents: read + id-token: write + +jobs: + deploy: + name: Deploy + runs-on: ubuntu-latest + environment: production + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f $DOCKERFILE_PATH . + docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest + docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:latest" >> $GITHUB_OUTPUT \ No newline at end of file From 6676c61928e7a8a5ddaea66b9f89095439179aa0 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Wed, 4 Dec 2024 15:28:11 +0500 Subject: [PATCH 07/25] Update ECS Deployment Workflow --- .github/workflows/ecs-deployment.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ecs-deployment.yml b/.github/workflows/ecs-deployment.yml index 7adb9e62..f5973be0 100644 --- a/.github/workflows/ecs-deployment.yml +++ b/.github/workflows/ecs-deployment.yml @@ -15,8 +15,7 @@ on: - 'packages/evm/contracts/**' env: - AWS_REGION: us-east-1 - ECR_REPOSITORY: ciphernode + ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }} DOCKERFILE_PATH: packages/ciphernode/Dockerfile permissions: @@ -37,7 +36,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Login to Amazon ECR id: login-ecr From 13c24a8f431d3d8f5b73b2b7525712d6acbc1fc6 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Wed, 4 Dec 2024 16:08:39 +0500 Subject: [PATCH 08/25] Update ECS Deployment Workflow --- .github/workflows/ecs-deployment.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ecs-deployment.yml b/.github/workflows/ecs-deployment.yml index f5973be0..ec6d0f50 100644 --- a/.github/workflows/ecs-deployment.yml +++ b/.github/workflows/ecs-deployment.yml @@ -32,6 +32,10 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Mask Infomation + run: | + echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: From b6c065970e48151e656608a6a1a5281acd20178e Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Wed, 4 Dec 2024 16:21:17 +0500 Subject: [PATCH 09/25] Update ECS Deployment Workflow --- .github/workflows/ecs-deployment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ecs-deployment.yml b/.github/workflows/ecs-deployment.yml index ec6d0f50..71e3b244 100644 --- a/.github/workflows/ecs-deployment.yml +++ b/.github/workflows/ecs-deployment.yml @@ -54,5 +54,7 @@ jobs: run: | docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f $DOCKERFILE_PATH . docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:latest" >> $GITHUB_OUTPUT \ No newline at end of file + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:latest" >> $GITHUB_OUTPUT From 0d9eb817ff8793972d9b3ba9d07f2faf895f7db5 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Thu, 5 Dec 2024 00:42:15 +0500 Subject: [PATCH 10/25] Update node version to 22 --- packages/ciphernode/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ciphernode/Dockerfile b/packages/ciphernode/Dockerfile index db82847e..6d17b061 100644 --- a/packages/ciphernode/Dockerfile +++ b/packages/ciphernode/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20 AS evm-builder +FROM node:22 AS evm-builder WORKDIR /build/packages/evm COPY ./packages/evm ./ From 6713c87fde30bd7150c43a4e6b440ad042027e18 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Thu, 5 Dec 2024 12:43:51 +0500 Subject: [PATCH 11/25] Update Entrypoint --- packages/ciphernode/ciphernode-entrypoint.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/ciphernode/ciphernode-entrypoint.sh b/packages/ciphernode/ciphernode-entrypoint.sh index e31a1193..2f6d0711 100644 --- a/packages/ciphernode/ciphernode-entrypoint.sh +++ b/packages/ciphernode/ciphernode-entrypoint.sh @@ -4,7 +4,6 @@ set -e # Paths to config and secrets CONFIG_FILE="$CONFIG_DIR/config.yaml" SECRETS_FILE="$SECRETS_DIR/secrets.json" -KEYFILE="$CONFIG_DIR/key" AGGREGATOR="$AGGREGATOR" # Ensure required files exist @@ -30,13 +29,12 @@ fi # Set password and private key echo "Setting password" enclave password create --config "$CONFIG_FILE" --password "$PASSWORD" + if [ "$AGGREGATOR" = "true" ]; then - if [ -f "$KEYFILE" ]; then - echo "Setting private key" - enclave wallet set --config "$CONFIG_FILE" --private-key "$PRIVATE_KEY" - fi + echo "Setting private key" + enclave wallet set --config "$CONFIG_FILE" --private-key "$PRIVATE_KEY" + echo "Starting aggregator" - # Start the aggregator exec enclave aggregator start --config "$CONFIG_FILE" else echo "Starting Ciphernode" From 61edb02acc8d4555eedf2b9f095ac2eb1e60e475 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Thu, 5 Dec 2024 18:16:02 +0500 Subject: [PATCH 12/25] Use Docker Swarm to manage Ciphernodes --- .github/workflows/ecs-deployment.yml | 9 +- docker-compose.dev.yml | 8 +- docker-compose.yml | 89 +++++++++++++++++-- packages/ciphernode/ciphernode-entrypoint.sh | 2 +- packages/ciphernode/config/src/app_config.rs | 7 ++ .../ciphernode/enclave_node/src/aggregator.rs | 1 + .../ciphernode/enclave_node/src/ciphernode.rs | 1 + .../ciphernode/net/src/network_manager.rs | 3 +- 8 files changed, 110 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ecs-deployment.yml b/.github/workflows/ecs-deployment.yml index 71e3b244..fa0600cf 100644 --- a/.github/workflows/ecs-deployment.yml +++ b/.github/workflows/ecs-deployment.yml @@ -1,4 +1,4 @@ -name: Build and Deploy Ciphernode to ECS +name: Build and Deploy Ciphernode on: push: @@ -52,7 +52,14 @@ jobs: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} IMAGE_TAG: ${{ github.sha }} run: | + CURRENT_IMAGE_ID=$(docker images -q $ECR_REGISTRY/$ECR_REPOSITORY:latest) docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f $DOCKERFILE_PATH . + if [ -n "$CURRENT_IMAGE_ID" ]; then + SHORT_SHA=${CURRENT_IMAGE_ID:0:12} + docker tag $CURRENT_IMAGE_ID $ECR_REGISTRY/$ECR_REPOSITORY:$SHORT_SHA + docker rmi $ECR_REGISTRY/$ECR_REPOSITORY:latest + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$SHORT_SHA + fi docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index bf11b7e0..4e3fe593 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,3 +1,9 @@ services: - ciphernode: + cn1: + network_mode: "host" + cn2: + network_mode: "host" + cn3: + network_mode: "host" + aggregator: network_mode: "host" diff --git a/docker-compose.yml b/docker-compose.yml index e61e11be..76f1ee61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,94 @@ services: - ciphernode: + cn1: + container_name: cn1 build: context: . dockerfile: ./packages/ciphernode/Dockerfile image: ciphernode:latest volumes: - - ${CONFIG_FILE}:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory - - ${SECRETS_FILE}:/home/ciphernode/secrets/secrets.json:ro # Read-only secrets directory - - ciphernode-data:/home/ciphernode/.local/share/enclave # Persistent data + - ./configs/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory + - cn1-data:/home/ciphernode/.local/share/enclave # Persistent data + secrets: + - secrets.json environment: RUST_LOG: "info" AGGREGATOR: "false" - restart: unless-stopped + deploy: + replicas: 1 + networks: + - cn1-network + + cn2: + container_name: cn2 + build: + context: . + dockerfile: ./packages/ciphernode/Dockerfile + image: ciphernode:latest + volumes: + - ./configs/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory + - cn2-data:/home/ciphernode/.local/share/enclave # Persistent data + secrets: + - secrets.json + environment: + RUST_LOG: "info" + AGGREGATOR: "false" + deploy: + replicas: 1 + networks: + - cn2-network + + cn3: + container_name: cn3 + build: + context: . + dockerfile: ./packages/ciphernode/Dockerfile + image: ciphernode:latest + volumes: + - ./configs/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory + - cn3-data:/home/ciphernode/.local/share/enclave # Persistent data + secrets: + - secrets.json + environment: + RUST_LOG: "info" + AGGREGATOR: "false" + deploy: + replicas: 1 + networks: + - cn3-network + + + aggregator: + container_name: aggregator + build: + context: . + dockerfile: ./packages/ciphernode/Dockerfile + image: ciphernode:latest + volumes: + - ./configs/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory + - agg-data:/home/ciphernode/.local/share/enclave # Persistent data + secrets: + - secrets.json + environment: + RUST_LOG: "info" + AGGREGATOR: "true" + deploy: + replicas: 1 + networks: + - agg-network + +secrets: + secrets.json: + file: secrets.json + volumes: - ciphernode-data: + cn1-data: + cn2-data: + cn3-data: + agg-data: + +networks: + cn1-network: + cn2-network: + cn3-network: + agg-network: diff --git a/packages/ciphernode/ciphernode-entrypoint.sh b/packages/ciphernode/ciphernode-entrypoint.sh index 2f6d0711..3fa42fdd 100644 --- a/packages/ciphernode/ciphernode-entrypoint.sh +++ b/packages/ciphernode/ciphernode-entrypoint.sh @@ -3,7 +3,7 @@ set -e # Paths to config and secrets CONFIG_FILE="$CONFIG_DIR/config.yaml" -SECRETS_FILE="$SECRETS_DIR/secrets.json" +SECRETS_FILE="/run/secrets/secrets.json" AGGREGATOR="$AGGREGATOR" # Ensure required files exist diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index 903af725..e7e073b1 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -89,6 +89,8 @@ pub struct AppConfig { address: Option
, /// A list of libp2p multiaddrs to dial to as peers when joining the network peers: Vec, + /// The port to use for the quic listener + quic_port: u16, } impl Default for AppConfig { @@ -104,6 +106,7 @@ impl Default for AppConfig { peers: vec![], // NOTE: This should remain empty and we should look at config // generation via ipns fetch for the latest nodes address: None, + quic_port: 9091, } } } @@ -179,6 +182,10 @@ impl AppConfig { pub fn peers(&self) -> Vec { self.peers.clone() } + + pub fn quic_port(&self) -> u16 { + self.quic_port + } } /// Load the config at the config_file or the default location if not provided diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 1cfbfbdf..b1802520 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -85,6 +85,7 @@ pub async fn setup_aggregator( bus.clone(), config.peers(), &cipher, + config.quic_port(), repositories.libp2pid(), ) .await?; diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 4e760614..714cea34 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -77,6 +77,7 @@ pub async fn setup_ciphernode( bus.clone(), config.peers(), &cipher, + config.quic_port(), repositories.libp2pid(), ) .await?; diff --git a/packages/ciphernode/net/src/network_manager.rs b/packages/ciphernode/net/src/network_manager.rs index 2c1b3f02..5133fc69 100644 --- a/packages/ciphernode/net/src/network_manager.rs +++ b/packages/ciphernode/net/src/network_manager.rs @@ -72,6 +72,7 @@ impl NetworkManager { bus: Addr, peers: Vec, cipher: &Arc, + quic_port: u16, repository: Repository>, ) -> Result<(Addr, tokio::task::JoinHandle>, String)> { info!("Reading from repository"); @@ -93,7 +94,7 @@ impl NetworkManager { let ed25519_keypair = ed25519::Keypair::try_from_bytes(&mut bytes)?; let keypair: libp2p::identity::Keypair = ed25519_keypair.try_into()?; - let mut peer = NetworkPeer::new(&keypair, peers, None, "tmp-enclave-gossip-topic")?; + let mut peer = NetworkPeer::new(&keypair, peers, Some(quic_port), "tmp-enclave-gossip-topic")?; let rx = peer.rx().ok_or(anyhow!("Peer rx already taken"))?; let p2p_addr = NetworkManager::setup(bus, peer.tx(), rx); let handle = tokio::spawn(async move { Ok(peer.start().await?) }); From 9b5c097de071d241c9853f9914cc8cfc190838e2 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Thu, 5 Dec 2024 19:34:43 +0500 Subject: [PATCH 13/25] Mount Secrets --- docker-compose.yml | 26 +++++++------------ packages/ciphernode/Dockerfile | 8 +++--- .../ciphernode/net/src/network_manager.rs | 3 ++- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 76f1ee61..83261fbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,5 @@ services: cn1: - container_name: cn1 - build: - context: . - dockerfile: ./packages/ciphernode/Dockerfile image: ciphernode:latest volumes: - ./configs/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory @@ -13,6 +9,8 @@ services: environment: RUST_LOG: "info" AGGREGATOR: "false" + ports: + - 9091:9091 deploy: replicas: 1 networks: @@ -20,10 +18,6 @@ services: cn2: - container_name: cn2 - build: - context: . - dockerfile: ./packages/ciphernode/Dockerfile image: ciphernode:latest volumes: - ./configs/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory @@ -33,16 +27,14 @@ services: environment: RUST_LOG: "info" AGGREGATOR: "false" + ports: + - 9092:9092 deploy: replicas: 1 networks: - cn2-network cn3: - container_name: cn3 - build: - context: . - dockerfile: ./packages/ciphernode/Dockerfile image: ciphernode:latest volumes: - ./configs/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory @@ -52,6 +44,8 @@ services: environment: RUST_LOG: "info" AGGREGATOR: "false" + ports: + - 9093:9093 deploy: replicas: 1 networks: @@ -59,10 +53,6 @@ services: aggregator: - container_name: aggregator - build: - context: . - dockerfile: ./packages/ciphernode/Dockerfile image: ciphernode:latest volumes: - ./configs/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory @@ -72,6 +62,8 @@ services: environment: RUST_LOG: "info" AGGREGATOR: "true" + ports: + - 9094:9094 deploy: replicas: 1 networks: @@ -79,7 +71,7 @@ services: secrets: secrets.json: - file: secrets.json + file: ./configs/secrets.json volumes: cn1-data: diff --git a/packages/ciphernode/Dockerfile b/packages/ciphernode/Dockerfile index 6d17b061..b58b1541 100644 --- a/packages/ciphernode/Dockerfile +++ b/packages/ciphernode/Dockerfile @@ -26,9 +26,8 @@ RUN useradd -m -u 1000 -s /bin/bash ciphernode # Create necessary directories with proper permissions RUN mkdir -p /home/ciphernode/.config/enclave \ /home/ciphernode/.local/share/enclave \ - /home/ciphernode/secrets && \ - chown -R ciphernode:ciphernode /home/ciphernode && \ - chmod 700 /home/ciphernode/secrets + /run/secrets && \ + chown -R ciphernode:ciphernode /home/ciphernode /run/secrets # Switch to non-root user USER ciphernode @@ -36,14 +35,13 @@ WORKDIR /home/ciphernode # Copy binary from builder COPY --from=ciphernode-builder --chown=ciphernode:ciphernode /build/packages/ciphernode/target/release/enclave /usr/local/bin/ +COPY --from=ciphernode-builder --chmod=755 --chown=ciphernode:ciphernode /build/packages/ciphernode/ciphernode-entrypoint.sh /usr/local/bin/ # Environment variables for configuration ENV CONFIG_DIR=/home/ciphernode/.config/enclave -ENV SECRETS_DIR=/home/ciphernode/secrets ENV DATA_DIR=/home/ciphernode/.local/share/enclave ENV RUST_LOG=info # Add entrypoint script -COPY --from=ciphernode-builder --chmod=755 --chown=ciphernode:ciphernode /build/packages/ciphernode/ciphernode-entrypoint.sh /usr/local/bin/ ENTRYPOINT ["ciphernode-entrypoint.sh"] \ No newline at end of file diff --git a/packages/ciphernode/net/src/network_manager.rs b/packages/ciphernode/net/src/network_manager.rs index 5133fc69..d5e2e966 100644 --- a/packages/ciphernode/net/src/network_manager.rs +++ b/packages/ciphernode/net/src/network_manager.rs @@ -94,7 +94,8 @@ impl NetworkManager { let ed25519_keypair = ed25519::Keypair::try_from_bytes(&mut bytes)?; let keypair: libp2p::identity::Keypair = ed25519_keypair.try_into()?; - let mut peer = NetworkPeer::new(&keypair, peers, Some(quic_port), "tmp-enclave-gossip-topic")?; + let mut peer = + NetworkPeer::new(&keypair, peers, Some(quic_port), "tmp-enclave-gossip-topic")?; let rx = peer.rx().ok_or(anyhow!("Peer rx already taken"))?; let p2p_addr = NetworkManager::setup(bus, peer.tx(), rx); let handle = tokio::spawn(async move { Ok(peer.start().await?) }); From 454e6577181662097c94731665df6658cad56d13 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Thu, 5 Dec 2024 20:26:47 +0500 Subject: [PATCH 14/25] Switch to ghcr.io --- .github/workflows/ecs-deployment.yml | 51 +++++++++++----------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ecs-deployment.yml b/.github/workflows/ecs-deployment.yml index fa0600cf..4aaf5b02 100644 --- a/.github/workflows/ecs-deployment.yml +++ b/.github/workflows/ecs-deployment.yml @@ -1,4 +1,4 @@ -name: Build and Deploy Ciphernode +name: Build and Deploy Ciphernode on: push: @@ -15,16 +15,16 @@ on: - 'packages/evm/contracts/**' env: - ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }} DOCKERFILE_PATH: packages/ciphernode/Dockerfile + IMAGE_NAME: ghcr.io/gnosisguild/ciphernode permissions: contents: read - id-token: write + packages: write jobs: deploy: - name: Deploy + name: Deploy to GHCR runs-on: ubuntu-latest environment: production @@ -32,36 +32,25 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Mask Infomation - run: | - echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ vars.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Build, tag, and push image to Amazon ECR + - name: Build, tag, and push image to GHCR id: build-image env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} IMAGE_TAG: ${{ github.sha }} run: | - CURRENT_IMAGE_ID=$(docker images -q $ECR_REGISTRY/$ECR_REPOSITORY:latest) - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f $DOCKERFILE_PATH . - if [ -n "$CURRENT_IMAGE_ID" ]; then - SHORT_SHA=${CURRENT_IMAGE_ID:0:12} - docker tag $CURRENT_IMAGE_ID $ECR_REGISTRY/$ECR_REPOSITORY:$SHORT_SHA - docker rmi $ECR_REGISTRY/$ECR_REPOSITORY:latest - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$SHORT_SHA - fi - docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:latest" >> $GITHUB_OUTPUT + # Build the image + docker build -t $IMAGE_NAME:$IMAGE_TAG -t $IMAGE_NAME:latest -f $DOCKERFILE_PATH . + + # Push both tagged and latest images + docker push $IMAGE_NAME:$IMAGE_TAG + docker push $IMAGE_NAME:latest + + # Output image details + echo "image=$IMAGE_NAME:$IMAGE_TAG" >> $GITHUB_OUTPUT + echo "image=$IMAGE_NAME:latest" >> $GITHUB_OUTPUT From 68a3976cb034c6cd7ea65b5df016c8bb3bda7ecf Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Thu, 5 Dec 2024 20:51:19 +0500 Subject: [PATCH 15/25] Update image tag --- docker-compose.dev.yml | 15 +++++++++++---- docker-compose.yml | 24 ++++++++++++------------ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 4e3fe593..c43f4d29 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,9 +1,16 @@ services: cn1: - network_mode: "host" + networks: + - cn-network cn2: - network_mode: "host" + networks: + - cn-network cn3: - network_mode: "host" + networks: + - cn-network aggregator: - network_mode: "host" + networks: + - cn-network + +networks: + cn-network: diff --git a/docker-compose.yml b/docker-compose.yml index 83261fbe..c3b1eebd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,9 @@ services: cn1: - image: ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode:latest volumes: - - ./configs/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory - - cn1-data:/home/ciphernode/.local/share/enclave # Persistent data + - ./configs/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - cn1-data:/home/ciphernode/.local/share/enclave secrets: - secrets.json environment: @@ -18,10 +18,10 @@ services: cn2: - image: ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode:latest volumes: - - ./configs/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory - - cn2-data:/home/ciphernode/.local/share/enclave # Persistent data + - ./configs/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - cn2-data:/home/ciphernode/.local/share/enclave secrets: - secrets.json environment: @@ -35,10 +35,10 @@ services: - cn2-network cn3: - image: ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode:latest volumes: - - ./configs/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory - - cn3-data:/home/ciphernode/.local/share/enclave # Persistent data + - ./configs/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - cn3-data:/home/ciphernode/.local/share/enclave secrets: - secrets.json environment: @@ -53,10 +53,10 @@ services: aggregator: - image: ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode:latest volumes: - - ./configs/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro # Read-only config directory - - agg-data:/home/ciphernode/.local/share/enclave # Persistent data + - ./configs/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - agg-data:/home/ciphernode/.local/share/enclave secrets: - secrets.json environment: From 81ddc6ca4c21cdafe40de01dd46e2538d0b17ddc Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Thu, 5 Dec 2024 21:45:30 +0500 Subject: [PATCH 16/25] Deploy to EC2 --- ...{ecs-deployment.yml => ec2-deployment.yml} | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) rename .github/workflows/{ecs-deployment.yml => ec2-deployment.yml} (69%) diff --git a/.github/workflows/ecs-deployment.yml b/.github/workflows/ec2-deployment.yml similarity index 69% rename from .github/workflows/ecs-deployment.yml rename to .github/workflows/ec2-deployment.yml index 4aaf5b02..2ece067a 100644 --- a/.github/workflows/ecs-deployment.yml +++ b/.github/workflows/ec2-deployment.yml @@ -54,3 +54,23 @@ jobs: # Output image details echo "image=$IMAGE_NAME:$IMAGE_TAG" >> $GITHUB_OUTPUT echo "image=$IMAGE_NAME:latest" >> $GITHUB_OUTPUT + + - name: Deploy to EC2 + uses: appleboy/ssh-action@v1.2.0 + if: github.ref == 'refs/heads/main' + with: + host: ${{ secrets.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_KEY }} + script: | + # Pull the latest image + docker pull $IMAGE_NAME:latest + + # Cd into the directory + cd /home/ec2-user/enclave + + # Pull the latest changes + git pull origin main + + # Deploy the stack + docker stack deploy -c docker-compose.yml ciphernode-stack From 43a9795e7fb7d0376b77d06bf827a922745ab32e Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Fri, 6 Dec 2024 17:52:27 +0500 Subject: [PATCH 17/25] Testing EC2 Deployment --- .github/workflows/ec2-deployment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ec2-deployment.yml b/.github/workflows/ec2-deployment.yml index 2ece067a..2742cd5f 100644 --- a/.github/workflows/ec2-deployment.yml +++ b/.github/workflows/ec2-deployment.yml @@ -57,7 +57,6 @@ jobs: - name: Deploy to EC2 uses: appleboy/ssh-action@v1.2.0 - if: github.ref == 'refs/heads/main' with: host: ${{ secrets.EC2_HOST }} username: ${{ secrets.EC2_USERNAME }} @@ -70,7 +69,7 @@ jobs: cd /home/ec2-user/enclave # Pull the latest changes - git pull origin main + git pull # Deploy the stack docker stack deploy -c docker-compose.yml ciphernode-stack From a57604576bda833b03b2dd33f6fd7bf6b3ab37df Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 9 Dec 2024 11:53:10 +0500 Subject: [PATCH 18/25] Optional mdns --- .github/workflows/ec2-deployment.yml | 2 + packages/ciphernode/Dockerfile | 2 +- packages/ciphernode/config/src/app_config.rs | 7 ++ .../ciphernode/enclave_node/src/aggregator.rs | 1 + .../ciphernode/enclave_node/src/ciphernode.rs | 1 + .../evm/src/ciphernode_registry_sol.rs | 2 +- packages/ciphernode/net/src/bin/p2p_test.rs | 7 +- .../ciphernode/net/src/network_manager.rs | 13 ++- packages/ciphernode/net/src/network_peer.rs | 98 ++++++++++--------- .../ciphernode/net/tests/docker-compose.yaml | 15 +-- packages/ciphernode/net/tests/entrypoint.sh | 7 -- 11 files changed, 82 insertions(+), 73 deletions(-) diff --git a/.github/workflows/ec2-deployment.yml b/.github/workflows/ec2-deployment.yml index 2742cd5f..39766064 100644 --- a/.github/workflows/ec2-deployment.yml +++ b/.github/workflows/ec2-deployment.yml @@ -57,12 +57,14 @@ jobs: - name: Deploy to EC2 uses: appleboy/ssh-action@v1.2.0 + if: github.ref == 'refs/heads/main' with: host: ${{ secrets.EC2_HOST }} username: ${{ secrets.EC2_USERNAME }} key: ${{ secrets.EC2_KEY }} script: | # Pull the latest image + echo "Pulling latest image $IMAGE_NAME:latest" docker pull $IMAGE_NAME:latest # Cd into the directory diff --git a/packages/ciphernode/Dockerfile b/packages/ciphernode/Dockerfile index b58b1541..03fe4cd4 100644 --- a/packages/ciphernode/Dockerfile +++ b/packages/ciphernode/Dockerfile @@ -14,7 +14,7 @@ COPY --from=evm-builder /build/packages/evm/artifacts ../evm/artifacts RUN cargo build --release # Runtime stage -FROM debian:bookworm-slim +FROM debian:stable-slim # Install runtime dependencies RUN apt-get update && apt-get install -y --no-install-recommends iptables ca-certificates jq && \ diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index e7e073b1..49f7d0ac 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -91,6 +91,8 @@ pub struct AppConfig { peers: Vec, /// The port to use for the quic listener quic_port: u16, + /// Whether to enable mDNS discovery + enable_mdns: bool, } impl Default for AppConfig { @@ -107,6 +109,7 @@ impl Default for AppConfig { // generation via ipns fetch for the latest nodes address: None, quic_port: 9091, + enable_mdns: false, } } } @@ -186,6 +189,10 @@ impl AppConfig { pub fn quic_port(&self) -> u16 { self.quic_port } + + pub fn enable_mdns(&self) -> bool { + self.enable_mdns + } } /// Load the config at the config_file or the default location if not provided diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index b1802520..4a7fc431 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -86,6 +86,7 @@ pub async fn setup_aggregator( config.peers(), &cipher, config.quic_port(), + config.enable_mdns(), repositories.libp2pid(), ) .await?; diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 714cea34..2b60177c 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -78,6 +78,7 @@ pub async fn setup_ciphernode( config.peers(), &cipher, config.quic_port(), + config.enable_mdns(), repositories.libp2pid(), ) .await?; diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index a616411b..d4846a50 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -3,7 +3,7 @@ use crate::{ helpers::{ReadonlyProvider, WithChainId}, EvmEventReader, }; -use actix::{Actor, Addr}; +use actix::Addr; use alloy::{ primitives::{LogData, B256}, sol, diff --git a/packages/ciphernode/net/src/bin/p2p_test.rs b/packages/ciphernode/net/src/bin/p2p_test.rs index 15435f2a..e44f43b8 100644 --- a/packages/ciphernode/net/src/bin/p2p_test.rs +++ b/packages/ciphernode/net/src/bin/p2p_test.rs @@ -30,10 +30,15 @@ async fn main() -> Result<()> { .ok() .and_then(|p| p.parse::().ok()); + let enable_mdns = env::var("ENABLE_MDNS") + .unwrap_or("false".to_string()) + .parse::() + .unwrap(); + let peers: Vec = dial_to.iter().cloned().collect(); let id = libp2p::identity::Keypair::generate_ed25519(); - let mut peer = NetworkPeer::new(&id, peers, udp_port, "test-topic")?; + let mut peer = NetworkPeer::new(&id, peers, udp_port, "test-topic", enable_mdns)?; // Extract input and outputs let tx = peer.tx(); diff --git a/packages/ciphernode/net/src/network_manager.rs b/packages/ciphernode/net/src/network_manager.rs index d5e2e966..8969c908 100644 --- a/packages/ciphernode/net/src/network_manager.rs +++ b/packages/ciphernode/net/src/network_manager.rs @@ -8,11 +8,10 @@ use cipher::Cipher; use data::Repository; use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; use libp2p::identity::ed25519; +use std::collections::HashSet; use std::sync::Arc; -use std::{collections::HashSet, error::Error}; use tokio::sync::mpsc::{Receiver, Sender}; use tracing::{error, info, instrument, trace}; -use zeroize::Zeroize; /// NetworkManager Actor converts between EventBus events and Libp2p events forwarding them to a /// NetworkPeer for propagation over the p2p network @@ -73,6 +72,7 @@ impl NetworkManager { peers: Vec, cipher: &Arc, quic_port: u16, + enable_mdns: bool, repository: Repository>, ) -> Result<(Addr, tokio::task::JoinHandle>, String)> { info!("Reading from repository"); @@ -94,8 +94,13 @@ impl NetworkManager { let ed25519_keypair = ed25519::Keypair::try_from_bytes(&mut bytes)?; let keypair: libp2p::identity::Keypair = ed25519_keypair.try_into()?; - let mut peer = - NetworkPeer::new(&keypair, peers, Some(quic_port), "tmp-enclave-gossip-topic")?; + let mut peer = NetworkPeer::new( + &keypair, + peers, + Some(quic_port), + "tmp-enclave-gossip-topic", + enable_mdns, + )?; let rx = peer.rx().ok_or(anyhow!("Peer rx already taken"))?; let p2p_addr = NetworkManager::setup(bus, peer.tx(), rx); let handle = tokio::spawn(async move { Ok(peer.start().await?) }); diff --git a/packages/ciphernode/net/src/network_peer.rs b/packages/ciphernode/net/src/network_peer.rs index 8e1d80d6..17151076 100644 --- a/packages/ciphernode/net/src/network_peer.rs +++ b/packages/ciphernode/net/src/network_peer.rs @@ -7,14 +7,14 @@ use libp2p::{ identity::Keypair, kad::{store::MemoryStore, Behaviour as KademliaBehaviour}, mdns, - swarm::{NetworkBehaviour, SwarmEvent}, + swarm::{behaviour::toggle::Toggle, NetworkBehaviour, SwarmEvent}, Multiaddr, Swarm, }; use std::hash::{Hash, Hasher}; use std::{hash::DefaultHasher, io::Error, time::Duration}; use tokio::{ select, - sync::mpsc::{self, channel, Receiver, Sender}, + sync::mpsc::{channel, Receiver, Sender}, }; use tracing::{debug, error, info, trace, warn}; @@ -23,7 +23,7 @@ pub struct NodeBehaviour { gossipsub: gossipsub::Behaviour, kademlia: KademliaBehaviour, connection_limits: connection_limits::Behaviour, - mdns: mdns::tokio::Behaviour, + mdns: Toggle, identify: IdentifyBehaviour, } @@ -44,6 +44,7 @@ impl NetworkPeer { peers: Vec, udp_port: Option, topic: &str, + enable_mdns: bool, ) -> Result { let (to_bus_tx, from_net_rx) = channel(100); // TODO : tune this param let (to_net_tx, from_bus_rx) = channel(100); // TODO : tune this param @@ -51,7 +52,7 @@ impl NetworkPeer { let swarm = libp2p::SwarmBuilder::with_existing_identity(id.clone()) .with_tokio() .with_quic() - .with_behaviour(create_mdns_kad_behaviour())? + .with_behaviour(|key| create_mdns_kad_behaviour(enable_mdns, key))? .with_swarm_config(|c| c.with_idle_connection_timeout(Duration::from_secs(60))) .build(); @@ -115,50 +116,53 @@ impl NetworkPeer { } } -fn create_mdns_kad_behaviour() -> impl FnOnce( - &Keypair, -) -> std::result::Result< - NodeBehaviour, - Box, -> { - |key| { - let connection_limits = connection_limits::Behaviour::new(ConnectionLimits::default()); - let identify_config = IdentifyBehaviour::new( - identify::Config::new("/kad/0.1.0".into(), key.public()) - .with_interval(Duration::from_secs(60)), // do this so we can get timeouts for dropped WebRTC connections - ); - let message_id_fn = |message: &gossipsub::Message| { - let mut s = DefaultHasher::new(); - message.data.hash(&mut s); - gossipsub::MessageId::from(s.finish().to_string()) - }; +fn create_mdns_kad_behaviour( + enable_mdns: bool, + key: &Keypair, +) -> std::result::Result> { + let connection_limits = connection_limits::Behaviour::new(ConnectionLimits::default()); + let identify_config = IdentifyBehaviour::new( + identify::Config::new("/kad/0.1.0".into(), key.public()) + .with_interval(Duration::from_secs(60)), + ); + + let message_id_fn = |message: &gossipsub::Message| { + let mut s = DefaultHasher::new(); + message.data.hash(&mut s); + gossipsub::MessageId::from(s.finish().to_string()) + }; - // TODO: Allow for config inputs to new() - let gossipsub_config = gossipsub::ConfigBuilder::default() - .heartbeat_interval(Duration::from_secs(10)) - .validation_mode(gossipsub::ValidationMode::Strict) - .message_id_fn(message_id_fn) - .build() - .map_err(|msg| Error::new(std::io::ErrorKind::Other, msg))?; - - let gossipsub = gossipsub::Behaviour::new( - gossipsub::MessageAuthenticity::Signed(key.clone()), - gossipsub_config, - )?; - - let mdns = mdns::tokio::Behaviour::new(mdns::Config::default(), key.public().to_peer_id())?; - - Ok(NodeBehaviour { - gossipsub, - kademlia: KademliaBehaviour::new( - key.public().to_peer_id(), - MemoryStore::new(key.public().to_peer_id()), - ), - mdns, - connection_limits, - identify: identify_config, - }) - } + let gossipsub_config = gossipsub::ConfigBuilder::default() + .heartbeat_interval(Duration::from_secs(10)) + .validation_mode(gossipsub::ValidationMode::Strict) + .message_id_fn(message_id_fn) + .build() + .map_err(|msg| Error::new(std::io::ErrorKind::Other, msg))?; + + let gossipsub = gossipsub::Behaviour::new( + gossipsub::MessageAuthenticity::Signed(key.clone()), + gossipsub_config, + )?; + + let mdns = if enable_mdns { + Toggle::from(Some(mdns::tokio::Behaviour::new( + mdns::Config::default(), + key.public().to_peer_id(), + )?)) + } else { + Toggle::from(None) + }; + + Ok(NodeBehaviour { + gossipsub, + kademlia: KademliaBehaviour::new( + key.public().to_peer_id(), + MemoryStore::new(key.public().to_peer_id()), + ), + mdns, + connection_limits, + identify: identify_config, + }) } async fn process_swarm_event( diff --git a/packages/ciphernode/net/tests/docker-compose.yaml b/packages/ciphernode/net/tests/docker-compose.yaml index f55d3698..d54a3046 100644 --- a/packages/ciphernode/net/tests/docker-compose.yaml +++ b/packages/ciphernode/net/tests/docker-compose.yaml @@ -11,11 +11,8 @@ services: environment: QUIC_PORT: 9091 DIAL_TO: "/ip4/172.16.238.12/udp/9091/quic-v1" - BLOCK_MDNS: "${BLOCK_MDNS:-false}" + ENABLE_MDNS: "${ENABLE_MDNS:-true}" entrypoint: ["/app/entrypoint.sh"] - cap_add: - - NET_ADMIN - - NET_RAW bob: image: p2p-test-image @@ -26,11 +23,8 @@ services: environment: QUIC_PORT: 9091 DIAL_TO: "/ip4/172.16.238.12/udp/9091/quic-v1" - BLOCK_MDNS: "${BLOCK_MDNS:-false}" + ENABLE_MDNS: "${ENABLE_MDNS:-true}" entrypoint: ["/app/entrypoint.sh"] - cap_add: - - NET_ADMIN - - NET_RAW charlie: image: p2p-test-image @@ -40,11 +34,8 @@ services: command: ["/app/p2p_test", "charlie"] environment: QUIC_PORT: 9091 - BLOCK_MDNS: "${BLOCK_MDNS:-false}" + ENABLE_MDNS: "${ENABLE_MDNS:-true}" entrypoint: ["/app/entrypoint.sh"] - cap_add: - - NET_ADMIN - - NET_RAW networks: app_net: diff --git a/packages/ciphernode/net/tests/entrypoint.sh b/packages/ciphernode/net/tests/entrypoint.sh index dfc54f53..a6453106 100755 --- a/packages/ciphernode/net/tests/entrypoint.sh +++ b/packages/ciphernode/net/tests/entrypoint.sh @@ -1,11 +1,4 @@ #!/bin/bash set -e -if [ "${BLOCK_MDNS:-false}" = "true" ]; then - iptables -A INPUT -p udp --dport 5353 -j DROP - iptables -A OUTPUT -p udp --dport 5353 -j DROP - iptables -L | grep DROP -fi - -# Execute the original command exec "$@" From 2232d39c282c55cc1b7ce4bfc3f018b9f70ab237 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 9 Dec 2024 11:56:59 +0500 Subject: [PATCH 19/25] Update Configs --- .github/workflows/ec2-deployment.yml | 2 +- tests/basic_integration/lib/ag/config.yaml | 2 ++ tests/basic_integration/lib/cn1/config.yaml | 2 ++ tests/basic_integration/lib/cn2/config.yaml | 2 ++ tests/basic_integration/lib/cn3/config.yaml | 2 ++ tests/basic_integration/lib/cn4/config.yaml | 2 ++ 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ec2-deployment.yml b/.github/workflows/ec2-deployment.yml index 39766064..cc90dcd3 100644 --- a/.github/workflows/ec2-deployment.yml +++ b/.github/workflows/ec2-deployment.yml @@ -57,7 +57,7 @@ jobs: - name: Deploy to EC2 uses: appleboy/ssh-action@v1.2.0 - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/release' with: host: ${{ secrets.EC2_HOST }} username: ${{ secrets.EC2_USERNAME }} diff --git a/tests/basic_integration/lib/ag/config.yaml b/tests/basic_integration/lib/ag/config.yaml index 68b919f1..30a968e5 100644 --- a/tests/basic_integration/lib/ag/config.yaml +++ b/tests/basic_integration/lib/ag/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199" +quic_port: 9091 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545" diff --git a/tests/basic_integration/lib/cn1/config.yaml b/tests/basic_integration/lib/cn1/config.yaml index 508112cd..3cd8d55c 100644 --- a/tests/basic_integration/lib/cn1/config.yaml +++ b/tests/basic_integration/lib/cn1/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0x2546BcD3c84621e976D8185a91A922aE77ECEc30" +quic_port: 9091 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545" diff --git a/tests/basic_integration/lib/cn2/config.yaml b/tests/basic_integration/lib/cn2/config.yaml index 5cbce324..3deb156d 100644 --- a/tests/basic_integration/lib/cn2/config.yaml +++ b/tests/basic_integration/lib/cn2/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" +quic_port: 9091 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545" diff --git a/tests/basic_integration/lib/cn3/config.yaml b/tests/basic_integration/lib/cn3/config.yaml index 258b8488..069aacc0 100644 --- a/tests/basic_integration/lib/cn3/config.yaml +++ b/tests/basic_integration/lib/cn3/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0xdD2FD4581271e230360230F9337D5c0430Bf44C0" +quic_port: 9091 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545" diff --git a/tests/basic_integration/lib/cn4/config.yaml b/tests/basic_integration/lib/cn4/config.yaml index 55bdb526..7b710825 100644 --- a/tests/basic_integration/lib/cn4/config.yaml +++ b/tests/basic_integration/lib/cn4/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0x8626f6940E2eb28930eFb4CeF49B2d1F2C9C1199" +quic_port: 9091 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545" From d575f73520dccf58adf237377919539e031637d9 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 9 Dec 2024 12:11:49 +0500 Subject: [PATCH 20/25] Update config --- tests/basic_integration/lib/ag/config.yaml | 2 +- tests/basic_integration/lib/cn2/config.yaml | 2 +- tests/basic_integration/lib/cn3/config.yaml | 2 +- tests/basic_integration/lib/cn4/config.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/basic_integration/lib/ag/config.yaml b/tests/basic_integration/lib/ag/config.yaml index 30a968e5..21e84dea 100644 --- a/tests/basic_integration/lib/ag/config.yaml +++ b/tests/basic_integration/lib/ag/config.yaml @@ -1,7 +1,7 @@ config_dir: . data_dir: . address: "0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199" -quic_port: 9091 +quic_port: 9095 enable_mdns: true chains: - name: "hardhat" diff --git a/tests/basic_integration/lib/cn2/config.yaml b/tests/basic_integration/lib/cn2/config.yaml index 3deb156d..f3e8e48e 100644 --- a/tests/basic_integration/lib/cn2/config.yaml +++ b/tests/basic_integration/lib/cn2/config.yaml @@ -1,7 +1,7 @@ config_dir: . data_dir: . address: "0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" -quic_port: 9091 +quic_port: 9092 enable_mdns: true chains: - name: "hardhat" diff --git a/tests/basic_integration/lib/cn3/config.yaml b/tests/basic_integration/lib/cn3/config.yaml index 069aacc0..9c04ab47 100644 --- a/tests/basic_integration/lib/cn3/config.yaml +++ b/tests/basic_integration/lib/cn3/config.yaml @@ -1,7 +1,7 @@ config_dir: . data_dir: . address: "0xdD2FD4581271e230360230F9337D5c0430Bf44C0" -quic_port: 9091 +quic_port: 9093 enable_mdns: true chains: - name: "hardhat" diff --git a/tests/basic_integration/lib/cn4/config.yaml b/tests/basic_integration/lib/cn4/config.yaml index 7b710825..ff73bf71 100644 --- a/tests/basic_integration/lib/cn4/config.yaml +++ b/tests/basic_integration/lib/cn4/config.yaml @@ -1,7 +1,7 @@ config_dir: . data_dir: . address: "0x8626f6940E2eb28930eFb4CeF49B2d1F2C9C1199" -quic_port: 9091 +quic_port: 9094 enable_mdns: true chains: - name: "hardhat" From 24787fac97212bf5ec273c5d1be8aef39c151271 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 9 Dec 2024 12:45:42 +0500 Subject: [PATCH 21/25] Update Deployment workflow --- .github/workflows/ec2-deployment.yml | 98 +++++++++++++++------------- tests/basic_integration/base.sh | 2 + tests/basic_integration/persist.sh | 2 + 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ec2-deployment.yml b/.github/workflows/ec2-deployment.yml index cc90dcd3..612b1024 100644 --- a/.github/workflows/ec2-deployment.yml +++ b/.github/workflows/ec2-deployment.yml @@ -3,12 +3,14 @@ name: Build and Deploy Ciphernode on: push: branches: + - release - main paths: - 'packages/ciphernode/**' - 'packages/evm/contracts/**' pull_request: branches: + - release - main paths: - 'packages/ciphernode/**' @@ -23,55 +25,61 @@ permissions: packages: write jobs: - deploy: - name: Deploy to GHCR + build: + name: Build Image runs-on: ubuntu-latest - environment: production - + outputs: + image_tag: ${{ steps.version.outputs.version }} steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build, tag, and push image to GHCR - id: build-image - env: - IMAGE_TAG: ${{ github.sha }} - run: | - # Build the image - docker build -t $IMAGE_NAME:$IMAGE_TAG -t $IMAGE_NAME:latest -f $DOCKERFILE_PATH . - - # Push both tagged and latest images - docker push $IMAGE_NAME:$IMAGE_TAG - docker push $IMAGE_NAME:latest + - uses: actions/checkout@v3 + + - name: Generate version tag + id: version + run: echo "version=$(date +'%Y%m%d')-${GITHUB_SHA::8}" >> $GITHUB_OUTPUT - # Output image details - echo "image=$IMAGE_NAME:$IMAGE_TAG" >> $GITHUB_OUTPUT - echo "image=$IMAGE_NAME:latest" >> $GITHUB_OUTPUT + - name: Build image + run: | + docker build -t $IMAGE_NAME:${{ steps.version.outputs.version }} -f $DOCKERFILE_PATH . - - name: Deploy to EC2 - uses: appleboy/ssh-action@v1.2.0 - if: github.ref == 'refs/heads/release' - with: - host: ${{ secrets.EC2_HOST }} - username: ${{ secrets.EC2_USERNAME }} - key: ${{ secrets.EC2_KEY }} - script: | - # Pull the latest image - echo "Pulling latest image $IMAGE_NAME:latest" - docker pull $IMAGE_NAME:latest + - name: Log in to GitHub Container Registry + if: github.ref == 'refs/heads/release' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - # Cd into the directory - cd /home/ec2-user/enclave + - name: Push to GHCR + if: github.ref == 'refs/heads/release' + env: + IMAGE_TAG: ${{ steps.version.outputs.version }} + run: | + docker tag $IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:latest + docker push $IMAGE_NAME:$IMAGE_TAG + docker push $IMAGE_NAME:latest - # Pull the latest changes - git pull + deploy: + name: Deploy to Production + needs: build + runs-on: ubuntu-latest + environment: + name: production + if: github.ref == 'refs/heads/release' + + steps: + - name: Deploy to EC2 + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_KEY }} + script: | + IMAGE_TAG="${{ needs.build.outputs.image_tag }}" + echo "Deploying version: $IMAGE_TAG" + docker pull $IMAGE_NAME:$IMAGE_TAG + + cd /home/ec2-user/enclave + git pull + + docker stack deploy -c docker-compose.yml ciphernode-stack - # Deploy the stack - docker stack deploy -c docker-compose.yml ciphernode-stack diff --git a/tests/basic_integration/base.sh b/tests/basic_integration/base.sh index 8765b7c5..45eeaa93 100755 --- a/tests/basic_integration/base.sh +++ b/tests/basic_integration/base.sh @@ -51,6 +51,8 @@ ENCODED_PARAMS=0x$($SCRIPT_DIR/lib/pack_e3_params.sh --moduli 0x3FFFFFFF000001 - yarn committee:new --network localhost --duration 4 --e3-params "$ENCODED_PARAMS" +sleep 2 + waiton "$SCRIPT_DIR/output/pubkey.bin" PUBLIC_KEY=$(xxd -p -c 10000000 "$SCRIPT_DIR/output/pubkey.bin") diff --git a/tests/basic_integration/persist.sh b/tests/basic_integration/persist.sh index d4f4f489..9ed7c33d 100755 --- a/tests/basic_integration/persist.sh +++ b/tests/basic_integration/persist.sh @@ -51,6 +51,8 @@ ENCODED_PARAMS=0x$($SCRIPT_DIR/lib/pack_e3_params.sh --moduli 0x3FFFFFFF000001 - yarn committee:new --network localhost --duration 4 --e3-params "$ENCODED_PARAMS" +sleep 2 + waiton "$SCRIPT_DIR/output/pubkey.bin" PUBLIC_KEY=$(xxd -p -c 10000000 "$SCRIPT_DIR/output/pubkey.bin") From 93af147e96c82af27718d4784d377cb8168be3a0 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 10 Dec 2024 15:15:37 +0500 Subject: [PATCH 22/25] Bypass overlay and use host network --- docker-compose.yml | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c3b1eebd..5573cce5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,15 +10,20 @@ services: RUST_LOG: "info" AGGREGATOR: "false" ports: - - 9091:9091 + - target: 9091 + published: 9091 + protocol: udp + mode: host deploy: replicas: 1 networks: - - cn1-network + - global-network cn2: image: ghcr.io/gnosisguild/ciphernode:latest + depends_on: + - cn1 volumes: - ./configs/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn2-data:/home/ciphernode/.local/share/enclave @@ -28,14 +33,19 @@ services: RUST_LOG: "info" AGGREGATOR: "false" ports: - - 9092:9092 + - target: 9092 + published: 9092 + protocol: udp + mode: host deploy: replicas: 1 networks: - - cn2-network + - global-network cn3: image: ghcr.io/gnosisguild/ciphernode:latest + depends_on: + - cn1 volumes: - ./configs/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn3-data:/home/ciphernode/.local/share/enclave @@ -45,15 +55,20 @@ services: RUST_LOG: "info" AGGREGATOR: "false" ports: - - 9093:9093 + - target: 9093 + published: 9093 + protocol: udp + mode: host deploy: replicas: 1 networks: - - cn3-network + - global-network aggregator: image: ghcr.io/gnosisguild/ciphernode:latest + depends_on: + - cn1 volumes: - ./configs/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - agg-data:/home/ciphernode/.local/share/enclave @@ -63,11 +78,14 @@ services: RUST_LOG: "info" AGGREGATOR: "true" ports: - - 9094:9094 + - target: 9094 + published: 9094 + protocol: udp + mode: host deploy: replicas: 1 networks: - - agg-network + - global-network secrets: secrets.json: @@ -80,7 +98,5 @@ volumes: agg-data: networks: - cn1-network: - cn2-network: - cn3-network: - agg-network: + global-network: + driver: overlay From d155714f0beec4225768ddcff6305c372248e4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Tue, 10 Dec 2024 21:27:45 +1100 Subject: [PATCH 23/25] Env substitution (#203) * Env substitution works * Move to dev-dependencies * Formatting * Ensure that there are no quotes or anything around substitution * use non normal env vars to avoid future issues --- packages/ciphernode/Cargo.lock | 55 ++++++++++------- packages/ciphernode/Cargo.toml | 2 + packages/ciphernode/config/Cargo.toml | 5 ++ packages/ciphernode/config/src/app_config.rs | 64 +++++++++++++++++++- packages/ciphernode/config/src/lib.rs | 1 + packages/ciphernode/config/src/yaml.rs | 51 ++++++++++++++++ 6 files changed, 155 insertions(+), 23 deletions(-) create mode 100644 packages/ciphernode/config/src/yaml.rs diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 580d8beb..675eb7bd 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -1104,7 +1104,7 @@ checksum = "d7ebdfa2ebdab6b1760375fa7d6f382b9f486eac35fc994625a00e89280bdbb7" dependencies = [ "async-task", "concurrent-queue", - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-lite 2.3.0", "slab", ] @@ -1169,7 +1169,7 @@ dependencies = [ "futures-lite 2.3.0", "parking", "polling 3.7.2", - "rustix 0.38.34", + "rustix 0.38.42", "slab", "tracing", "windows-sys 0.52.0", @@ -1219,7 +1219,7 @@ dependencies = [ "cfg-if", "event-listener 3.1.0", "futures-lite 1.13.0", - "rustix 0.38.34", + "rustix 0.38.42", "windows-sys 0.48.0", ] @@ -1235,7 +1235,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 0.38.34", + "rustix 0.38.42", "signal-hook-registry", "slab", "windows-sys 0.59.0", @@ -1694,6 +1694,8 @@ dependencies = [ "dirs", "figment", "serde", + "shellexpand", + "tempfile", ] [[package]] @@ -2250,12 +2252,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2337,9 +2339,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fastrlp" @@ -2614,7 +2616,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-core", "futures-io", "parking", @@ -3491,9 +3493,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libm" @@ -4626,7 +4628,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391" dependencies = [ "atomic-waker", - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-io", ] @@ -4672,7 +4674,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.4.0", "pin-project-lite", - "rustix 0.38.34", + "rustix 0.38.42", "tracing", "windows-sys 0.52.0", ] @@ -5363,15 +5365,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys 0.4.14", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5667,6 +5669,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shellexpand" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -5939,15 +5950,15 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.11.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", - "fastrand 2.1.0", + "fastrand 2.3.0", "once_cell", - "rustix 0.38.34", - "windows-sys 0.52.0", + "rustix 0.38.42", + "windows-sys 0.59.0", ] [[package]] diff --git a/packages/ciphernode/Cargo.toml b/packages/ciphernode/Cargo.toml index c7df1e8c..c19f1b9d 100644 --- a/packages/ciphernode/Cargo.toml +++ b/packages/ciphernode/Cargo.toml @@ -39,6 +39,7 @@ clap = { version = "4.5.17", features = ["derive"] } cipher = { path = "./cipher" } dirs = "5.0.1" data = { path = "./data" } +shellexpand = "3.1.0" figment = { version = "0.10.19", features = ["yaml", "test"] } fhe_rs = { package = "fhe", git = "https://github.com/gnosisguild/fhe.rs", version = "0.1.0-beta.7" } fhe-traits = { git = "https://github.com/gnosisguild/fhe.rs", version = "0.1.0-beta.7" } @@ -53,6 +54,7 @@ rand = "0.8.5" serde = { version = "1.0.208", features = ["derive"] } sled = "0.34.7" sha2 = "0.10.8" +tempfile = "3.14.0" tokio = { version = "1.38", features = ["full"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/packages/ciphernode/config/Cargo.toml b/packages/ciphernode/config/Cargo.toml index a694ceb5..93a4d74f 100644 --- a/packages/ciphernode/config/Cargo.toml +++ b/packages/ciphernode/config/Cargo.toml @@ -9,3 +9,8 @@ anyhow = { workspace = true } serde = { workspace = true } figment = { workspace = true } alloy = { workspace = true } +shellexpand = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } + diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index 49f7d0ac..9caa5da7 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -10,6 +10,8 @@ use std::{ path::{Path, PathBuf}, }; +use crate::yaml::load_yaml_with_env; + #[derive(Debug, Deserialize, Serialize, PartialEq)] #[serde(untagged)] pub enum Contract { @@ -202,8 +204,10 @@ pub fn load_config(config_file: Option<&str>) -> Result { defaults.config_file = file.into(); } + let with_envs = load_yaml_with_env(&defaults.config_file())?; + let config = Figment::from(Serialized::defaults(&defaults)) - .merge(Yaml::file(defaults.config_file())) + .merge(Yaml::string(&with_envs)) .extract()?; Ok(config) @@ -445,4 +449,62 @@ chains: Ok(()) }); } + + #[test] + fn test_config_env_vars() { + Jail::expect_with(|jail| { + let home = format!("{}", jail.directory().to_string_lossy()); + jail.set_env("HOME", &home); + jail.set_env("XDG_CONFIG_HOME", &format!("{}/.config", home)); + jail.set_env("TEST_RPC_URL_PORT", "8545"); + jail.set_env("TEST_USERNAME", "envUser"); + jail.set_env("TEST_PASSWORD", "envPassword"); + jail.set_env( + "TEST_CONTRACT_ADDRESS", + "0x1234567890123456789012345678901234567890", + ); + + let filename = format!("{}/.config/enclave/config.yaml", home); + let filedir = format!("{}/.config/enclave", home); + jail.create_dir(filedir)?; + jail.create_file( + filename, + r#" +chains: + - name: "hardhat" + rpc_url: "ws://test-endpoint:${TEST_RPC_URL_PORT}" + rpc_auth: + type: "Basic" + credentials: + username: "${TEST_USERNAME}" + password: "${TEST_PASSWORD}" + contracts: + enclave: "${TEST_CONTRACT_ADDRESS}" + ciphernode_registry: + address: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" + deploy_block: 1764352873645 + filter_registry: "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" +"#, + )?; + + let config: AppConfig = load_config(None).map_err(|err| err.to_string())?; + let chain = config.chains().first().unwrap(); + + // Test that environment variables are properly substituted + assert_eq!(chain.rpc_url, "ws://test-endpoint:8545"); + assert_eq!( + chain.rpc_auth, + RpcAuth::Basic { + username: "envUser".to_string(), + password: "envPassword".to_string(), + } + ); + assert_eq!( + chain.contracts.enclave.address(), + "0x1234567890123456789012345678901234567890" + ); + + Ok(()) + }); + } } diff --git a/packages/ciphernode/config/src/lib.rs b/packages/ciphernode/config/src/lib.rs index ba182a1b..83e06ce9 100644 --- a/packages/ciphernode/config/src/lib.rs +++ b/packages/ciphernode/config/src/lib.rs @@ -1,2 +1,3 @@ mod app_config; +mod yaml; pub use app_config::*; diff --git a/packages/ciphernode/config/src/yaml.rs b/packages/ciphernode/config/src/yaml.rs new file mode 100644 index 00000000..92da78e1 --- /dev/null +++ b/packages/ciphernode/config/src/yaml.rs @@ -0,0 +1,51 @@ +use anyhow::Result; +use std::{fs, path::PathBuf}; + +pub fn load_yaml_with_env(file_path: &PathBuf) -> Result { + // Read the file content to string + let content = match fs::read_to_string(file_path) { + Ok(val) => val, + Err(_) => "".to_string(), + }; + + // Collect environment variables and perform substitution + Ok(shellexpand::env(&content)?.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + use std::fs::File; + use std::io::Write; + use tempfile::tempdir; + + #[test] + fn test_yaml_env_substitution() -> Result<()> { + // Create a temporary directory and file + let dir = tempdir()?; + let file_path = dir.path().join("test.yaml"); + let mut file = File::create(&file_path)?; + + // Write test YAML content + writeln!( + file, + "database:\n url: $MY_DATABASE_URL\n password: ${{MY_DB_PASSWORD}}" + )?; + + // Set environment variables + env::set_var("MY_DATABASE_URL", "postgres://localhost:5432"); + env::set_var("MY_DB_PASSWORD", "secret123"); + + // Test the function + let processed = load_yaml_with_env(&file_path)?; + + env::remove_var("MY_DATABASE_URL"); + env::remove_var("MY_DB_PASSWORD"); + + assert!(processed.contains("postgres://localhost:5432")); + assert!(processed.contains("secret123")); + + Ok(()) + } +} From e99feb42227b2c2b9e0f42bccc30a3cd202d5f52 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 10 Dec 2024 16:11:29 +0500 Subject: [PATCH 24/25] merge main and update workflow deployment --- .github/workflows/ec2-deployment.yml | 15 ++++++++------- packages/ciphernode/Cargo.lock | 2 +- packages/ciphernode/config/Cargo.toml | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ec2-deployment.yml b/.github/workflows/ec2-deployment.yml index 612b1024..793b5c01 100644 --- a/.github/workflows/ec2-deployment.yml +++ b/.github/workflows/ec2-deployment.yml @@ -37,17 +37,19 @@ jobs: id: version run: echo "version=$(date +'%Y%m%d')-${GITHUB_SHA::8}" >> $GITHUB_OUTPUT - - name: Build image - run: | - docker build -t $IMAGE_NAME:${{ steps.version.outputs.version }} -f $DOCKERFILE_PATH . - - name: Log in to GitHub Container Registry - if: github.ref == 'refs/heads/release' uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build image + env: + IMAGE_TAG: ${{ steps.version.outputs.version }} + run: | + docker build -t $IMAGE_NAME:${{ steps.version.outputs.version }} -f $DOCKERFILE_PATH . + docker push $IMAGE_NAME:$IMAGE_TAG - name: Push to GHCR if: github.ref == 'refs/heads/release' @@ -55,7 +57,6 @@ jobs: IMAGE_TAG: ${{ steps.version.outputs.version }} run: | docker tag $IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:latest - docker push $IMAGE_NAME:$IMAGE_TAG docker push $IMAGE_NAME:latest deploy: diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 017e2515..0684a617 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -1694,9 +1694,9 @@ dependencies = [ "dirs", "figment", "serde", - "url", "shellexpand", "tempfile", + "url", ] [[package]] diff --git a/packages/ciphernode/config/Cargo.toml b/packages/ciphernode/config/Cargo.toml index dd6442d7..e58c695f 100644 --- a/packages/ciphernode/config/Cargo.toml +++ b/packages/ciphernode/config/Cargo.toml @@ -10,8 +10,8 @@ serde = { workspace = true } figment = { workspace = true } alloy = { workspace = true } shellexpand = { workspace = true } +url = { workspace = true } [dev-dependencies] tempfile = { workspace = true } -url = { workspace = true } From bc9e6dbb76d57be8a04a27a0d436c8a57f3f8a41 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 10 Dec 2024 16:24:51 +0500 Subject: [PATCH 25/25] Update Dockerfile and add deployments artifacts --- packages/ciphernode/Dockerfile | 1 + tests/basic_integration/base.sh | 2 -- tests/basic_integration/persist.sh | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/ciphernode/Dockerfile b/packages/ciphernode/Dockerfile index 03fe4cd4..3f559d95 100644 --- a/packages/ciphernode/Dockerfile +++ b/packages/ciphernode/Dockerfile @@ -11,6 +11,7 @@ FROM rust:1.81 AS ciphernode-builder WORKDIR /build/packages/ciphernode COPY ./packages/ciphernode ./ COPY --from=evm-builder /build/packages/evm/artifacts ../evm/artifacts +COPY --from=evm-builder /build/packages/evm/deployments ../evm/deployments RUN cargo build --release # Runtime stage diff --git a/tests/basic_integration/base.sh b/tests/basic_integration/base.sh index 45eeaa93..8765b7c5 100755 --- a/tests/basic_integration/base.sh +++ b/tests/basic_integration/base.sh @@ -51,8 +51,6 @@ ENCODED_PARAMS=0x$($SCRIPT_DIR/lib/pack_e3_params.sh --moduli 0x3FFFFFFF000001 - yarn committee:new --network localhost --duration 4 --e3-params "$ENCODED_PARAMS" -sleep 2 - waiton "$SCRIPT_DIR/output/pubkey.bin" PUBLIC_KEY=$(xxd -p -c 10000000 "$SCRIPT_DIR/output/pubkey.bin") diff --git a/tests/basic_integration/persist.sh b/tests/basic_integration/persist.sh index 9ed7c33d..d4f4f489 100755 --- a/tests/basic_integration/persist.sh +++ b/tests/basic_integration/persist.sh @@ -51,8 +51,6 @@ ENCODED_PARAMS=0x$($SCRIPT_DIR/lib/pack_e3_params.sh --moduli 0x3FFFFFFF000001 - yarn committee:new --network localhost --duration 4 --e3-params "$ENCODED_PARAMS" -sleep 2 - waiton "$SCRIPT_DIR/output/pubkey.bin" PUBLIC_KEY=$(xxd -p -c 10000000 "$SCRIPT_DIR/output/pubkey.bin")