[Push CDN] Add whitelist update binary (#2815)

* update pcdn, add whitelist adapter * remove capnproto dependency * remove self hosted ARM * add self-hosted arm back * use other orchestrator function
EspressoSystems · Mar 21, 2024 · 98ac462 · 98ac462
1 parent 573a54a
commit 98ac462
Show file tree

Hide file tree

Showing 22 changed files with 194 additions and 121 deletions.
diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml
@@ -21,10 +21,5 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
 
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler capnproto
-
       - name: Audit Dependencies
         run: cargo audit
diff --git a/.github/workflows/build-and-test-self-hosted.yml b/.github/workflows/build-and-test-self-hosted.yml
@@ -19,11 +19,6 @@ jobs:
     runs-on: [self-hosted]
     container: ghcr.io/espressosystems/devops-rust:stable
     steps:
-      - name: Install Cap'n'Proto
-        run: |
-          apt-get update
-          apt-get install -y capnproto
-
       - uses: actions/checkout@v4
         name: Checkout Repository
 

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -34,11 +34,6 @@ jobs:
           shared-key: ""
           prefix-key: ${{ matrix.just_variants }}
 
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler capnproto
-
       - name: Install Just
         run: |
           wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz
@@ -84,11 +79,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         name: Checkout Repository
-
-      - name: Install Cap'n'Proto
-        run: |
-          apt-get update
-          apt-get install -y capnproto
 
       - uses: Swatinem/rust-cache@v2
         name: Enable Rust Caching
@@ -262,10 +252,6 @@ jobs:
           shared-key: ""
           prefix-key: ${{ matrix.just_variants }}
 
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler capnproto
       - name: Install Just
         run: |
           wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz

diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
@@ -21,11 +21,6 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
 
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler capnproto
-
       - name: Install Just
         run: |
           wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz

diff --git a/.github/workflows/lint-self-hosted.yml b/.github/workflows/lint-self-hosted.yml
@@ -19,11 +19,6 @@ jobs:
     runs-on: [self-hosted]
     container: ghcr.io/espressosystems/devops-rust:stable
     steps:
-      - name: Install Cap'n'Proto
-        run: |
-          apt-get update
-          apt-get install -y capnproto
-
       - uses: actions/checkout@v4
         name: Checkout Repository
 

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -34,11 +34,6 @@ jobs:
           shared-key: ""
           prefix-key: ${{ matrix.just_variants }}
 
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler capnproto
-
       - name: Install Just
         run: |
           wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -132,9 +132,9 @@ anyhow = "1.0.81"
 
 
 # Push CDN imports
-cdn-client = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.9" }
-cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.9" }
-cdn-marshal = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.9" }
+cdn-client = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.11" }
+cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.11" }
+cdn-marshal = { git = "https://github.com/EspressoSystems/Push-CDN", tag = "0.1.11" }
 
 ### Profiles
 ###

diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml
@@ -96,6 +96,10 @@ path = "push-cdn/broker.rs"
 name = "marshal-push-cdn"
 path = "push-cdn/marshal.rs"
 
+[[example]]
+name = "whitelist-push-cdn"
+path = "push-cdn/whitelist-adapter.rs"
+
 [dependencies]
 async-broadcast = { workspace = true }
 async-compatibility-layer = { workspace = true }
@@ -137,7 +141,6 @@ cdn-broker = { workspace = true, features = [
   "insecure",
   "runtime-tokio",
   "strong_consistency",
-  "local_discovery",
 ] }
 cdn-marshal = { workspace = true, features = ["insecure", "runtime-tokio"] }
 
@@ -148,7 +151,6 @@ cdn-broker = { workspace = true, features = [
   "insecure",
   "runtime-async-std",
   "strong_consistency",
-  "local_discovery",
 ] }
 cdn-marshal = { workspace = true, features = ["insecure", "runtime-async-std"] }
 

diff --git a/crates/examples/push-cdn/all.rs b/crates/examples/push-cdn/all.rs
@@ -5,11 +5,10 @@ pub mod types;
 use crate::infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs};
 use crate::types::{DANetwork, NodeImpl, QuorumNetwork, ThisRun};
 use async_compatibility_layer::art::async_spawn;
-use cdn_broker::reexports::connection::protocols::{Quic, Tcp};
 use cdn_broker::reexports::crypto::signature::KeyPair;
 use cdn_broker::Broker;
 use cdn_marshal::Marshal;
-use hotshot::traits::implementations::WrappedSignatureKey;
+use hotshot::traits::implementations::{TestingDef, WrappedSignatureKey};
 use hotshot::types::SignatureKey;
 use hotshot_example_types::state_types::TestTypes;
 use hotshot_orchestrator::client::ValidatorArgs;
@@ -79,12 +78,8 @@ async fn main() {
 
         // Create and spawn the broker
         async_spawn(async move {
-            let broker: Broker<
-                WrappedSignatureKey<<TestTypes as NodeType>::SignatureKey>,
-                WrappedSignatureKey<<TestTypes as NodeType>::SignatureKey>,
-                Tcp,
-                Quic,
-            > = Broker::new(config).await.expect("broker failed to start");
+            let broker: Broker<TestingDef<TestTypes>> =
+                Broker::new(config).await.expect("broker failed to start");
 
             // Error if we stopped unexpectedly
             if let Err(err) = broker.start().await {
@@ -106,10 +101,9 @@ async fn main() {
 
     // Spawn the marshal
     async_spawn(async move {
-        let marshal: Marshal<WrappedSignatureKey<<TestTypes as NodeType>::SignatureKey>, Quic> =
-            Marshal::new(marshal_config)
-                .await
-                .expect("failed to spawn marshal");
+        let marshal: Marshal<TestingDef<TestTypes>> = Marshal::new(marshal_config)
+            .await
+            .expect("failed to spawn marshal");
 
         // Error if we stopped unexpectedly
         if let Err(err) = marshal.start().await {

diff --git a/crates/examples/push-cdn/broker.rs b/crates/examples/push-cdn/broker.rs
@@ -2,10 +2,9 @@
 //! a `Broker` object.
 
 use anyhow::{Context, Result};
-use cdn_broker::reexports::connection::protocols::{Quic, Tcp};
 use cdn_broker::{reexports::crypto::signature::KeyPair, Broker, Config, ConfigBuilder};
 use clap::Parser;
-use hotshot::traits::implementations::WrappedSignatureKey;
+use hotshot::traits::implementations::{ProductionDef, WrappedSignatureKey};
 use hotshot::types::SignatureKey;
 use hotshot_example_types::node_types::TestTypes;
 use hotshot_types::traits::node_implementation::NodeType;
@@ -80,13 +79,7 @@ async fn main() -> Result<()> {
 
     // Create new `Broker`
     // Uses TCP from broker connections and Quic for user connections.
-    let broker = Broker::<
-        WrappedSignatureKey<<TestTypes as NodeType>::SignatureKey>,
-        WrappedSignatureKey<<TestTypes as NodeType>::SignatureKey>,
-        Tcp,
-        Quic,
-    >::new(broker_config)
-    .await?;
+    let broker = Broker::<ProductionDef<TestTypes>>::new(broker_config).await?;
 
     // Start the main loop, consuming it
     broker.start().await?;

diff --git a/crates/examples/push-cdn/marshal.rs b/crates/examples/push-cdn/marshal.rs
@@ -2,12 +2,10 @@
 //! a `Marshal` object with the `HotShot` types.
 //!
 use anyhow::{Context, Result};
-use cdn_broker::reexports::connection::protocols::Quic;
 use cdn_marshal::{ConfigBuilder, Marshal};
 use clap::Parser;
-use hotshot::traits::implementations::WrappedSignatureKey;
+use hotshot::traits::implementations::ProductionDef;
 use hotshot_example_types::node_types::TestTypes;
-use hotshot_types::traits::node_implementation::NodeType;
 
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -41,9 +39,7 @@ async fn main() -> Result<()> {
         .with_context(|| "failed to build Marshal config")?;
 
     // Create new `Marshal` from the config
-    let marshal =
-        Marshal::<WrappedSignatureKey<<TestTypes as NodeType>::SignatureKey>, Quic>::new(config)
-            .await?;
+    let marshal = Marshal::<ProductionDef<TestTypes>>::new(config).await?;
 
     // Start the main loop, consuming it
     marshal.start().await?;

diff --git a/crates/examples/push-cdn/types.rs b/crates/examples/push-cdn/types.rs
@@ -1,7 +1,6 @@
 use crate::infra::PushCdnDaRun;
-use hotshot::traits::implementations::PushCdnNetwork;
+use hotshot::traits::{implementations::PushCdnNetwork, NodeImplementation};
 use hotshot_example_types::{state_types::TestTypes, storage_types::TestStorage};
-use hotshot_types::traits::node_implementation::NodeImplementation;
 use serde::{Deserialize, Serialize};
 
 #[derive(Clone, Deserialize, Serialize, Hash, PartialEq, Eq)]