Try to fix an issue where cloud-provisioned ExitNodes don't get their…

… secret refs set
FyraLabs · Oct 19, 2024 · b818521 · b818521
1 parent ee53caa
commit b818521
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 17 deletions.
diff --git a/src/cloud/aws.rs b/src/cloud/aws.rs
@@ -112,7 +112,7 @@ impl Provisioner for AWSProvisioner {
         &self,
         auth: Secret,
         exit_node: ExitNode,
-    ) -> color_eyre::Result<ExitNodeStatus> {
+    ) -> color_eyre::Result<(ExitNodeStatus, Secret)> {
         let provisioner = exit_node
             .metadata
             .annotations
@@ -127,6 +127,8 @@ impl Provisioner for AWSProvisioner {
 
         let password = generate_password(32);
 
+        let secret = exit_node.generate_secret(password.clone()).await?;
+
         let cloud_init_config = generate_cloud_init_config(&password, CHISEL_PORT);
         let user_data = base64::engine::general_purpose::STANDARD.encode(cloud_init_config);
 
@@ -222,7 +224,7 @@ impl Provisioner for AWSProvisioner {
             instance.instance_id.map(|id| id.to_string()).as_deref(),
         );
 
-        Ok(exit_node)
+        Ok((exit_node, secret))
     }
 
     async fn update_exit_node(
@@ -268,7 +270,7 @@ impl Provisioner for AWSProvisioner {
         } else {
             warn!("No status found for exit node, creating new instance");
             // TODO: this should be handled by the controller logic
-            return self.create_exit_node(auth, exit_node).await;
+            return self.create_exit_node(auth, exit_node).await.map(|(status, _)| status);
         }
     }
 

diff --git a/src/cloud/digitalocean.rs b/src/cloud/digitalocean.rs
@@ -61,12 +61,12 @@ impl Provisioner for DigitalOceanProvisioner {
         &self,
         auth: Secret,
         exit_node: ExitNode,
-    ) -> color_eyre::Result<ExitNodeStatus> {
+    ) -> color_eyre::Result<(ExitNodeStatus, Secret)> {
         let password = generate_password(32);
 
         // create secret for password too
 
-        let _secret = exit_node.generate_secret(password.clone()).await?;
+        let secret = exit_node.generate_secret(password.clone()).await?;
 
         let config = generate_cloud_init_config(&password, exit_node.spec.port);
 
@@ -138,7 +138,7 @@ impl Provisioner for DigitalOceanProvisioner {
             Some(&droplet_id),
         );
 
-        Ok(exit_node)
+        Ok((exit_node, secret))
     }
 
     async fn update_exit_node(
@@ -170,7 +170,8 @@ impl Provisioner for DigitalOceanProvisioner {
         } else {
             warn!("No status found for exit node, creating new droplet");
             // TODO: this should be handled by the controller logic
-            return self.create_exit_node(auth, exit_node).await;
+            let (status, _) = self.create_exit_node(auth, exit_node).await?;
+            return Ok(status);
         }
     }
 

diff --git a/src/cloud/linode.rs b/src/cloud/linode.rs
@@ -54,10 +54,11 @@ impl Provisioner for LinodeProvisioner {
         &self,
         auth: Secret,
         exit_node: ExitNode,
-    ) -> color_eyre::Result<ExitNodeStatus> {
+    ) -> color_eyre::Result<(ExitNodeStatus, Secret)> {
         let password = generate_password(32);
 
-        let _secret = exit_node.generate_secret(password.clone()).await?;
+        // Password for the server
+        let secret = exit_node.generate_secret(password.clone()).await?;
 
         let config = generate_cloud_init_config(&password, exit_node.spec.port);
 
@@ -126,7 +127,7 @@ impl Provisioner for LinodeProvisioner {
             Some(&instance.id.to_string()),
         );
 
-        Ok(status)
+        Ok((status, secret))
     }
 
     async fn delete_exit_node(&self, auth: Secret, exit_node: ExitNode) -> color_eyre::Result<()> {
@@ -178,7 +179,7 @@ impl Provisioner for LinodeProvisioner {
             Ok(status)
         } else {
             warn!("No instance status found, creating new instance");
-            return self.create_exit_node(auth.clone(), exit_node).await;
+            return self.create_exit_node(auth.clone(), exit_node).await.map(|(status, _)| status);
         }
     }
 }
diff --git a/src/cloud/mod.rs b/src/cloud/mod.rs
@@ -27,7 +27,8 @@ pub trait Provisioner {
         &self,
         auth: Secret,
         exit_node: ExitNode,
-    ) -> color_eyre::Result<ExitNodeStatus>;
+        // Should return the pointer to the password secret for the exit node
+    ) -> color_eyre::Result<(ExitNodeStatus, Secret)>;
     async fn update_exit_node(
         &self,
         auth: Secret,

diff --git a/src/daemon.rs b/src/daemon.rs
@@ -58,6 +58,8 @@ use crate::{deployment::create_owned_deployment, error::ReconcileError};
 pub const EXIT_NODE_FINALIZER: &str = "exitnode.chisel-operator.io/finalizer";
 pub const SVCS_FINALIZER: &str = "service.chisel-operator.io/finalizer";
 
+// todo: Refactor everything in here into separate functions, then we can write unit tests for them
+
 // pub fn get_trace_id() -> opentelemetry::trace::TraceId {
 //     // opentelemetry::Context -> opentelemetry::trace::Span
 //     use opentelemetry::trace::TraceContextExt as _;
@@ -582,6 +584,7 @@ async fn reconcile_nodes(obj: Arc<ExitNode>, ctx: Arc<Context>) -> Result<Action
             }
         }
 
+        // Find provisioner
         let provisioner = find_exit_node_provisioner_from_label(
             ctx.clone(),
             &obj.namespace().unwrap(),
@@ -590,6 +593,7 @@ async fn reconcile_nodes(obj: Arc<ExitNode>, ctx: Arc<Context>) -> Result<Action
         .await
         .ok_or(ReconcileError::CloudProvisionerNotFound)?;
 
+        // Get provisioner API handle
         let provisioner_api = provisioner.clone().spec.get_inner();
 
         let secret = provisioner
@@ -598,6 +602,8 @@ async fn reconcile_nodes(obj: Arc<ExitNode>, ctx: Arc<Context>) -> Result<Action
             .map_err(|_| crate::error::ReconcileError::CloudProvisionerSecretNotFound)?
             .ok_or(ReconcileError::CloudProvisionerSecretNotFound)?;
 
+        debug!(?secret, "Secret");
+
         finalizer::finalizer(
             &exit_nodes.clone(),
             EXIT_NODE_FINALIZER,
@@ -607,21 +613,41 @@ async fn reconcile_nodes(obj: Arc<ExitNode>, ctx: Arc<Context>) -> Result<Action
                 {
                     Event::Apply(node) => {
                         let _node = {
+
+                            let mut pass_secret: Option<k8s_openapi::api::core::v1::Secret> = None;
+                            // if status exists, update, else create
                             let cloud_resource = if let Some(_status) = node.status.as_ref() {
                                 info!("Updating cloud resource for {}", node.name_any());
                                 provisioner_api
                                     .update_exit_node(secret.clone(), (*node).clone())
                                     .await
                             } else {
+                                // todo: probably update the Provisioner trait to accept a provisioner API handle or
+                                // the provisioner API token *and* then a password secret
+                                // Right now we have the create_exit_node method which returns the password secret alongside the status
+
+                                // create cloud resource
                                 info!("Creating cloud resource for {}", node.name_any());
-                                provisioner_api
+
+                                let (resource, new_pass_secret) = provisioner_api
                                     .create_exit_node(secret.clone(), (*node).clone())
-                                    .await
+                                    .await?;
+                                pass_secret = Some(new_pass_secret);
+                                Ok(resource)
                             };
                             // TODO: Don't replace the entire status and object, sadly JSON is better here
-                            let exitnode_patch = serde_json::json!({
-                                "status": cloud_resource?
-                            });
+                            let exitnode_patch = if let Some(p_secret) = pass_secret {
+                                serde_json::json!({
+                                    "status": cloud_resource?,
+                                    "spec": {
+                                        "auth": p_secret.name_any(),
+                                    }
+                                })
+                            } else {
+                                serde_json::json!({
+                                    "status": cloud_resource?,
+                                })
+                            };
 
                             exit_nodes
                                 .patch_status(

diff --git a/src/ops.rs b/src/ops.rs
@@ -88,6 +88,7 @@ impl ExitNode {
     ///
     /// Generates a new secret with the `auth` key containing the auth string for chisel in the same namespace as the ExitNode
     pub async fn generate_secret(&self, password: String) -> Result<Secret> {
+        debug!("Generating secret for ExitNode");
         let secret_name = self.get_secret_name();
 
         let auth_tmpl = format!("{}:{}", crate::cloud::pwgen::DEFAULT_USERNAME, password);