feat: Google VPC Scanner last mile

ctrlplanedev · Dec 31, 2024 · acdfafe · acdfafe
1 parent 141f048
commit acdfafe
Show file tree

Hide file tree

Showing 10 changed files with 4,634 additions and 72 deletions.
diff --git a/apps/docs/pages/integrations/google-cloud/compute-scanner.mdx b/apps/docs/pages/integrations/google-cloud/compute-scanner.mdx
@@ -6,6 +6,9 @@ and creates them for related deployments automatically.
 Currently the compute scanner supports importing the following resources:
 
 - Google Kubernetes Engine Clusters (GKE)
+  - Namespaces
+  - VClusters (Virtual Clusters)
+- Google Virtual Private Cloud (VPC)
 
 ## Managed Compute Scanner
 

diff --git a/apps/event-worker/src/resource-scan/google/index.ts b/apps/event-worker/src/resource-scan/google/index.ts
@@ -2,6 +2,7 @@ import { logger } from "@ctrlplane/logger";
 
 import { createResourceScanWorker } from "../utils.js";
 import { getGkeResources } from "./gke.js";
+import { getVpcResources } from "./vpc.js";
 
 const log = logger.child({ label: "resource-scan/google" });
 
@@ -14,10 +15,15 @@ export const createGoogleResourceScanWorker = () =>
       return [];
     }
 
-    const resources = await getGkeResources(
+    const gkeResources = await getGkeResources(
       rp.workspace,
       rp.resource_provider_google,
     );
 
-    return resources;
+    const vpcResources = await getVpcResources(
+      rp.workspace,
+      rp.resource_provider_google,
+    );
+
+    return [...gkeResources, ...vpcResources];
   });
diff --git a/apps/event-worker/src/resource-scan/google/vpc.ts b/apps/event-worker/src/resource-scan/google/vpc.ts
@@ -5,8 +5,9 @@ import type {
 } from "@ctrlplane/db/schema";
 import type { CloudVPCV1 } from "@ctrlplane/validators/resources";
 import type { google } from "@google-cloud/compute/build/protos/protos.js";
-import { NetworksClient } from "@google-cloud/compute";
+import { NetworksClient, SubnetworksClient } from "@google-cloud/compute";
 import _ from "lodash";
+import { isPresent } from "ts-is-present";
 
 import { logger } from "@ctrlplane/logger";
 import { ReservedMetadataKey } from "@ctrlplane/validators/conditions";
@@ -16,61 +17,119 @@ import { getGoogleClient } from "./client.js";
 
 const log = logger.child({ label: "resource-scan/google/vpc" });
 
-const getNetworksClient = async (targetPrincipal?: string | null) =>
-  getGoogleClient(NetworksClient, targetPrincipal, "Networks Client");
+type GoogleSubnetDetails = {
+  name: string;
+  region: string;
+  cidr: string;
+  type: "internal" | "external";
+  gatewayAddress: string;
+  secondaryCidrs: { name: string; cidr: string }[] | undefined;
+};
+
+const getNetworksClient = async (targetPrincipal?: string | null) => {
+  const [networksClient] = await getGoogleClient(
+    NetworksClient,
+    targetPrincipal,
+    "Networks Client",
+  );
+  const [subnetsClient] = await getGoogleClient(
+    SubnetworksClient,
+    targetPrincipal,
+    "Subnets Client",
+  );
+  return { networksClient, subnetsClient };
+};
+
+const getSubnetDetails = (
+  subnetsClient: SubnetworksClient,
+  project: string,
+  subnetSelfLink: string,
+): Promise<GoogleSubnetDetails | null> => {
+  const parts = subnetSelfLink.split("/");
+  const region = parts.at(-3) ?? "";
+  const name = parts.at(-1) ?? "";
+
+  return subnetsClient
+    .list({
+      project,
+      region,
+      filter: `name eq ${name}`,
+    })
+    .then(([subnets]) =>
+      _.chain(subnets)
+        .find((subnet) => subnet.name === name)
+        .thru(
+          (subnet): GoogleSubnetDetails => ({
+            name,
+            region,
+            gatewayAddress: subnet.gatewayAddress ?? "",
+            cidr: subnet.ipCidrRange ?? "",
+            type: subnet.purpose === "INTERNAL" ? "internal" : "external",
+            secondaryCidrs: subnet.secondaryIpRanges?.map((r) => ({
+              name: r.rangeName ?? "",
+              cidr: r.ipCidrRange ?? "",
+            })),
+          }),
+        )
+        .value(),
+    );
+};
 
-const getNetworkResources = (
+const getNetworkResources = async (
+  clients: { networksClient: NetworksClient; subnetsClient: SubnetworksClient },
   project: string,
   networks: google.cloud.compute.v1.INetwork[],
-): CloudVPCV1[] =>
-  networks
-    .filter((n) => n.name != null)
-    .map((network) => {
-      return {
-        name: network.name!,
-        identifier: `${project}/${network.name}`,
-        version: "cloud/v1",
-        kind: "VPC",
-        config: {
+): Promise<CloudVPCV1[]> =>
+  await Promise.all(
+    networks
+      .filter((n) => n.name != null)
+      .map(async (network) => {
+        const subnets = await Promise.all(
+          (network.subnetworks ?? []).map((subnet) =>
+            getSubnetDetails(clients.subnetsClient, project, subnet),
+          ),
+        );
+        return {
           name: network.name!,
-          provider: "google",
-          region: "global", // GCP VPC is global; subnets have regional scope
-          project,
-          cidr: network.IPv4Range ?? undefined,
-          mtu: network.mtu ?? undefined,
-          subnets: network.subnetworks?.map((subnet) => {
-            const parts = subnet.split("/");
-            const region = parts.at(-3) ?? "";
-            const name = parts.at(-1) ?? "";
-            return { name, region };
-          }),
-        },
-        metadata: omitNullUndefined({
-          [ReservedMetadataKey.ExternalId]: network.id?.toString(),
-          [ReservedMetadataKey.Links]: JSON.stringify({
-            "Google Console": `https://console.cloud.google.com/networking/networks/details/${network.name}?project=${project}`,
-          }),
-          "google/project": project,
-          "google/self-link": network.selfLink,
-          "google/creation-timestamp": network.creationTimestamp,
-          "google/description": network.description,
-          ...network.peerings?.reduce(
-            (acc, peering) => ({
-              ...acc,
-              [`google/peering/${peering.name}`]: JSON.stringify({
-                network: peering.network,
-                state: peering.state,
-                autoCreateRoutes: peering.autoCreateRoutes,
-              }),
+          identifier: `${project}/${network.name}`,
+          version: "cloud/v1" as const,
+          kind: "VPC" as const,
+          config: {
+            name: network.name!,
+            provider: "google" as const,
+            region: "global",
+            project,
+            cidr: network.IPv4Range ?? undefined,
+            mtu: network.mtu ?? undefined,
+            subnets: subnets.filter(isPresent),
+          },
+          metadata: omitNullUndefined({
+            [ReservedMetadataKey.ExternalId]: network.id?.toString(),
+            [ReservedMetadataKey.Links]: JSON.stringify({
+              "Google Console": `https://console.cloud.google.com/networking/networks/details/${network.name}?project=${project}`,
             }),
-            {},
-          ),
-        }),
-      };
-    });
+            "google/project": project,
+            "google/self-link": network.selfLink,
+            "google/creation-timestamp": network.creationTimestamp,
+            "google/description": network.description,
+            ...network.peerings?.reduce(
+              (acc, peering) => ({
+                ...acc,
+                [`google/peering/${peering.name}`]: JSON.stringify({
+                  network: peering.network,
+                  state: peering.state,
+                  autoCreateRoutes: peering.autoCreateRoutes,
+                }),
+              }),
+              {},
+            ),
+          }),
+        };
+      }),
+  );
 
 const fetchProjectNetworks = async (
-  networksClient: NetworksClient,
+  clients: { networksClient: NetworksClient; subnetsClient: SubnetworksClient },
   project: string,
   workspaceId: string,
   providerId: string,
@@ -80,14 +139,19 @@ const fetchProjectNetworks = async (
     let pageToken: string | undefined | null;
 
     do {
-      const [networkList, request] = await networksClient.list({
+      const [networkList, request] = await clients.networksClient.list({
         project,
         maxResults: 500,
         pageToken,
       });
 
+      const resources = await getNetworkResources(
+        clients,
+        project,
+        networkList,
+      );
       networks.push(
-        ...getNetworkResources(project, networkList).map((resource) => ({
+        ...resources.map((resource) => ({
           ...resource,
           workspaceId,
           providerId,
@@ -97,15 +161,15 @@ const fetchProjectNetworks = async (
     } while (pageToken != null);
 
     return networks;
-  } catch (error: any) {
+  } catch (err) {
+    const error = err as { message?: string; code?: number };
     const isPermissionError =
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-call
-      error.message?.includes("PERMISSION_DENIED") || error.code === 403;
+      error.message?.includes("PERMISSION_DENIED") ?? error.code === 403;
     log.error(
       `Unable to get VPCs for project: ${project} - ${
         isPermissionError
           ? 'Missing required permissions. Please ensure the service account has the "Compute Network Viewer" role.'
-          : error.message
+          : (error.message ?? "Unknown error")
       }`,
       { error, project },
     );
@@ -124,14 +188,16 @@ export const getVpcResources = async (
     { workspaceId, config, googleServiceAccountEmail, resourceProviderId },
   );
 
-  const [networksClient] = await getNetworksClient(googleServiceAccountEmail);
-  const resources: InsertResource[] = await _.chain(config.projectIds)
-    .map((id) =>
-      fetchProjectNetworks(networksClient, id, workspaceId, resourceProviderId),
-    )
-    .thru((promises) => Promise.all(promises))
-    .value()
-    .then((results) => results.flat());
+  const clients = await getNetworksClient(googleServiceAccountEmail);
+  const resources: InsertResource[] = config.importVpc
+    ? await _.chain(config.projectIds)
+        .map((id) =>
+          fetchProjectNetworks(clients, id, workspaceId, resourceProviderId),
+        )
+        .thru((promises) => Promise.all(promises))
+        .value()
+        .then((results) => results.flat())
+    : [];
 
   log.info(`Found ${resources.length} VPC resources`);
 

diff --git a/...[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/GoogleDialog.tsx b/...[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/GoogleDialog.tsx
@@ -43,6 +43,7 @@ export const createGoogleSchema = z.object({
   importGke: z.boolean().default(false),
   importNamespaces: z.boolean().default(false),
   importVCluster: z.boolean().default(false),
+  importVpc: z.boolean().default(false),
 });
 
 export const GoogleDialog: React.FC<{
@@ -56,6 +57,7 @@ export const GoogleDialog: React.FC<{
       importGke: true,
       importNamespaces: false,
       importVCluster: false,
+      importVpc: false,
     },
     mode: "onChange",
   });
@@ -82,10 +84,8 @@ export const GoogleDialog: React.FC<{
       ...data,
       workspaceId: workspace.id,
       config: {
+        ...data,
         projectIds: data.projectIds.map((p) => p.value),
-        importGke: data.importGke,
-        importVCluster: data.importVCluster,
-        importNamespaces: data.importNamespaces,
       },
     });
     await utils.resource.provider.byWorkspaceId.invalidate();
@@ -256,6 +256,25 @@ export const GoogleDialog: React.FC<{
               )}
             />
 
+            <FormField
+              control={form.control}
+              name="importVpc"
+              render={({ field }) => (
+                <FormItem className="flex flex-row items-center justify-between rounded-lg border p-4">
+                  <div className="space-y-0.5">
+                    <FormLabel>Import VPC</FormLabel>
+                    <FormDescription>Enable importing of VPCs</FormDescription>
+                  </div>
+                  <FormControl>
+                    <Switch
+                      checked={field.value}
+                      onCheckedChange={field.onChange}
+                    />
+                  </FormControl>
+                </FormItem>
+              )}
+            />
+
             <DialogFooter>
               <Button type="submit">Create</Button>
             </DialogFooter>

diff --git a/...]/(app)/(resources)/resource-providers/integrations/google/UpdateGoogleProviderDialog.tsx b/...]/(app)/(resources)/resource-providers/integrations/google/UpdateGoogleProviderDialog.tsx
@@ -72,10 +72,8 @@ export const UpdateGoogleProviderDialog: React.FC<{
       ...data,
       resourceProviderId: providerId,
       config: {
+        ...data,
         projectIds: data.projectIds.map((p) => p.value),
-        importGke: data.importGke,
-        importVCluster: data.importVCluster,
-        importNamespaces: data.importNamespaces,
       },
       repeatSeconds: data.repeatSeconds === 0 ? null : data.repeatSeconds,
     });
@@ -288,6 +286,25 @@ export const UpdateGoogleProviderDialog: React.FC<{
               )}
             />
 
+            <FormField
+              control={form.control}
+              name="importVpc"
+              render={({ field }) => (
+                <FormItem className="flex flex-row items-center justify-between rounded-lg border p-4">
+                  <div className="space-y-0.5">
+                    <FormLabel>Import VPC</FormLabel>
+                    <FormDescription>Enable importing of VPCs</FormDescription>
+                  </div>
+                  <FormControl>
+                    <Switch
+                      checked={field.value}
+                      onCheckedChange={field.onChange}
+                    />
+                  </FormControl>
+                </FormItem>
+              )}
+            />
+
             <FormField
               control={form.control}
               name="repeatSeconds"

diff --git a/packages/db/drizzle/0051_fat_iron_lad.sql b/packages/db/drizzle/0051_fat_iron_lad.sql
@@ -0,0 +1 @@
+ALTER TABLE "resource_provider_google" ADD COLUMN "import_vpc" boolean DEFAULT false NOT NULL;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ALTER TABLE "resource_provider_google" ADD COLUMN "import_vpc" boolean DEFAULT false NOT NULL;