diff --git a/apps/docs/pages/integrations/google-cloud/compute-scanner.mdx b/apps/docs/pages/integrations/google-cloud/compute-scanner.mdx index 1a6bd93c6..f991f37de 100644 --- a/apps/docs/pages/integrations/google-cloud/compute-scanner.mdx +++ b/apps/docs/pages/integrations/google-cloud/compute-scanner.mdx @@ -6,6 +6,9 @@ and creates them for related deployments automatically. Currently the compute scanner supports importing the following resources: - Google Kubernetes Engine Clusters (GKE) + - Namespaces + - VClusters (Virtual Clusters) +- Google Virtual Private Cloud (VPC) ## Managed Compute Scanner diff --git a/apps/event-worker/src/resource-scan/google/index.ts b/apps/event-worker/src/resource-scan/google/index.ts index 5222bd135..1d9c29f0d 100644 --- a/apps/event-worker/src/resource-scan/google/index.ts +++ b/apps/event-worker/src/resource-scan/google/index.ts @@ -2,6 +2,7 @@ import { logger } from "@ctrlplane/logger"; import { createResourceScanWorker } from "../utils.js"; import { getGkeResources } from "./gke.js"; +import { getVpcResources } from "./vpc.js"; const log = logger.child({ label: "resource-scan/google" }); @@ -14,10 +15,15 @@ export const createGoogleResourceScanWorker = () => return []; } - const resources = await getGkeResources( + const gkeResources = await getGkeResources( rp.workspace, rp.resource_provider_google, ); - return resources; + const vpcResources = await getVpcResources( + rp.workspace, + rp.resource_provider_google, + ); + + return [...gkeResources, ...vpcResources]; }); diff --git a/apps/event-worker/src/resource-scan/google/vpc.ts b/apps/event-worker/src/resource-scan/google/vpc.ts index 4657fe188..a29a2b941 100644 --- a/apps/event-worker/src/resource-scan/google/vpc.ts +++ b/apps/event-worker/src/resource-scan/google/vpc.ts @@ -5,8 +5,9 @@ import type { } from "@ctrlplane/db/schema"; import type { CloudVPCV1 } from "@ctrlplane/validators/resources"; import type { google } from "@google-cloud/compute/build/protos/protos.js"; -import { NetworksClient } from "@google-cloud/compute"; +import { NetworksClient, SubnetworksClient } from "@google-cloud/compute"; import _ from "lodash"; +import { isPresent } from "ts-is-present"; import { logger } from "@ctrlplane/logger"; import { ReservedMetadataKey } from "@ctrlplane/validators/conditions"; @@ -16,61 +17,119 @@ import { getGoogleClient } from "./client.js"; const log = logger.child({ label: "resource-scan/google/vpc" }); -const getNetworksClient = async (targetPrincipal?: string | null) => - getGoogleClient(NetworksClient, targetPrincipal, "Networks Client"); +type GoogleSubnetDetails = { + name: string; + region: string; + cidr: string; + type: "internal" | "external"; + gatewayAddress: string; + secondaryCidrs: { name: string; cidr: string }[] | undefined; +}; + +const getNetworksClient = async (targetPrincipal?: string | null) => { + const [networksClient] = await getGoogleClient( + NetworksClient, + targetPrincipal, + "Networks Client", + ); + const [subnetsClient] = await getGoogleClient( + SubnetworksClient, + targetPrincipal, + "Subnets Client", + ); + return { networksClient, subnetsClient }; +}; + +const getSubnetDetails = ( + subnetsClient: SubnetworksClient, + project: string, + subnetSelfLink: string, +): Promise => { + const parts = subnetSelfLink.split("/"); + const region = parts.at(-3) ?? ""; + const name = parts.at(-1) ?? ""; + + return subnetsClient + .list({ + project, + region, + filter: `name eq ${name}`, + }) + .then(([subnets]) => + _.chain(subnets) + .find((subnet) => subnet.name === name) + .thru( + (subnet): GoogleSubnetDetails => ({ + name, + region, + gatewayAddress: subnet.gatewayAddress ?? "", + cidr: subnet.ipCidrRange ?? "", + type: subnet.purpose === "INTERNAL" ? "internal" : "external", + secondaryCidrs: subnet.secondaryIpRanges?.map((r) => ({ + name: r.rangeName ?? "", + cidr: r.ipCidrRange ?? "", + })), + }), + ) + .value(), + ); +}; -const getNetworkResources = ( +const getNetworkResources = async ( + clients: { networksClient: NetworksClient; subnetsClient: SubnetworksClient }, project: string, networks: google.cloud.compute.v1.INetwork[], -): CloudVPCV1[] => - networks - .filter((n) => n.name != null) - .map((network) => { - return { - name: network.name!, - identifier: `${project}/${network.name}`, - version: "cloud/v1", - kind: "VPC", - config: { +): Promise => + await Promise.all( + networks + .filter((n) => n.name != null) + .map(async (network) => { + const subnets = await Promise.all( + (network.subnetworks ?? []).map((subnet) => + getSubnetDetails(clients.subnetsClient, project, subnet), + ), + ); + return { name: network.name!, - provider: "google", - region: "global", // GCP VPC is global; subnets have regional scope - project, - cidr: network.IPv4Range ?? undefined, - mtu: network.mtu ?? undefined, - subnets: network.subnetworks?.map((subnet) => { - const parts = subnet.split("/"); - const region = parts.at(-3) ?? ""; - const name = parts.at(-1) ?? ""; - return { name, region }; - }), - }, - metadata: omitNullUndefined({ - [ReservedMetadataKey.ExternalId]: network.id?.toString(), - [ReservedMetadataKey.Links]: JSON.stringify({ - "Google Console": `https://console.cloud.google.com/networking/networks/details/${network.name}?project=${project}`, - }), - "google/project": project, - "google/self-link": network.selfLink, - "google/creation-timestamp": network.creationTimestamp, - "google/description": network.description, - ...network.peerings?.reduce( - (acc, peering) => ({ - ...acc, - [`google/peering/${peering.name}`]: JSON.stringify({ - network: peering.network, - state: peering.state, - autoCreateRoutes: peering.autoCreateRoutes, - }), + identifier: `${project}/${network.name}`, + version: "cloud/v1" as const, + kind: "VPC" as const, + config: { + name: network.name!, + provider: "google" as const, + region: "global", + project, + cidr: network.IPv4Range ?? undefined, + mtu: network.mtu ?? undefined, + subnets: subnets.filter(isPresent), + }, + metadata: omitNullUndefined({ + [ReservedMetadataKey.ExternalId]: network.id?.toString(), + [ReservedMetadataKey.Links]: JSON.stringify({ + "Google Console": `https://console.cloud.google.com/networking/networks/details/${network.name}?project=${project}`, }), - {}, - ), - }), - }; - }); + "google/project": project, + "google/self-link": network.selfLink, + "google/creation-timestamp": network.creationTimestamp, + "google/description": network.description, + ...network.peerings?.reduce( + (acc, peering) => ({ + ...acc, + [`google/peering/${peering.name}`]: JSON.stringify({ + network: peering.network, + state: peering.state, + autoCreateRoutes: peering.autoCreateRoutes, + }), + }), + {}, + ), + }), + }; + }), + ); const fetchProjectNetworks = async ( - networksClient: NetworksClient, + clients: { networksClient: NetworksClient; subnetsClient: SubnetworksClient }, project: string, workspaceId: string, providerId: string, @@ -80,14 +139,19 @@ const fetchProjectNetworks = async ( let pageToken: string | undefined | null; do { - const [networkList, request] = await networksClient.list({ + const [networkList, request] = await clients.networksClient.list({ project, maxResults: 500, pageToken, }); + const resources = await getNetworkResources( + clients, + project, + networkList, + ); networks.push( - ...getNetworkResources(project, networkList).map((resource) => ({ + ...resources.map((resource) => ({ ...resource, workspaceId, providerId, @@ -97,15 +161,15 @@ const fetchProjectNetworks = async ( } while (pageToken != null); return networks; - } catch (error: any) { + } catch (err) { + const error = err as { message?: string; code?: number }; const isPermissionError = - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - error.message?.includes("PERMISSION_DENIED") || error.code === 403; + error.message?.includes("PERMISSION_DENIED") ?? error.code === 403; log.error( `Unable to get VPCs for project: ${project} - ${ isPermissionError ? 'Missing required permissions. Please ensure the service account has the "Compute Network Viewer" role.' - : error.message + : (error.message ?? "Unknown error") }`, { error, project }, ); @@ -124,14 +188,16 @@ export const getVpcResources = async ( { workspaceId, config, googleServiceAccountEmail, resourceProviderId }, ); - const [networksClient] = await getNetworksClient(googleServiceAccountEmail); - const resources: InsertResource[] = await _.chain(config.projectIds) - .map((id) => - fetchProjectNetworks(networksClient, id, workspaceId, resourceProviderId), - ) - .thru((promises) => Promise.all(promises)) - .value() - .then((results) => results.flat()); + const clients = await getNetworksClient(googleServiceAccountEmail); + const resources: InsertResource[] = config.importVpc + ? await _.chain(config.projectIds) + .map((id) => + fetchProjectNetworks(clients, id, workspaceId, resourceProviderId), + ) + .thru((promises) => Promise.all(promises)) + .value() + .then((results) => results.flat()) + : []; log.info(`Found ${resources.length} VPC resources`); diff --git a/apps/webservice/src/app/[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/GoogleDialog.tsx b/apps/webservice/src/app/[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/GoogleDialog.tsx index a06295785..5d3cd8255 100644 --- a/apps/webservice/src/app/[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/GoogleDialog.tsx +++ b/apps/webservice/src/app/[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/GoogleDialog.tsx @@ -43,6 +43,7 @@ export const createGoogleSchema = z.object({ importGke: z.boolean().default(false), importNamespaces: z.boolean().default(false), importVCluster: z.boolean().default(false), + importVpc: z.boolean().default(false), }); export const GoogleDialog: React.FC<{ @@ -56,6 +57,7 @@ export const GoogleDialog: React.FC<{ importGke: true, importNamespaces: false, importVCluster: false, + importVpc: false, }, mode: "onChange", }); @@ -82,10 +84,8 @@ export const GoogleDialog: React.FC<{ ...data, workspaceId: workspace.id, config: { + ...data, projectIds: data.projectIds.map((p) => p.value), - importGke: data.importGke, - importVCluster: data.importVCluster, - importNamespaces: data.importNamespaces, }, }); await utils.resource.provider.byWorkspaceId.invalidate(); @@ -256,6 +256,25 @@ export const GoogleDialog: React.FC<{ )} /> + ( + +
+ Import VPC + Enable importing of VPCs +
+ + + +
+ )} + /> + diff --git a/apps/webservice/src/app/[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/UpdateGoogleProviderDialog.tsx b/apps/webservice/src/app/[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/UpdateGoogleProviderDialog.tsx index 3c76b591e..d8dd4aa3c 100644 --- a/apps/webservice/src/app/[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/UpdateGoogleProviderDialog.tsx +++ b/apps/webservice/src/app/[workspaceSlug]/(app)/(resources)/resource-providers/integrations/google/UpdateGoogleProviderDialog.tsx @@ -72,10 +72,8 @@ export const UpdateGoogleProviderDialog: React.FC<{ ...data, resourceProviderId: providerId, config: { + ...data, projectIds: data.projectIds.map((p) => p.value), - importGke: data.importGke, - importVCluster: data.importVCluster, - importNamespaces: data.importNamespaces, }, repeatSeconds: data.repeatSeconds === 0 ? null : data.repeatSeconds, }); @@ -288,6 +286,25 @@ export const UpdateGoogleProviderDialog: React.FC<{ )} /> + ( + +
+ Import VPC + Enable importing of VPCs +
+ + + +
+ )} + /> +