diff --git a/apps/webservice/src/app/api/v1/config/route.ts b/apps/webservice/src/app/api/v1/config/route.ts new file mode 100644 index 00000000..4c29b84b --- /dev/null +++ b/apps/webservice/src/app/api/v1/config/route.ts @@ -0,0 +1,299 @@ +import type { Tx } from "@ctrlplane/db"; +import type { CacV1 } from "@ctrlplane/validators/cac"; +import type { NextRequest } from "next/server"; +import { NextResponse } from "next/server"; +import jsYaml from "js-yaml"; +import _ from "lodash"; +import { isPresent } from "ts-is-present"; + +import { can } from "@ctrlplane/auth/utils"; +import { + and, + buildConflictUpdateColumns, + eq, + inArray, + takeFirstOrNull, +} from "@ctrlplane/db"; +import { db } from "@ctrlplane/db/client"; +import * as schema from "@ctrlplane/db/schema"; +import { + cancelOldReleaseJobTriggersOnJobDispatch, + createJobApprovals, + createReleaseJobTriggers, + dispatchReleaseJobTriggers, + isPassingAllPolicies, + isPassingReleaseStringCheckPolicy, +} from "@ctrlplane/job-dispatch"; +import { Permission } from "@ctrlplane/validators/auth"; +import { cacV1 } from "@ctrlplane/validators/cac"; + +import { getUser } from "~/app/api/v1/auth"; + +const { entries, fromEntries } = Object; + +const upsertSystems = async (db: Tx, config: CacV1) => { + if (config.systems == null || entries(config.systems).length == 0) return; + + const systemInserts = entries(config.systems ?? {}).map(([slug, system]) => ({ + name: system.name ?? slug, + description: system.description ?? "", + workspaceId: config.workspace, + slug, + })); + + if (systemInserts.length == 0) return; + + await db + .insert(schema.system) + .values(systemInserts) + .onConflictDoUpdate({ + target: [schema.system.workspaceId, schema.system.slug], + set: buildConflictUpdateColumns(schema.system, ["name", "description"]), + }); +}; + +const upsertDeployments = async (db: Tx, config: CacV1) => { + if (config.deployments == null || entries(config.deployments).length == 0) + return; + + const deploymentInfo = entries(config.deployments ?? {}) + .map(([slug, deployment]) => { + const [systemSlug, deploymentSlug] = slug.split("/"); + if (systemSlug == null || deploymentSlug == null) return null; + + return { + systemSlug, + deployment: { + name: deployment.name ?? deploymentSlug, + slug: deploymentSlug, + description: deployment.description ?? "", + jobAgentId: deployment.jobAgent?.id, + jobAgentConfig: deployment.jobAgent?.config ?? {}, + }, + }; + }) + .filter(isPresent); + const systemSlugs = deploymentInfo.map(({ systemSlug }) => systemSlug); + + const systems = await db + .select() + .from(schema.system) + .where(inArray(schema.system.slug, systemSlugs)); + + const jobAgentsIds = await db + .select({ id: schema.jobAgent.id }) + .from(schema.jobAgent) + .where( + and( + inArray( + schema.jobAgent.id, + deploymentInfo + .map(({ deployment }) => deployment.jobAgentId) + .filter(isPresent), + ), + eq(schema.jobAgent.workspaceId, config.workspace), + ), + ) + .then((rows) => rows.map(({ id }) => id)); + + const systemMap = fromEntries(systems.map((system) => [system.slug, system])); + + const deploymentInserts = deploymentInfo + .map(({ systemSlug, deployment }) => { + const system = systemMap[systemSlug]; + if (system == null) return null; + if ( + deployment.jobAgentId != null && + !jobAgentsIds.includes(deployment.jobAgentId) + ) + return null; + return { ...deployment, systemId: system.id }; + }) + .filter(isPresent); + + if (deploymentInserts.length == 0) return; + + await db + .insert(schema.deployment) + .values(deploymentInserts) + .onConflictDoUpdate({ + target: [schema.deployment.systemId, schema.deployment.slug], + set: buildConflictUpdateColumns(schema.deployment, [ + "name", + "description", + "jobAgentId", + "jobAgentConfig", + ]), + }); +}; + +const upsertReleases = async (db: Tx, config: CacV1, userId: string) => { + if (config.releases == null || entries(config.releases).length == 0) return; + + const releaseInfo = entries(config.releases ?? {}) + .map(([slug, release]) => { + const [systemSlug, deploymentSlug, version] = slug.split("/"); + if (systemSlug == null || deploymentSlug == null || version == null) + return null; + + const name = release.name ?? version; + return { + systemSlug, + deploymentSlug, + release: { ...release, version, name }, + }; + }) + .filter(isPresent); + + const systemSlugs = releaseInfo.map(({ systemSlug }) => systemSlug); + const deploymentSlugs = releaseInfo.map( + ({ deploymentSlug }) => deploymentSlug, + ); + + const deployments = await db + .select() + .from(schema.deployment) + .innerJoin(schema.system, eq(schema.deployment.systemId, schema.system.id)) + .innerJoin( + schema.workspace, + eq(schema.system.workspaceId, schema.workspace.id), + ) + .leftJoin( + schema.release, + eq(schema.release.deploymentId, schema.deployment.id), + ) + .where( + and( + eq(schema.workspace.id, config.workspace), + inArray(schema.system.slug, systemSlugs), + inArray(schema.deployment.slug, deploymentSlugs), + ), + ) + .then((rows) => + _.chain(rows) + .groupBy((d) => d.deployment.id) + .map((deploymentGroup) => ({ + ...deploymentGroup[0]!, + releases: deploymentGroup.map((d) => d.release).filter(isPresent), + })) + .value(), + ); + + const newReleases = releaseInfo.filter( + ({ systemSlug, deploymentSlug, release }) => { + const deployment = deployments.find( + (d) => + d.deployment.slug === deploymentSlug && d.system.slug === systemSlug, + ); + if (deployment == null) return false; + const existingRelease = deployment.releases.find( + (r) => r.version === release.version, + ); + return existingRelease == null; + }, + ); + + if (newReleases.length == 0) return; + + const releaseInserts = newReleases + .map(({ systemSlug, deploymentSlug, release }) => { + const deployment = deployments.find( + (d) => + d.deployment.slug === deploymentSlug && d.system.slug === systemSlug, + ); + if (deployment == null) return null; + return { ...release, deploymentId: deployment.deployment.id }; + }) + .filter(isPresent); + + const releases = await db + .insert(schema.release) + .values(releaseInserts) + .returning(); + + const releaseMetadataInserts = newReleases + .flatMap(({ systemSlug, deploymentSlug, release }) => { + const deployment = deployments.find( + (d) => + d.deployment.slug === deploymentSlug && d.system.slug === systemSlug, + ); + if (deployment == null) return []; + const rel = releases.find( + (r) => + r.version === release.version && + r.deploymentId === deployment.deployment.id, + ); + if (rel == null) return []; + return entries(release.metadata ?? {}).map(([key, value]) => ({ + releaseId: rel.id, + key, + value, + })); + }) + .filter(isPresent); + + if (releaseMetadataInserts.length > 0) + await db.insert(schema.releaseMetadata).values(releaseMetadataInserts); + + await createReleaseJobTriggers(db, "new_release") + .causedById(userId) + .filter(isPassingReleaseStringCheckPolicy) + .releases(releases.map((r) => r.id)) + .then(createJobApprovals) + .insert() + .then((releaseJobTriggers) => { + dispatchReleaseJobTriggers(db) + .releaseTriggers(releaseJobTriggers) + .filter(isPassingAllPolicies) + .then(cancelOldReleaseJobTriggersOnJobDispatch) + .dispatch(); + }); +}; + +export const PATCH = async (req: NextRequest) => { + const body = await req.text(); + const bodyObj = jsYaml.load(body); + const parsed = cacV1.safeParse(bodyObj); + if (!parsed.success) + return NextResponse.json({ error: parsed.error.message }, { status: 400 }); + + const workspaceId = parsed.data.workspace; + + return db.transaction(async (db) => { + const workspace = await db + .select() + .from(schema.workspace) + .where(eq(schema.workspace.id, workspaceId)) + .then(takeFirstOrNull); + + if (workspace == null) + return NextResponse.json( + { error: "Workspace not found" }, + { status: 404 }, + ); + + const user = await getUser(req); + if (user == null) + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + + const canAccess = await can() + .user(user.id) + .perform(Permission.SystemUpdate) + .on({ type: "workspace", id: workspace.id }); + + if (!canAccess) + return NextResponse.json({ error: "Permission denied" }, { status: 403 }); + + const config = parsed.data; + + try { + await upsertSystems(db, config); + await upsertDeployments(db, config); + await upsertReleases(db, config, user.id); + } catch (e) { + return NextResponse.json({ error: e }, { status: 500 }); + } + + return NextResponse.json({ success: true }, { status: 200 }); + }); +}; diff --git a/packages/api/src/root.ts b/packages/api/src/root.ts index bb2d03af..57381512 100644 --- a/packages/api/src/root.ts +++ b/packages/api/src/root.ts @@ -1,7 +1,7 @@ import { dashboardRouter } from "./router/dashboard"; import { deploymentRouter } from "./router/deployment"; import { environmentRouter } from "./router/environment"; -import { githubRouter } from "./router/github/github"; +import { githubRouter } from "./router/github"; import { jobRouter } from "./router/job"; import { releaseRouter } from "./router/release"; import { runbookRouter } from "./router/runbook"; diff --git a/packages/api/src/router/github/github.ts b/packages/api/src/router/github.ts similarity index 97% rename from packages/api/src/router/github/github.ts rename to packages/api/src/router/github.ts index 4fca2b11..bec1c241 100644 --- a/packages/api/src/router/github/github.ts +++ b/packages/api/src/router/github.ts @@ -17,9 +17,8 @@ import { } from "@ctrlplane/db/schema"; import { Permission } from "@ctrlplane/validators/auth"; -import { env } from "../../config"; -import { createTRPCRouter, protectedProcedure } from "../../trpc"; -import { createNewGithubOrganization } from "./create-github-org"; +import { env } from "../config"; +import { createTRPCRouter, protectedProcedure } from "../trpc"; const octokit = env.GITHUB_BOT_APP_ID == null @@ -338,7 +337,13 @@ export const githubRouter = createTRPCRouter({ }), }) .input(githubOrganizationInsert) - .mutation(({ ctx, input }) => createNewGithubOrganization(ctx.db, input)), + .mutation(({ ctx, input }) => + ctx.db + .insert(githubOrganization) + .values(input) + .returning() + .then(takeFirst), + ), delete: protectedProcedure .meta({ diff --git a/packages/api/src/router/github/create-github-org.ts b/packages/api/src/router/github/create-github-org.ts deleted file mode 100644 index 22bf33da..00000000 --- a/packages/api/src/router/github/create-github-org.ts +++ /dev/null @@ -1,230 +0,0 @@ -import type { Tx } from "@ctrlplane/db"; -import type { - GithubOrganization, - GithubOrganizationInsert, -} from "@ctrlplane/db/schema"; -import type { RestEndpointMethodTypes } from "@octokit/rest"; -import { createAppAuth } from "@octokit/auth-app"; -import { Octokit } from "@octokit/rest"; -import * as yaml from "js-yaml"; -import { isPresent } from "ts-is-present"; - -import { and, eq, inArray, sql, takeFirst } from "@ctrlplane/db"; -import { - deployment, - githubConfigFile, - githubOrganization, - system, - workspace, -} from "@ctrlplane/db/schema"; -import { configFile } from "@ctrlplane/validators"; - -import { env } from "../../config"; - -type ConfigFile = - RestEndpointMethodTypes["search"]["code"]["response"]["data"]["items"][number]; - -type ParsedConfigFile = ConfigFile & { - content: { - deployments: { - name: string; - slug: string; - system: string; - workspace: string; - description?: string | undefined; - }[]; - }; -}; - -const isValidGithubAppConfiguration = - env.GITHUB_BOT_APP_ID != null && - env.GITHUB_BOT_PRIVATE_KEY != null && - env.GITHUB_BOT_CLIENT_ID != null && - env.GITHUB_BOT_CLIENT_SECRET != null; - -const octokit = isValidGithubAppConfiguration - ? new Octokit({ - authStrategy: createAppAuth, - auth: { - appId: env.GITHUB_BOT_APP_ID, - privateKey: env.GITHUB_BOT_PRIVATE_KEY, - clientId: env.GITHUB_BOT_CLIENT_ID, - clientSecret: env.GITHUB_BOT_CLIENT_SECRET, - }, - }) - : null; - -const getOctokitInstallation = (installationId: number) => - isValidGithubAppConfiguration - ? new Octokit({ - authStrategy: createAppAuth, - auth: { - appId: env.GITHUB_BOT_APP_ID, - privateKey: env.GITHUB_BOT_PRIVATE_KEY, - clientId: env.GITHUB_BOT_CLIENT_ID, - clientSecret: env.GITHUB_BOT_CLIENT_SECRET, - installationId, - }, - }) - : null; - -const parseConfigFile = async ( - cf: ConfigFile, - orgName: string, - branch: string, - installationOctokit: Octokit, -) => { - const content = await installationOctokit.repos - .getContent({ - owner: orgName, - repo: cf.repository.name, - path: cf.path, - ref: branch, - }) - .then(({ data }) => { - if (!("content" in data)) throw new Error("Invalid response data"); - return Buffer.from(data.content, "base64").toString("utf-8"); - }); - - const yamlContent = yaml.load(content); - const parsed = configFile.safeParse(yamlContent); - if (!parsed.success) throw new Error("Invalid config file"); - return { ...cf, content: parsed.data }; -}; - -const processParsedConfigFiles = async ( - db: Tx, - parsedConfigFiles: ParsedConfigFile[], - org: GithubOrganization, -) => { - const deploymentInfo = await db - .select() - .from(system) - .innerJoin(workspace, eq(system.workspaceId, workspace.id)) - .where( - and( - inArray( - system.slug, - parsedConfigFiles - .map((d) => d.content.deployments.map((d) => d.system)) - .flat(), - ), - inArray( - workspace.slug, - parsedConfigFiles.flatMap((d) => - d.content.deployments.map((d) => d.workspace), - ), - ), - ), - ); - - const insertedConfigFiles = await db - .insert(githubConfigFile) - .values( - parsedConfigFiles.map((d) => ({ - ...d, - workspaceId: org.workspaceId, - organizationId: org.id, - repositoryName: d.repository.name, - })), - ) - .returning(); - - const deployments = parsedConfigFiles.flatMap((cf) => - cf.content.deployments.map((d) => { - const info = deploymentInfo.find( - (i) => i.system.slug === d.system && i.workspace.slug === d.workspace, - ); - if (info == null) throw new Error("Deployment info not found"); - const { system, workspace } = info; - - return { - ...d, - systemId: system.id, - workspaceId: workspace.id, - description: d.description ?? "", - githubConfigFileId: insertedConfigFiles.find( - (icf) => - icf.path === cf.path && icf.repositoryName === cf.repository.name, - )?.id, - }; - }), - ); - - await db - .insert(deployment) - .values(deployments) - .onConflictDoUpdate({ - target: [deployment.systemId, deployment.slug], - set: { - githubConfigFileId: sql`excluded.github_config_file_id`, - }, - }); -}; - -export const createNewGithubOrganization = async ( - db: Tx, - githubOrganizationConfig: GithubOrganizationInsert, -) => - db.transaction(async (db) => { - const org = await db - .insert(githubOrganization) - .values(githubOrganizationConfig) - .returning() - .then(takeFirst); - - const installation = await octokit?.apps.getInstallation({ - installation_id: org.installationId, - }); - if (installation == null) throw new Error("Failed to get installation"); - - const installationOctokit = getOctokitInstallation(installation.data.id); - if (installationOctokit == null) - throw new Error("Failed to get authenticated Github client"); - const installationToken = (await installationOctokit.auth({ - type: "installation", - installationId: installation.data.id, - })) as { token: string }; - - const configFiles = await Promise.all([ - installationOctokit.search.code({ - q: `org:${org.organizationName} filename:ctrlplane.yaml`, - per_page: 100, - headers: { - "X-GitHub-Api-Version": "2022-11-28", - authorization: `Bearer ${installationToken.token}`, - }, - }), - installationOctokit.search.code({ - q: `org:${org.organizationName} filename:ctrlplane.yml`, - per_page: 100, - headers: { - "X-GitHub-Api-Version": "2022-11-28", - authorization: `Bearer ${installationToken.token}`, - }, - }), - ]).then(([yamlFiles, ymlFiles]) => [ - ...yamlFiles.data.items, - ...ymlFiles.data.items, - ]); - - if (configFiles.length === 0) return; - - const parsedConfigFiles = await Promise.allSettled( - configFiles.map((cf) => - parseConfigFile( - cf, - org.organizationName, - org.branch, - installationOctokit, - ), - ), - ).then((results) => - results - .map((r) => (r.status === "fulfilled" ? r.value : null)) - .filter(isPresent), - ); - - if (parsedConfigFiles.length === 0) return; - await processParsedConfigFiles(db, parsedConfigFiles, org); - }); diff --git a/packages/node-sdk/openapi.yaml b/packages/node-sdk/openapi.yaml index 6b737cd8..9400f404 100644 --- a/packages/node-sdk/openapi.yaml +++ b/packages/node-sdk/openapi.yaml @@ -3,6 +3,36 @@ info: title: Ctrlplane API version: 1.0.0 paths: + /v1/workspaces/{workspace}/config: + post: + summary: Create resources from a config file + operationId: updateConfig + parameters: + - name: workspace + in: path + required: true + schema: + type: string + description: Name of the workspace + requestBody: + required: true + content: + application/yaml: + schema: + type: string + responses: + "200": + description: Successfully created resources + "400": + description: Invalid request + "401": + description: Unauthorized + "403": + description: Permission denied + "404": + description: Workspace not found + "500": + description: Internal server error /v1/workspaces/{workspace}/job-agents/name: patch: summary: Upserts the agent diff --git a/packages/node-sdk/src/apis/DefaultApi.ts b/packages/node-sdk/src/apis/DefaultApi.ts index 55272f85..6bdd0c25 100644 --- a/packages/node-sdk/src/apis/DefaultApi.ts +++ b/packages/node-sdk/src/apis/DefaultApi.ts @@ -76,6 +76,11 @@ export interface SetTargetProvidersTargetsOperationRequest { setTargetProvidersTargetsRequest: SetTargetProvidersTargetsRequest; } +export interface UpdateConfigRequest { + workspace: string; + body: string; +} + export interface UpdateJobOperationRequest { jobId: string; updateJobRequest: UpdateJobRequest; @@ -430,6 +435,65 @@ export class DefaultApi extends runtime.BaseAPI { await this.setTargetProvidersTargetsRaw(requestParameters, initOverrides); } + /** + * Create resources from a config file + */ + async updateConfigRaw( + requestParameters: UpdateConfigRequest, + initOverrides?: RequestInit | runtime.InitOverrideFunction, + ): Promise> { + if (requestParameters["workspace"] == null) { + throw new runtime.RequiredError( + "workspace", + 'Required parameter "workspace" was null or undefined when calling updateConfig().', + ); + } + + if (requestParameters["body"] == null) { + throw new runtime.RequiredError( + "body", + 'Required parameter "body" was null or undefined when calling updateConfig().', + ); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + headerParameters["Content-Type"] = "application/yaml"; + + if (this.configuration && this.configuration.apiKey) { + headerParameters["x-api-key"] = + await this.configuration.apiKey("x-api-key"); // apiKey authentication + } + + const response = await this.request( + { + path: `/v1/workspaces/{workspace}/config`.replace( + `{${"workspace"}}`, + encodeURIComponent(String(requestParameters["workspace"])), + ), + method: "POST", + headers: headerParameters, + query: queryParameters, + body: requestParameters["body"] as any, + }, + initOverrides, + ); + + return new runtime.VoidApiResponse(response); + } + + /** + * Create resources from a config file + */ + async updateConfig( + requestParameters: UpdateConfigRequest, + initOverrides?: RequestInit | runtime.InitOverrideFunction, + ): Promise { + await this.updateConfigRaw(requestParameters, initOverrides); + } + /** * Update a job */ diff --git a/packages/validators/package.json b/packages/validators/package.json index 287f5da2..dbc4af3d 100644 --- a/packages/validators/package.json +++ b/packages/validators/package.json @@ -39,6 +39,10 @@ "./environment-policies": { "types": "./src/environment-policies/index.ts", "default": "./dist/environment-policies/index.js" + }, + "./cac": { + "types": "./src/cac/index.ts", + "default": "./dist/cac/index.js" } }, "license": "MIT", diff --git a/packages/validators/src/cac/index.ts b/packages/validators/src/cac/index.ts index 74f3f10c..261e96dc 100644 --- a/packages/validators/src/cac/index.ts +++ b/packages/validators/src/cac/index.ts @@ -1,13 +1,14 @@ import { z } from "zod"; export const release = z.object({ - name: z.string(), - config: z.record(z.any()), - metadata: z.record(z.string()), + name: z.string().optional(), + config: z.record(z.any()).optional(), + metadata: z.record(z.string()).optional(), }); export const deployment = z.object({ name: z.string().optional(), + description: z.string().optional(), releases: z.array(release).optional(), jobAgent: z.object({ id: z.string(), config: z.record(z.any()) }).optional(), }); @@ -26,3 +27,4 @@ export const cacV1 = z.object({ deployments: z.record(deployment).optional(), releases: z.record(release).optional(), }); +export type CacV1 = z.infer;