diff --git a/.env.example b/.env.example index 5fafd320..b0bc43b8 100644 --- a/.env.example +++ b/.env.example @@ -13,9 +13,6 @@ POSTGRES_URL="postgres://postgres.[USERNAME]:[PASSWORD]@aws-0-eu-central-1.poole AUTH_SECRET='supersecret' -AMQP_URL="amqp://ctrlplane:ctrlplane@127.0.0.1:5672" - - JOB_AGENT_WORKSPACE="ctrlplane" JOB_AGENT_NAME="agent" JOB_AGENT_API_KEY= \ No newline at end of file diff --git a/agents/github-app/.env.example b/agents/github-app/.env.example deleted file mode 100644 index ae6f6d98..00000000 --- a/agents/github-app/.env.example +++ /dev/null @@ -1,8 +0,0 @@ -GITHUB_BOT_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----..." -GITHUB_BOT_CLIENT_ID=your-client-id -GITHUB_BOT_CLIENT_SECRET=your-client-secret -GITHUB_BOT_APP_ID=your-app-id - -GITHUB_JOB_AGENT_WORKSPACE="ctrlplane" -GITHUB_JOB_AGENT_NAME="ctrl-plane" -GITHUB_JOB_AGENT_API_KEY=your-api-key \ No newline at end of file diff --git a/agents/github-app/Dockerfile b/agents/github-app/Dockerfile deleted file mode 100644 index 47b63ed1..00000000 --- a/agents/github-app/Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -ARG NODE_VERSION=22 -FROM node:${NODE_VERSION}-alpine AS base - - -FROM base AS builder -RUN apk add --no-cache libc6-compat -RUN apk update - -RUN npm install -g turbo - - -FROM base AS installer - -RUN apk add --no-cache libc6-compat -RUN apk update - -RUN npm install -g pnpm@$PNPM_VERSION - -ARG PNPM_VERSION=9.4.0 -ENV PNPM_HOME="/pnpm" -ENV PATH="$PNPM_HOME:$PATH" - -RUN corepack enable -RUN corepack prepare pnpm@$PNPM_VERSION --activate - -WORKDIR /app - -COPY .gitignore .gitignore -COPY turbo.json turbo.json -RUN pnpm add -g turbo - -COPY package.json package.json -COPY pnpm-*.yaml . - -COPY tooling/tailwind/package.json ./tooling/tailwind/package.json -COPY tooling/prettier/package.json ./tooling/prettier/package.json -COPY tooling/eslint/package.json ./tooling/eslint/package.json -COPY tooling/typescript/package.json ./tooling/typescript/package.json - -COPY packages/api/package.json ./packages/api/package.json -COPY packages/db/package.json ./packages/db/package.json -COPY packages/dispatch/package.json ./packages/dispatch/package.json - -COPY agents/github-app/package.json ./apps/github-app/package.json - -RUN pnpm install --frozen-lockfile - -COPY . . - -RUN turbo build --filter=...@ctrlplane/agent-github-app - -FROM base AS runner -WORKDIR /app - -RUN addgroup --system --gid 1001 nodejs -RUN adduser --system --uid 1001 nodejs -USER nodejs - -COPY --from=installer --chown=nodejs:nodejs /app/agents/github-app/dist ./agents/github-app/dist - -EXPOSE 3000 -ENV PORT 3000 - -CMD node agents/github-app/dist/index.js \ No newline at end of file diff --git a/agents/github-app/README.md b/agents/github-app/README.md deleted file mode 100644 index 1c7e117a..00000000 --- a/agents/github-app/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Github Agent - -Agent for github workflows - -Add this configuration to your workflow in order to use it for testing. - -```YAML -run-name: Simple Workflow [${{ inputs.job_execution_id && inputs.job_execution_id || '' }}] - -on: - workflow_dispatch: - inputs: - job_execution_id: - description: 'Job execution ID' - required: true -``` diff --git a/agents/github-app/eslint.config.js b/agents/github-app/eslint.config.js deleted file mode 100644 index 7e2fed8a..00000000 --- a/agents/github-app/eslint.config.js +++ /dev/null @@ -1,9 +0,0 @@ -import baseConfig from "@ctrlplane/eslint-config/base"; - -/** @type {import('typescript-eslint').Config} */ -export default [ - { - ignores: [".nitro/**", ".output/**", "dist/**", "node_modules/**"], - }, - ...baseConfig, -]; diff --git a/agents/github-app/package.json b/agents/github-app/package.json deleted file mode 100644 index 6fb5a18b..00000000 --- a/agents/github-app/package.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "@ctrlplane/agent-github-app", - "private": true, - "type": "module", - "scripts": { - "clean": "rm -rf .turbo node_modules", - "dev": "pnpm with-env tsx watch --clear-screen=false src/index.ts", - "build": "pnpm with-env tsc", - "lint": "eslint", - "format": "prettier --check . --ignore-path ../../.gitignore", - "typecheck": "tsc --noEmit", - "with-env": "dotenv -e ../../.env --" - }, - "dependencies": { - "@ctrlplane/api": "workspace:*", - "@ctrlplane/db": "workspace:*", - "@octokit/auth-app": "^7.1.0", - "@octokit/rest": "^20.1.1", - "@t3-oss/env-core": "^0.10.1", - "@trpc/client": "11.0.0-rc.364", - "cron": "^3.1.7", - "dotenv": "^16.4.5", - "lodash": "^4.17.21", - "p-retry": "^6.2.0" - }, - "devDependencies": { - "@ctrlplane/eslint-config": "workspace:^", - "@ctrlplane/prettier-config": "workspace:^", - "@ctrlplane/tailwind-config": "workspace:*", - "@ctrlplane/tsconfig": "workspace:*", - "@types/lodash": "^4.17.5", - "@types/node": "^20.12.9", - "eslint": "catalog:", - "prettier": "catalog:", - "tsx": "^4.11.0", - "typescript": "^5.4.5" - }, - "prettier": "@ctrlplane/prettier-config" -} diff --git a/agents/github-app/src/api.ts b/agents/github-app/src/api.ts deleted file mode 100644 index 28758875..00000000 --- a/agents/github-app/src/api.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { AppRouter } from "@ctrlplane/api"; -import { createTRPCClient, httpBatchLink } from "@trpc/client"; -import SuperJSON from "superjson"; - -import { env } from "./config"; - -export const api = createTRPCClient({ - links: [ - httpBatchLink({ - url: `${env.API}/api/trpc`, - headers: () => { - const basic = Buffer.from( - `${env.GITHUB_JOB_AGENT_NAME}:${env.GITHUB_JOB_AGENT_API_KEY}`, - ).toString("base64"); - return { - "x-trpc-source": "github-app", - authorization: `Basic ${basic}`, - }; - }, - transformer: SuperJSON, - }), - ], -}); diff --git a/agents/github-app/src/config.ts b/agents/github-app/src/config.ts deleted file mode 100644 index 9636aece..00000000 --- a/agents/github-app/src/config.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { createEnv } from "@t3-oss/env-core"; -import dotenv from "dotenv"; -import { z } from "zod"; - -dotenv.config(); - -export const env = createEnv({ - server: { - API: z.string().default("http://localhost:3000"), - CRON_TIME: z.string().default("* * * * *"), - - GITHUB_JOB_AGENT_WORKSPACE: z.string().min(3), - GITHUB_JOB_AGENT_NAME: z.string().min(4), - GITHUB_JOB_AGENT_API_KEY: z.string(), - - GITHUB_URL: z.string().url().default("https://github.com"), - GITHUB_BOT_NAME: z.string().min(1), - GITHUB_BOT_PRIVATE_KEY: z.string().min(1), - GITHUB_BOT_CLIENT_ID: z.string().min(1), - GITHUB_BOT_CLIENT_SECRET: z.string().min(1), - GITHUB_BOT_APP_ID: z.string().min(1), - }, - runtimeEnv: process.env, -}); diff --git a/agents/github-app/src/index.ts b/agents/github-app/src/index.ts deleted file mode 100644 index 16d1671f..00000000 --- a/agents/github-app/src/index.ts +++ /dev/null @@ -1,174 +0,0 @@ -import type { JobExecution, JobExecutionStatus } from "@ctrlplane/db/schema"; -import { createAppAuth } from "@octokit/auth-app"; -import { Octokit } from "@octokit/rest"; -import { CronJob } from "cron"; -import pRetry from "p-retry"; -import { z } from "zod"; - -import { api } from "./api"; -import { env } from "./config"; - -const configSchema = z.object({ - installationId: z.number(), - login: z.string().min(1), - repo: z.string().min(1), - workflowId: z.number(), -}); - -const convertStatus = (status: string): JobExecutionStatus => { - if (status === "success" || status === "neutral") return "completed"; - if (status === "queued" || status === "requested" || status === "waiting") - return "pending"; - if (status === "timed_out" || status === "stale") return "failure"; - return status as JobExecutionStatus; -}; - -const dispatchGithubJobExecution = async (jobExecution: JobExecution) => { - console.log(`Dispatching job execution ${jobExecution.id}...`); - - const config = jobExecution.jobAgentConfig; - configSchema.parse(config); - - const octokit = new Octokit({ - authStrategy: createAppAuth, - auth: { - appId: env.GITHUB_BOT_APP_ID, - privateKey: env.GITHUB_BOT_PRIVATE_KEY, - clientId: env.GITHUB_BOT_CLIENT_ID, - clientSecret: env.GITHUB_BOT_CLIENT_SECRET, - installationId: config.installationId, - }, - }); - - const installationToken = (await octokit.auth({ - type: "installation", - installationId: config.installationId, - })) as { token: string }; - - await octokit.actions.createWorkflowDispatch({ - owner: config.login, - repo: config.repo, - workflow_id: config.workflowId, - ref: "main", - inputs: { - job_execution_id: jobExecution.id.slice(0, 8), - }, - headers: { - "X-GitHub-Api-Version": "2022-11-28", - authorization: `Bearer ${installationToken.token}`, - }, - }); - - const { runId, status } = await pRetry( - async () => { - const runs = await octokit.actions.listWorkflowRuns({ - owner: config.login, - repo: config.repo, - workflow_id: config.workflowId, - }); - - const run = runs.data.workflow_runs.find((run) => - run.name?.includes(jobExecution.id.slice(0, 8)), - ); - - return { runId: run?.id, status: run?.status }; - }, - { retries: 15, minTimeout: 1000 }, - ); - - if (runId == null) { - console.error(`Run ID not found for job execution ${jobExecution.id}`); - return; - } - - await api.job.execution.update.mutate({ - id: jobExecution.id, - data: { - externalRunId: runId.toString(), - status: convertStatus(status ?? "pending"), - }, - }); -}; - -const updateJobExecutionStatus = async (jobExecution: JobExecution) => { - const config = jobExecution.jobAgentConfig; - configSchema.parse(config); - - const octokit = new Octokit({ - authStrategy: createAppAuth, - auth: { - appId: env.GITHUB_BOT_APP_ID, - privateKey: env.GITHUB_BOT_PRIVATE_KEY, - clientId: env.GITHUB_BOT_CLIENT_ID, - clientSecret: env.GITHUB_BOT_CLIENT_SECRET, - installationId: config.installationId, - }, - }); - - const runId = await pRetry( - async () => { - const runs = await octokit.actions.listWorkflowRuns({ - owner: config.login, - repo: config.repo, - workflow_id: config.workflowId, - }); - - const run = runs.data.workflow_runs.find((run) => - run.name?.includes(jobExecution.id.slice(0, 8)), - ); - - return run?.id; - }, - { retries: 15, minTimeout: 1000 }, - ); - - if (runId == null) { - console.error(`Run ID not found for job execution ${jobExecution.id}`); - return; - } - - const { data: workflowState } = await octokit.actions.getWorkflowRun({ - owner: config.login, - repo: config.repo, - run_id: runId, - }); - - const status = convertStatus(workflowState.status ?? "pending"); - - return api.job.execution.update.mutate({ - id: jobExecution.id, - data: { status }, - }); -}; - -const run = async () => { - console.log("Running job agent..."); - - const githubAgents = await api.job.agent.byType.query("github-app"); - - console.log(`[*] Found ${githubAgents.length} GitHub job agent(s) to run.`); - - await Promise.allSettled( - githubAgents.map(async (jobAgent) => { - const jobExecutions = await api.job.execution.list.byAgentId.query( - jobAgent.id, - ); - return Promise.allSettled( - jobExecutions - .filter((jobExecution) => jobExecution.status !== "completed") - .map((jobExecution) => - jobExecution.externalRunId == null - ? dispatchGithubJobExecution(jobExecution) - : updateJobExecutionStatus(jobExecution), - ), - ); - }), - ); -}; - -const job = new CronJob(env.CRON_TIME, run, null, true); - -console.log("Starting cron job..."); - -run().catch(console.error); -job.start(); diff --git a/agents/github-app/tsconfig.json b/agents/github-app/tsconfig.json deleted file mode 100644 index ef186d5c..00000000 --- a/agents/github-app/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "extends": "@ctrlplane/tsconfig/base.json", - "compilerOptions": { - "target": "ESNext", - "outDir": "dist", - "noEmit": false, - "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json", - "baseUrl": ".", - "paths": { - "@/*": ["src/*"] - }, - "esModuleInterop": true, - "importsNotUsedAsValues": "remove" - }, - "include": ["src", "*.ts"], - "exclude": ["node_modules"] -} diff --git a/apps/event-worker/eslint.config.js b/apps/event-worker/eslint.config.js index 3a9fadb9..48eb41bc 100644 --- a/apps/event-worker/eslint.config.js +++ b/apps/event-worker/eslint.config.js @@ -3,7 +3,7 @@ import baseConfig, { requireJsSuffix } from "@ctrlplane/eslint-config/base"; /** @type {import('typescript-eslint').Config} */ export default [ { - ignores: [".nitro/**", ".output/**"], + ignores: [".nitro/**", ".output/**", "dist/**"], }, ...requireJsSuffix, ...baseConfig, diff --git a/apps/event-worker/package.json b/apps/event-worker/package.json index 70607048..48adea92 100644 --- a/apps/event-worker/package.json +++ b/apps/event-worker/package.json @@ -17,6 +17,8 @@ "@ctrlplane/validators": "workspace:*", "@google-cloud/container": "^5.16.0", "@kubernetes/client-node": "^0.21.0", + "@octokit/auth-app": "^7.1.0", + "@octokit/rest": "^20.1.1", "@t3-oss/env-core": "^0.10.1", "bullmq": "^5.12.10", "cron": "^3.1.7", @@ -25,6 +27,7 @@ "ioredis": "^5.4.1", "lodash": "^4.17.21", "ms": "^2.1.3", + "p-retry": "^6.2.0", "semver": "^7.6.2", "zod": "catalog:" }, @@ -33,6 +36,7 @@ "@ctrlplane/prettier-config": "workspace:^", "@ctrlplane/tsconfig": "workspace:*", "@types/lodash": "^4.17.5", + "@types/ms": "^0.7.34", "@types/node": "^20.12.9", "@types/semver": "^7.5.8", "eslint": "catalog:", diff --git a/apps/event-worker/src/config.ts b/apps/event-worker/src/config.ts index eec809ad..a94e2f5d 100644 --- a/apps/event-worker/src/config.ts +++ b/apps/event-worker/src/config.ts @@ -5,6 +5,13 @@ import { z } from "zod"; dotenv.config(); export const env = createEnv({ - server: { POSTGRES_URL: z.string().url(), REDIS_URL: z.string().url() }, + server: { + POSTGRES_URL: z.string().url(), + REDIS_URL: z.string().url(), + GITHUB_BOT_APP_ID: z.string().optional(), + GITHUB_BOT_PRIVATE_KEY: z.string().optional(), + GITHUB_BOT_CLIENT_ID: z.string().optional(), + GITHUB_BOT_CLIENT_SECRET: z.string().optional(), + }, runtimeEnv: process.env, }); diff --git a/apps/event-worker/src/github-utils.ts b/apps/event-worker/src/github-utils.ts new file mode 100644 index 00000000..6c14f75f --- /dev/null +++ b/apps/event-worker/src/github-utils.ts @@ -0,0 +1,31 @@ +import type { JobExecutionStatus } from "@ctrlplane/db/schema"; +import { createAppAuth } from "@octokit/auth-app"; +import { Octokit } from "@octokit/rest"; + +import { JobExecutionStatus as JEStatus } from "@ctrlplane/validators/jobs"; + +import { env } from "./config.js"; + +export const convertStatus = (status: string): JobExecutionStatus => { + if (status === "success" || status === "neutral") return JEStatus.Completed; + if (status === "queued" || status === "requested" || status === "waiting") + return JEStatus.Pending; + if (status === "timed_out" || status === "stale") return JEStatus.Failure; + return status as JobExecutionStatus; +}; + +export const getOctokit = () => + env.GITHUB_BOT_APP_ID && + env.GITHUB_BOT_PRIVATE_KEY && + env.GITHUB_BOT_CLIENT_ID && + env.GITHUB_BOT_CLIENT_SECRET + ? new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: env.GITHUB_BOT_APP_ID, + privateKey: env.GITHUB_BOT_PRIVATE_KEY, + clientId: env.GITHUB_BOT_CLIENT_ID, + clientSecret: env.GITHUB_BOT_CLIENT_SECRET, + }, + }) + : null; diff --git a/apps/event-worker/src/index.ts b/apps/event-worker/src/index.ts index f130e464..36f2ac08 100644 --- a/apps/event-worker/src/index.ts +++ b/apps/event-worker/src/index.ts @@ -1,14 +1,17 @@ import { logger } from "@ctrlplane/logger"; +import { createDispatchExecutionJobWorker } from "./job-execution-dispatch/index.js"; import { redis } from "./redis.js"; import { createTargetScanWorker } from "./target-scan/index.js"; const targetScanWorker = createTargetScanWorker(); +const dispatchExecutionJobWorker = createDispatchExecutionJobWorker(); const shutdown = () => { logger.warn("Exiting..."); targetScanWorker.close(); + dispatchExecutionJobWorker.close(); redis.quit(); process.exit(0); diff --git a/apps/event-worker/src/job-execution-dispatch/github.ts b/apps/event-worker/src/job-execution-dispatch/github.ts new file mode 100644 index 00000000..102cd412 --- /dev/null +++ b/apps/event-worker/src/job-execution-dispatch/github.ts @@ -0,0 +1,115 @@ +import type { JobExecution } from "@ctrlplane/db/schema"; +import { Queue } from "bullmq"; +import ms from "ms"; +import pRetry from "p-retry"; + +import { and, eq, takeFirstOrNull } from "@ctrlplane/db"; +import { db } from "@ctrlplane/db/client"; +import { githubOrganization, jobExecution } from "@ctrlplane/db/schema"; +import { Channel } from "@ctrlplane/validators/events"; +import { configSchema } from "@ctrlplane/validators/github"; +import { JobExecutionStatus } from "@ctrlplane/validators/jobs"; + +import { convertStatus, getOctokit } from "../github-utils.js"; +import { redis } from "../redis.js"; + +const jobExecutionSyncQueue = new Queue(Channel.JobExecutionSync, { + connection: redis, +}); + +export const dispatchGithubJobExecution = async (je: JobExecution) => { + console.log(`Dispatching job execution ${je.id}...`); + + const config = je.jobAgentConfig; + const parsed = configSchema.safeParse(config); + if (!parsed.success) { + await db.update(jobExecution).set({ + status: JobExecutionStatus.InvalidJobAgent, + message: `Invalid job agent config for job execution ${je.id}: ${parsed.error.message}`, + }); + return; + } + + const ghOrg = await db + .select() + .from(githubOrganization) + .where( + and( + eq(githubOrganization.installationId, config.installationId), + eq(githubOrganization.organizationName, config.organizationName), + ), + ) + .then(takeFirstOrNull); + + if (ghOrg == null) { + await db.update(jobExecution).set({ + status: JobExecutionStatus.InvalidIntegration, + message: `GitHub organization not found for job execution ${je.id}`, + }); + return; + } + + const octokit = getOctokit(); + if (octokit == null) { + await db.update(jobExecution).set({ + status: JobExecutionStatus.InvalidJobAgent, + message: "GitHub bot not configured", + }); + return; + } + + const installationToken = (await octokit.auth({ + type: "installation", + installationId: config.installationId, + })) as { token: string }; + + await octokit.actions.createWorkflowDispatch({ + owner: config.login, + repo: config.repo, + workflow_id: config.workflowId, + ref: ghOrg.branch, + inputs: { + job_execution_id: je.id.slice(0, 8), + }, + headers: { + "X-GitHub-Api-Version": "2022-11-28", + authorization: `Bearer ${installationToken.token}`, + }, + }); + + const { runId, status } = await pRetry( + async () => { + const runs = await octokit.actions.listWorkflowRuns({ + owner: config.login, + repo: config.repo, + workflow_id: config.workflowId, + }); + + const run = runs.data.workflow_runs.find((run) => + run.name?.includes(je.id.slice(0, 8)), + ); + + return { runId: run?.id, status: run?.status }; + }, + { retries: 15, minTimeout: 1000 }, + ); + + if (runId == null) { + await db.update(jobExecution).set({ + status: JobExecutionStatus.ExternalRunNotFound, + message: `Run ID not found for job execution ${je.id}`, + }); + return; + } + + await db.update(jobExecution).set({ + externalRunId: runId.toString(), + status: convertStatus(status ?? JobExecutionStatus.Pending), + }); + + await jobExecutionSyncQueue.add( + je.id, + { jobExecutionId: je.id }, + { repeat: { every: ms("10s") } }, + ); +}; diff --git a/apps/event-worker/src/job-execution-dispatch/index.ts b/apps/event-worker/src/job-execution-dispatch/index.ts new file mode 100644 index 00000000..fa231090 --- /dev/null +++ b/apps/event-worker/src/job-execution-dispatch/index.ts @@ -0,0 +1,44 @@ +import type { DispatchJobExecutionEvent } from "@ctrlplane/validators/events"; +import { Worker } from "bullmq"; + +import { eq, takeFirstOrNull } from "@ctrlplane/db"; +import { db } from "@ctrlplane/db/client"; +import { jobAgent, jobExecution } from "@ctrlplane/db/schema"; +import { Channel } from "@ctrlplane/validators/events"; +import { JobAgentType, JobExecutionStatus } from "@ctrlplane/validators/jobs"; + +import { redis } from "../redis.js"; +import { dispatchGithubJobExecution } from "./github.js"; + +export const createDispatchExecutionJobWorker = () => + new Worker( + Channel.DispatchJobExecution, + (job) => + db + .select() + .from(jobExecution) + .innerJoin(jobAgent, eq(jobExecution.jobAgentId, jobAgent.id)) + .where(eq(jobExecution.id, job.data.jobExecutionId)) + .then(takeFirstOrNull) + .then((je) => { + if (je == null) return; + + try { + if (je.job_agent.type === String(JobAgentType.GithubApp)) + dispatchGithubJobExecution(je.job_execution); + } catch (error) { + db.update(jobExecution) + .set({ + status: JobExecutionStatus.Failure, + message: (error as Error).message, + }) + .where(eq(jobExecution.id, je.job_execution.id)); + } + }), + { + connection: redis, + removeOnComplete: { age: 0, count: 0 }, + removeOnFail: { age: 0, count: 0 }, + concurrency: 10, + }, + ); diff --git a/apps/event-worker/src/job-execution-sync/github.ts b/apps/event-worker/src/job-execution-sync/github.ts new file mode 100644 index 00000000..b0af14c9 --- /dev/null +++ b/apps/event-worker/src/job-execution-sync/github.ts @@ -0,0 +1,66 @@ +import type { JobExecution } from "@ctrlplane/db/schema"; +import pRetry from "p-retry"; + +import { eq } from "@ctrlplane/db"; +import { db } from "@ctrlplane/db/client"; +import { jobExecution } from "@ctrlplane/db/schema"; +import { configSchema } from "@ctrlplane/validators/github"; +import { JobExecutionStatus } from "@ctrlplane/validators/jobs"; + +import { convertStatus, getOctokit } from "../github-utils.js"; + +export const syncGithubJobExecution = async (je: JobExecution) => { + const config = je.jobAgentConfig; + configSchema.parse(config); + + const octokit = getOctokit(); + if (octokit == null) { + await db.update(jobExecution).set({ + status: JobExecutionStatus.InvalidJobAgent, + message: "GitHub bot not configured", + }); + return; + } + + const runId = await pRetry( + async () => { + const runs = await octokit.actions.listWorkflowRuns({ + owner: config.login, + repo: config.repo, + workflow_id: config.workflowId, + }); + + const run = runs.data.workflow_runs.find((run) => + run.name?.includes(je.id.slice(0, 8)), + ); + + return run?.id; + }, + { retries: 15, minTimeout: 1000 }, + ); + + if (runId == null) { + await db.update(jobExecution).set({ + status: JobExecutionStatus.ExternalRunNotFound, + message: `Run ID not found for job execution ${je.id}`, + }); + return; + } + + const { data: workflowState } = await octokit.actions.getWorkflowRun({ + owner: config.login, + repo: config.repo, + run_id: runId, + }); + + const status = convertStatus( + workflowState.status ?? JobExecutionStatus.Pending, + ); + + await db + .update(jobExecution) + .set({ status }) + .where(eq(jobExecution.id, je.id)); + + return status === JobExecutionStatus.Completed; +}; diff --git a/apps/event-worker/src/job-execution-sync/index.ts b/apps/event-worker/src/job-execution-sync/index.ts new file mode 100644 index 00000000..6278c0cc --- /dev/null +++ b/apps/event-worker/src/job-execution-sync/index.ts @@ -0,0 +1,65 @@ +import type { JobExecution } from "@ctrlplane/db/schema"; +import type { DispatchJobExecutionEvent } from "@ctrlplane/validators/events"; +import type { Job } from "bullmq"; +import { Queue, Worker } from "bullmq"; + +import { eq, takeFirstOrNull } from "@ctrlplane/db"; +import { db } from "@ctrlplane/db/client"; +import { jobAgent, jobExecution } from "@ctrlplane/db/schema"; +import { Channel } from "@ctrlplane/validators/events"; +import { JobAgentType, JobExecutionStatus } from "@ctrlplane/validators/jobs"; + +import { redis } from "../redis.js"; +import { syncGithubJobExecution } from "./github.js"; + +const jobExecutionSyncQueue = new Queue(Channel.JobExecutionSync, { + connection: redis, +}); +const removeJobExecutionSyncJob = (job: Job) => + job.repeatJobKey != null + ? jobExecutionSyncQueue.removeRepeatableByKey(job.repeatJobKey) + : null; + +type SyncFunction = (je: JobExecution) => Promise; + +const getSyncFunction = (agentType: string): SyncFunction | null => { + if (agentType === String(JobAgentType.GithubApp)) + return syncGithubJobExecution; + return null; +}; + +export const createJobExecutionSyncWorker = () => { + new Worker( + Channel.JobExecutionSync, + (job) => + db + .select() + .from(jobExecution) + .innerJoin(jobAgent, eq(jobExecution.jobAgentId, jobAgent.id)) + .where(eq(jobExecution.id, job.data.jobExecutionId)) + .then(takeFirstOrNull) + .then((je) => { + if (je == null) return; + + const syncFunction = getSyncFunction(je.job_agent.type); + if (syncFunction == null) return; + + try { + syncFunction(je.job_execution).then( + (isCompleted) => isCompleted && removeJobExecutionSyncJob(job), + ); + } catch (error) { + db.update(jobExecution).set({ + status: JobExecutionStatus.Failure, + message: (error as Error).message, + }); + } + }), + { + connection: redis, + removeOnComplete: { age: 0, count: 0 }, + removeOnFail: { age: 0, count: 0 }, + concurrency: 10, + }, + ); +}; diff --git a/apps/job-policy-checker/Dockerfile b/apps/job-policy-checker/Dockerfile index 54c3b72a..4201316d 100644 --- a/apps/job-policy-checker/Dockerfile +++ b/apps/job-policy-checker/Dockerfile @@ -23,6 +23,7 @@ COPY tooling/eslint/package.json ./tooling/eslint/package.json COPY tooling/typescript/package.json ./tooling/typescript/package.json COPY packages/db/package.json ./packages/db/package.json +COPY packages/validators/package.json ./packages/validators/package.json COPY packages/job-dispatch/package.json ./packages/job-dispatch/package.json COPY apps/job-policy-checker/package.json ./apps/job-policy-checker/package.json diff --git a/apps/job-policy-checker/package.json b/apps/job-policy-checker/package.json index 2f9d33c1..5bf4183e 100644 --- a/apps/job-policy-checker/package.json +++ b/apps/job-policy-checker/package.json @@ -15,6 +15,7 @@ "dependencies": { "@ctrlplane/db": "workspace:*", "@ctrlplane/job-dispatch": "workspace:*", + "@ctrlplane/validators": "workspace:*", "@t3-oss/env-core": "^0.10.1", "cron": "^3.1.7", "zod": "catalog:" diff --git a/apps/webservice/package.json b/apps/webservice/package.json index 4613c2e9..b3f904c9 100644 --- a/apps/webservice/package.json +++ b/apps/webservice/package.json @@ -42,7 +42,6 @@ "@xterm/addon-search": "^0.15.0", "@xterm/addon-web-links": "^0.11.0", "@xterm/xterm": "^5.5.0", - "amqplib": "^0.10.4", "change-case": "^5.4.4", "dagre": "^0.8.5", "date-fns": "^3.6.0", @@ -81,7 +80,6 @@ "@ctrlplane/prettier-config": "workspace:*", "@ctrlplane/tailwind-config": "workspace:*", "@ctrlplane/tsconfig": "workspace:*", - "@types/amqplib": "^0.10.5", "@types/dagre": "^0.7.52", "@types/node": "^20.12.9", "@types/randomcolor": "^0.5.9", diff --git a/apps/webservice/src/app/[workspaceSlug]/settings/(settings)/workspace/integrations/(integration)/github/GithubOrgConfig.tsx b/apps/webservice/src/app/[workspaceSlug]/settings/(settings)/workspace/integrations/(integration)/github/GithubOrgConfig.tsx index f5d0d83c..d774a936 100644 --- a/apps/webservice/src/app/[workspaceSlug]/settings/(settings)/workspace/integrations/(integration)/github/GithubOrgConfig.tsx +++ b/apps/webservice/src/app/[workspaceSlug]/settings/(settings)/workspace/integrations/(integration)/github/GithubOrgConfig.tsx @@ -186,7 +186,7 @@ export const GithubOrgConfig: React.FC<{ name: org.login, config: { installationId: org.installationId, - login: org.login, + owner: org.login, }, }); diff --git a/apps/webservice/src/app/[workspaceSlug]/systems/[systemSlug]/deployments/TableCells.tsx b/apps/webservice/src/app/[workspaceSlug]/systems/[systemSlug]/deployments/TableCells.tsx index afeb9f7a..c16b4868 100644 --- a/apps/webservice/src/app/[workspaceSlug]/systems/[systemSlug]/deployments/TableCells.tsx +++ b/apps/webservice/src/app/[workspaceSlug]/systems/[systemSlug]/deployments/TableCells.tsx @@ -127,6 +127,8 @@ const statusColor: Record = { failure: colors.red[400], invalid_job_agent: colors.red[400], configured: colors.gray[400], + invalid_integration: colors.red[400], + external_run_not_found: colors.red[400], }; const getStatusColor = (status: string) => diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index d4aed6e7..a4c935cd 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -13,23 +13,12 @@ services: volumes: - postgres_data:/var/lib/postgresql/data - rabbitmq: - image: rabbitmq:3-management - container_name: rabbitmq - environment: - RABBITMQ_DEFAULT_USER: ctrlplane - RABBITMQ_DEFAULT_PASS: ctrlplane - ports: - - "5672:5672" - - "15672:15672" - webservice: image: ctrlplane/webservice container_name: webservice environment: AUTH_SECRET: "" DATABASE_URL: postgres://ctrlplane:ctrlplane@postgres:5432/ctrlplane - RABBITMQ_URL: amqp://ctrlplane:ctrlplane@rabbitmq:5672 depends_on: - postgres - rabbitmq @@ -41,7 +30,6 @@ services: container_name: job-policy-checker environment: POSTGRES_URL: postgres://ctrlplane:ctrlplane@postgres:5432/ctrlplane - RABBITMQ_URL: amqp://ctrlplane:ctrlplane@rabbitmq:5672 depends_on: - postgres - rabbitmq diff --git a/packages/db/drizzle/0002_stiff_tyger_tiger.sql b/packages/db/drizzle/0002_stiff_tyger_tiger.sql new file mode 100644 index 00000000..0d7591b2 --- /dev/null +++ b/packages/db/drizzle/0002_stiff_tyger_tiger.sql @@ -0,0 +1,2 @@ +ALTER TYPE "job_execution_status" ADD VALUE 'invalid_integration';--> statement-breakpoint +ALTER TYPE "job_execution_status" ADD VALUE 'external_run_not_found'; \ No newline at end of file diff --git a/packages/db/drizzle/meta/0002_snapshot.json b/packages/db/drizzle/meta/0002_snapshot.json new file mode 100644 index 00000000..e4ae55ad --- /dev/null +++ b/packages/db/drizzle/meta/0002_snapshot.json @@ -0,0 +1,2510 @@ +{ + "id": "675ebac0-2b64-40e8-ae50-f3f10d0bd7bf", + "prevId": "0f4c03e8-ef61-43b0-a394-0bebe96c52d6", + "version": "6", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "userId": { + "name": "userId", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "refresh_token": { + "name": "refresh_token", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "token_type": { + "name": "token_type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_state": { + "name": "session_state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_provider_providerAccountId_pk": { + "name": "account_provider_providerAccountId_pk", + "columns": ["provider", "providerAccountId"] + } + }, + "uniqueConstraints": {} + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "sessionToken": { + "name": "sessionToken", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "userId": { + "name": "userId", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "expires": { + "name": "expires", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.dashboard": { + "name": "dashboard", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "dashboard_workspace_id_workspace_id_fk": { + "name": "dashboard_workspace_id_workspace_id_fk", + "tableFrom": "dashboard", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.dashboard_widget": { + "name": "dashboard_widget", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "dashboard_id": { + "name": "dashboard_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "widget": { + "name": "widget", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "x": { + "name": "x", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "y": { + "name": "y", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "w": { + "name": "w", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "h": { + "name": "h", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "dashboard_widget_dashboard_id_dashboard_id_fk": { + "name": "dashboard_widget_dashboard_id_dashboard_id_fk", + "tableFrom": "dashboard_widget", + "tableTo": "dashboard", + "columnsFrom": ["dashboard_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.deployment_variable": { + "name": "deployment_variable", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "deployment_id": { + "name": "deployment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployment_variable_deployment_id_key_index": { + "name": "deployment_variable_deployment_id_key_index", + "columns": ["deployment_id", "key"], + "isUnique": true + } + }, + "foreignKeys": { + "deployment_variable_deployment_id_deployment_id_fk": { + "name": "deployment_variable_deployment_id_deployment_id_fk", + "tableFrom": "deployment_variable", + "tableTo": "deployment", + "columnsFrom": ["deployment_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.deployment_variable_value": { + "name": "deployment_variable_value", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "variable_id": { + "name": "variable_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "deployment_variable_value_variable_id_value_index": { + "name": "deployment_variable_value_variable_id_value_index", + "columns": ["variable_id", "value"], + "isUnique": true + } + }, + "foreignKeys": { + "deployment_variable_value_variable_id_deployment_variable_id_fk": { + "name": "deployment_variable_value_variable_id_deployment_variable_id_fk", + "tableFrom": "deployment_variable_value", + "tableTo": "deployment_variable", + "columnsFrom": ["variable_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.deployment_variable_value_target": { + "name": "deployment_variable_value_target", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "variable_value_id": { + "name": "variable_value_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "deployment_variable_value_target_variable_value_id_target_id_index": { + "name": "deployment_variable_value_target_variable_value_id_target_id_index", + "columns": ["variable_value_id", "target_id"], + "isUnique": true + } + }, + "foreignKeys": { + "deployment_variable_value_target_variable_value_id_deployment_variable_value_id_fk": { + "name": "deployment_variable_value_target_variable_value_id_deployment_variable_value_id_fk", + "tableFrom": "deployment_variable_value_target", + "tableTo": "deployment_variable_value", + "columnsFrom": ["variable_value_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "deployment_variable_value_target_target_id_target_id_fk": { + "name": "deployment_variable_value_target_target_id_target_id_fk", + "tableFrom": "deployment_variable_value_target", + "tableTo": "target", + "columnsFrom": ["target_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.deployment_variable_value_target_filter": { + "name": "deployment_variable_value_target_filter", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "variable_value_id": { + "name": "variable_value_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "labels": { + "name": "labels", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_variable_value_target_filter_variable_value_id_deployment_variable_value_id_fk": { + "name": "deployment_variable_value_target_filter_variable_value_id_deployment_variable_value_id_fk", + "tableFrom": "deployment_variable_value_target_filter", + "tableTo": "deployment_variable_value", + "columnsFrom": ["variable_value_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.deployment": { + "name": "deployment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "system_id": { + "name": "system_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "job_agent_id": { + "name": "job_agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "job_agent_config": { + "name": "job_agent_config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "github_config_file_id": { + "name": "github_config_file_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployment_system_id_slug_index": { + "name": "deployment_system_id_slug_index", + "columns": ["system_id", "slug"], + "isUnique": true + } + }, + "foreignKeys": { + "deployment_system_id_system_id_fk": { + "name": "deployment_system_id_system_id_fk", + "tableFrom": "deployment", + "tableTo": "system", + "columnsFrom": ["system_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "deployment_job_agent_id_job_agent_id_fk": { + "name": "deployment_job_agent_id_job_agent_id_fk", + "tableFrom": "deployment", + "tableTo": "job_agent", + "columnsFrom": ["job_agent_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "deployment_github_config_file_id_github_config_file_id_fk": { + "name": "deployment_github_config_file_id_github_config_file_id_fk", + "tableFrom": "deployment", + "tableTo": "github_config_file", + "columnsFrom": ["github_config_file_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.deployment_meta_dependency": { + "name": "deployment_meta_dependency", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "deployment_id": { + "name": "deployment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "depends_on_id": { + "name": "depends_on_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployment_meta_dependency_depends_on_id_deployment_id_index": { + "name": "deployment_meta_dependency_depends_on_id_deployment_id_index", + "columns": ["depends_on_id", "deployment_id"], + "isUnique": true + } + }, + "foreignKeys": { + "deployment_meta_dependency_deployment_id_deployment_id_fk": { + "name": "deployment_meta_dependency_deployment_id_deployment_id_fk", + "tableFrom": "deployment_meta_dependency", + "tableTo": "deployment", + "columnsFrom": ["deployment_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "deployment_meta_dependency_depends_on_id_deployment_id_fk": { + "name": "deployment_meta_dependency_depends_on_id_deployment_id_fk", + "tableFrom": "deployment_meta_dependency", + "tableTo": "deployment", + "columnsFrom": ["depends_on_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.environment": { + "name": "environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "system_id": { + "name": "system_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "policy_id": { + "name": "policy_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "target_filter": { + "name": "target_filter", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "environment_system_id_system_id_fk": { + "name": "environment_system_id_system_id_fk", + "tableFrom": "environment", + "tableTo": "system", + "columnsFrom": ["system_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "environment_policy_id_environment_policy_id_fk": { + "name": "environment_policy_id_environment_policy_id_fk", + "tableFrom": "environment", + "tableTo": "environment_policy", + "columnsFrom": ["policy_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.environment_policy": { + "name": "environment_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "system_id": { + "name": "system_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "approval_required": { + "name": "approval_required", + "type": "environment_policy_approval_requirement", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "success_status": { + "name": "success_status", + "type": "environment_policy_deployment_success_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "minimum_success": { + "name": "minimum_success", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "concurrency_type": { + "name": "concurrency_type", + "type": "concurrency_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "concurrency_limit": { + "name": "concurrency_limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "evaluate_with": { + "name": "evaluate_with", + "type": "evaluation_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "evaluate": { + "name": "evaluate", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "release_sequencing": { + "name": "release_sequencing", + "type": "release_sequencing_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'cancel'" + } + }, + "indexes": {}, + "foreignKeys": { + "environment_policy_system_id_system_id_fk": { + "name": "environment_policy_system_id_system_id_fk", + "tableFrom": "environment_policy", + "tableTo": "system", + "columnsFrom": ["system_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.environment_policy_approval": { + "name": "environment_policy_approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "policy_id": { + "name": "policy_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "release_id": { + "name": "release_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "approval_status_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + } + }, + "indexes": { + "environment_policy_approval_policy_id_release_id_index": { + "name": "environment_policy_approval_policy_id_release_id_index", + "columns": ["policy_id", "release_id"], + "isUnique": true + } + }, + "foreignKeys": { + "environment_policy_approval_policy_id_environment_policy_id_fk": { + "name": "environment_policy_approval_policy_id_environment_policy_id_fk", + "tableFrom": "environment_policy_approval", + "tableTo": "environment_policy", + "columnsFrom": ["policy_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "environment_policy_approval_release_id_release_id_fk": { + "name": "environment_policy_approval_release_id_release_id_fk", + "tableFrom": "environment_policy_approval", + "tableTo": "release", + "columnsFrom": ["release_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.environment_policy_deployment": { + "name": "environment_policy_deployment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "policy_id": { + "name": "policy_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "environment_policy_deployment_policy_id_environment_id_index": { + "name": "environment_policy_deployment_policy_id_environment_id_index", + "columns": ["policy_id", "environment_id"], + "isUnique": true + } + }, + "foreignKeys": { + "environment_policy_deployment_policy_id_environment_policy_id_fk": { + "name": "environment_policy_deployment_policy_id_environment_policy_id_fk", + "tableFrom": "environment_policy_deployment", + "tableTo": "environment_policy", + "columnsFrom": ["policy_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "environment_policy_deployment_environment_id_environment_id_fk": { + "name": "environment_policy_deployment_environment_id_environment_id_fk", + "tableFrom": "environment_policy_deployment", + "tableTo": "environment", + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.environment_policy_release_window": { + "name": "environment_policy_release_window", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "policy_id": { + "name": "policy_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "start_time": { + "name": "start_time", + "type": "timestamp (0) with time zone", + "primaryKey": false, + "notNull": true + }, + "end_time": { + "name": "end_time", + "type": "timestamp (0) with time zone", + "primaryKey": false, + "notNull": true + }, + "recurrence": { + "name": "recurrence", + "type": "recurrence_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "environment_policy_release_window_policy_id_environment_policy_id_fk": { + "name": "environment_policy_release_window_policy_id_environment_policy_id_fk", + "tableFrom": "environment_policy_release_window", + "tableTo": "environment_policy", + "columnsFrom": ["policy_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.github_config_file": { + "name": "github_config_file", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "repository_name": { + "name": "repository_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "unique_organization_repository_path": { + "name": "unique_organization_repository_path", + "columns": ["organization_id", "repository_name", "path"], + "isUnique": true + } + }, + "foreignKeys": { + "github_config_file_organization_id_github_organization_id_fk": { + "name": "github_config_file_organization_id_github_organization_id_fk", + "tableFrom": "github_config_file", + "tableTo": "github_organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "github_config_file_workspace_id_workspace_id_fk": { + "name": "github_config_file_workspace_id_workspace_id_fk", + "tableFrom": "github_config_file", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.github_organization": { + "name": "github_organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "installation_id": { + "name": "installation_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "organization_name": { + "name": "organization_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "added_by_user_id": { + "name": "added_by_user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "connected": { + "name": "connected", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'main'" + } + }, + "indexes": {}, + "foreignKeys": { + "github_organization_added_by_user_id_user_id_fk": { + "name": "github_organization_added_by_user_id_user_id_fk", + "tableFrom": "github_organization", + "tableTo": "user", + "columnsFrom": ["added_by_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "github_organization_workspace_id_workspace_id_fk": { + "name": "github_organization_workspace_id_workspace_id_fk", + "tableFrom": "github_organization", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.github_user": { + "name": "github_user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "github_user_id": { + "name": "github_user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "github_username": { + "name": "github_username", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "github_user_user_id_user_id_fk": { + "name": "github_user_user_id_user_id_fk", + "tableFrom": "github_user", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.target": { + "name": "target", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "labels": { + "name": "labels", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "locked_at": { + "name": "locked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "target_identifier_workspace_id_index": { + "name": "target_identifier_workspace_id_index", + "columns": ["identifier", "workspace_id"], + "isUnique": true + } + }, + "foreignKeys": { + "target_provider_id_target_provider_id_fk": { + "name": "target_provider_id_target_provider_id_fk", + "tableFrom": "target", + "tableTo": "target_provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "target_workspace_id_workspace_id_fk": { + "name": "target_workspace_id_workspace_id_fk", + "tableFrom": "target", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "target_name_unique": { + "name": "target_name_unique", + "nullsNotDistinct": false, + "columns": ["name"] + } + } + }, + "public.target_schema": { + "name": "target_schema", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "json_schema": { + "name": "json_schema", + "type": "json", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "target_schema_version_kind_workspace_id_index": { + "name": "target_schema_version_kind_workspace_id_index", + "columns": ["version", "kind", "workspace_id"], + "isUnique": true + } + }, + "foreignKeys": { + "target_schema_workspace_id_workspace_id_fk": { + "name": "target_schema_workspace_id_workspace_id_fk", + "tableFrom": "target_schema", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.target_provider": { + "name": "target_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "target_provider_workspace_id_name_index": { + "name": "target_provider_workspace_id_name_index", + "columns": ["workspace_id", "name"], + "isUnique": true + } + }, + "foreignKeys": { + "target_provider_workspace_id_workspace_id_fk": { + "name": "target_provider_workspace_id_workspace_id_fk", + "tableFrom": "target_provider", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.target_provider_google": { + "name": "target_provider_google", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "target_provider_id": { + "name": "target_provider_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "project_ids": { + "name": "project_ids", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "target_provider_google_target_provider_id_target_provider_id_fk": { + "name": "target_provider_google_target_provider_id_target_provider_id_fk", + "tableFrom": "target_provider_google", + "tableTo": "target_provider", + "columnsFrom": ["target_provider_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.release": { + "name": "release", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "deployment_id": { + "name": "deployment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "release_deployment_id_version_index": { + "name": "release_deployment_id_version_index", + "columns": ["deployment_id", "version"], + "isUnique": true + } + }, + "foreignKeys": { + "release_deployment_id_deployment_id_fk": { + "name": "release_deployment_id_deployment_id_fk", + "tableFrom": "release", + "tableTo": "deployment", + "columnsFrom": ["deployment_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.release_dependency": { + "name": "release_dependency", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "release_id": { + "name": "release_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "target_label_group_id": { + "name": "target_label_group_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "rule_type": { + "name": "rule_type", + "type": "release_dependency_rule_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "rule": { + "name": "rule", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "release_dependency_release_id_deployment_id_target_label_group_id_index": { + "name": "release_dependency_release_id_deployment_id_target_label_group_id_index", + "columns": ["release_id", "deployment_id", "target_label_group_id"], + "isUnique": true + } + }, + "foreignKeys": { + "release_dependency_release_id_release_id_fk": { + "name": "release_dependency_release_id_release_id_fk", + "tableFrom": "release_dependency", + "tableTo": "release", + "columnsFrom": ["release_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "release_dependency_deployment_id_deployment_id_fk": { + "name": "release_dependency_deployment_id_deployment_id_fk", + "tableFrom": "release_dependency", + "tableTo": "deployment", + "columnsFrom": ["deployment_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "release_dependency_target_label_group_id_target_label_group_id_fk": { + "name": "release_dependency_target_label_group_id_target_label_group_id_fk", + "tableFrom": "release_dependency", + "tableTo": "target_label_group", + "columnsFrom": ["target_label_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.system": { + "name": "system", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "system_workspace_id_slug_index": { + "name": "system_workspace_id_slug_index", + "columns": ["workspace_id", "slug"], + "isUnique": true + } + }, + "foreignKeys": { + "system_workspace_id_workspace_id_fk": { + "name": "system_workspace_id_workspace_id_fk", + "tableFrom": "system", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.runbook": { + "name": "runbook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "job_agent_id": { + "name": "job_agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "job_agent_config": { + "name": "job_agent_config", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "runbook_job_agent_id_job_agent_id_fk": { + "name": "runbook_job_agent_id_job_agent_id_fk", + "tableFrom": "runbook", + "tableTo": "job_agent", + "columnsFrom": ["job_agent_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.team": { + "name": "team", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "team_workspace_id_workspace_id_fk": { + "name": "team_workspace_id_workspace_id_fk", + "tableFrom": "team", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.team_member": { + "name": "team_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "team_id": { + "name": "team_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "team_member_team_id_user_id_index": { + "name": "team_member_team_id_user_id_index", + "columns": ["team_id", "user_id"], + "isUnique": true + } + }, + "foreignKeys": { + "team_member_team_id_team_id_fk": { + "name": "team_member_team_id_team_id_fk", + "tableFrom": "team_member", + "tableTo": "team", + "columnsFrom": ["team_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "team_member_user_id_user_id_fk": { + "name": "team_member_user_id_user_id_fk", + "tableFrom": "team_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.job_agent": { + "name": "job_agent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + } + }, + "indexes": { + "job_agent_workspace_id_name_index": { + "name": "job_agent_workspace_id_name_index", + "columns": ["workspace_id", "name"], + "isUnique": true + } + }, + "foreignKeys": { + "job_agent_workspace_id_workspace_id_fk": { + "name": "job_agent_workspace_id_workspace_id_fk", + "tableFrom": "job_agent", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.job_config": { + "name": "job_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "type": { + "name": "type", + "type": "job_config_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "caused_by_id": { + "name": "caused_by_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "release_id": { + "name": "release_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "environment_id": { + "name": "environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "runbook_id": { + "name": "runbook_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "job_config_caused_by_id_user_id_fk": { + "name": "job_config_caused_by_id_user_id_fk", + "tableFrom": "job_config", + "tableTo": "user", + "columnsFrom": ["caused_by_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "job_config_release_id_release_id_fk": { + "name": "job_config_release_id_release_id_fk", + "tableFrom": "job_config", + "tableTo": "release", + "columnsFrom": ["release_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "job_config_target_id_target_id_fk": { + "name": "job_config_target_id_target_id_fk", + "tableFrom": "job_config", + "tableTo": "target", + "columnsFrom": ["target_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "job_config_environment_id_environment_id_fk": { + "name": "job_config_environment_id_environment_id_fk", + "tableFrom": "job_config", + "tableTo": "environment", + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "job_config_runbook_id_runbook_id_fk": { + "name": "job_config_runbook_id_runbook_id_fk", + "tableFrom": "job_config", + "tableTo": "runbook", + "columnsFrom": ["runbook_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.job_execution": { + "name": "job_execution", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_config_id": { + "name": "job_config_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "job_agent_id": { + "name": "job_agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "job_agent_config": { + "name": "job_agent_config", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "external_run_id": { + "name": "external_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "job_execution_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "job_execution_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'policy_passing'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "job_execution_job_config_id_job_config_id_fk": { + "name": "job_execution_job_config_id_job_config_id_fk", + "tableFrom": "job_execution", + "tableTo": "job_config", + "columnsFrom": ["job_config_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "job_execution_job_agent_id_job_agent_id_fk": { + "name": "job_execution_job_agent_id_job_agent_id_fk", + "tableFrom": "job_execution", + "tableTo": "job_agent", + "columnsFrom": ["job_agent_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.workspace": { + "name": "workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "google_service_account_email": { + "name": "google_service_account_email", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_slug_unique": { + "name": "workspace_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + } + }, + "public.workspace_member": { + "name": "workspace_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "workspace_member_workspace_id_user_id_index": { + "name": "workspace_member_workspace_id_user_id_index", + "columns": ["workspace_id", "user_id"], + "isUnique": true + } + }, + "foreignKeys": { + "workspace_member_workspace_id_workspace_id_fk": { + "name": "workspace_member_workspace_id_workspace_id_fk", + "tableFrom": "workspace_member", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_member_user_id_user_id_fk": { + "name": "workspace_member_user_id_user_id_fk", + "tableFrom": "workspace_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.value": { + "name": "value", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "value_set_id": { + "name": "value_set_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "value_value_set_id_key_value_index": { + "name": "value_value_set_id_key_value_index", + "columns": ["value_set_id", "key", "value"], + "isUnique": true + } + }, + "foreignKeys": { + "value_value_set_id_value_set_id_fk": { + "name": "value_value_set_id_value_set_id_fk", + "tableFrom": "value", + "tableTo": "value_set", + "columnsFrom": ["value_set_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.value_set": { + "name": "value_set", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "system_id": { + "name": "system_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "value_set_system_id_system_id_fk": { + "name": "value_set_system_id_system_id_fk", + "tableFrom": "value_set", + "tableTo": "system", + "columnsFrom": ["system_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.workspace_invite_link": { + "name": "workspace_invite_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_member_id": { + "name": "workspace_member_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_invite_link_workspace_member_id_workspace_member_id_fk": { + "name": "workspace_invite_link_workspace_member_id_workspace_member_id_fk", + "tableFrom": "workspace_invite_link", + "tableTo": "workspace_member", + "columnsFrom": ["workspace_member_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_invite_link_token_unique": { + "name": "workspace_invite_link_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + } + }, + "public.target_label_group": { + "name": "target_label_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "keys": { + "name": "keys", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "target_label_group_workspace_id_workspace_id_fk": { + "name": "target_label_group_workspace_id_workspace_id_fk", + "tableFrom": "target_label_group", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.runbook_variable": { + "name": "runbook_variable", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "runbook_id": { + "name": "runbook_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "required": { + "name": "required", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "default_value": { + "name": "default_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "runbook_variable_runbook_id_runbook_id_fk": { + "name": "runbook_variable_runbook_id_runbook_id_fk", + "tableFrom": "runbook_variable", + "tableTo": "runbook", + "columnsFrom": ["runbook_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": { + "public.environment_policy_approval_requirement": { + "name": "environment_policy_approval_requirement", + "schema": "public", + "values": ["manual", "automatic"] + }, + "public.approval_status_type": { + "name": "approval_status_type", + "schema": "public", + "values": ["pending", "approved", "rejected"] + }, + "public.concurrency_type": { + "name": "concurrency_type", + "schema": "public", + "values": ["all", "some"] + }, + "public.environment_policy_deployment_success_type": { + "name": "environment_policy_deployment_success_type", + "schema": "public", + "values": ["all", "some", "optional"] + }, + "public.evaluation_type": { + "name": "evaluation_type", + "schema": "public", + "values": ["semver", "regex", "none"] + }, + "public.recurrence_type": { + "name": "recurrence_type", + "schema": "public", + "values": ["hourly", "daily", "weekly", "monthly"] + }, + "public.release_sequencing_type": { + "name": "release_sequencing_type", + "schema": "public", + "values": ["wait", "cancel"] + }, + "public.release_dependency_rule_type": { + "name": "release_dependency_rule_type", + "schema": "public", + "values": ["regex", "semver"] + }, + "public.job_config_type": { + "name": "job_config_type", + "schema": "public", + "values": [ + "new_release", + "new_target", + "target_changed", + "api", + "redeploy", + "runbook" + ] + }, + "public.job_execution_reason": { + "name": "job_execution_reason", + "schema": "public", + "values": [ + "policy_passing", + "policy_override", + "env_policy_override", + "config_policy_override" + ] + }, + "public.job_execution_status": { + "name": "job_execution_status", + "schema": "public", + "values": [ + "completed", + "cancelled", + "skipped", + "in_progress", + "action_required", + "pending", + "failure", + "invalid_job_agent", + "invalid_integration", + "external_run_not_found" + ] + } + }, + "schemas": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index ef6e2e59..f4b8c996 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1724814291791, "tag": "0001_tranquil_james_howlett", "breakpoints": true + }, + { + "idx": 2, + "version": "6", + "when": 1724984177867, + "tag": "0002_stiff_tyger_tiger", + "breakpoints": true } ] } diff --git a/packages/db/src/schema/job-execution.ts b/packages/db/src/schema/job-execution.ts index 615fb976..1e8ffbaa 100644 --- a/packages/db/src/schema/job-execution.ts +++ b/packages/db/src/schema/job-execution.ts @@ -51,6 +51,8 @@ export const jobExecutionStatus = pgEnum("job_execution_status", [ "pending", "failure", "invalid_job_agent", + "invalid_integration", + "external_run_not_found", ]); export const jobExecutionReason = pgEnum("job_execution_reason", [ diff --git a/packages/job-dispatch/package.json b/packages/job-dispatch/package.json index 45dfac62..f38aa8cb 100644 --- a/packages/job-dispatch/package.json +++ b/packages/job-dispatch/package.json @@ -25,8 +25,11 @@ }, "dependencies": { "@ctrlplane/db": "workspace:*", + "@ctrlplane/validators": "workspace:*", "@t3-oss/env-core": "^0.10.1", + "bullmq": "^5.12.10", "date-fns": "^3.6.0", + "ioredis": "^5.4.1", "lodash": "^4.17.21", "ms": "^2.1.3", "murmurhash": "^2.0.1", @@ -39,7 +42,6 @@ "@ctrlplane/eslint-config": "workspace:*", "@ctrlplane/prettier-config": "workspace:*", "@ctrlplane/tsconfig": "workspace:*", - "@types/amqplib": "^0.10.5", "@types/lodash": "^4.17.5", "@types/semver": "^7.5.8", "dotenv-cli": "^7.4.2", diff --git a/packages/job-dispatch/src/config.ts b/packages/job-dispatch/src/config.ts index c23990ca..fa215069 100644 --- a/packages/job-dispatch/src/config.ts +++ b/packages/job-dispatch/src/config.ts @@ -4,8 +4,7 @@ import { z } from "zod"; export const env = createEnv({ server: { - AMQP_URL: z.string().optional(), - AMQP_QUEUE: z.string().default("job_configs"), + REDIS_URL: z.string(), }, runtimeEnv: process.env, emptyStringAsUndefined: true, diff --git a/packages/job-dispatch/src/job-dispatch.ts b/packages/job-dispatch/src/job-dispatch.ts index a540ab99..ce317e70 100644 --- a/packages/job-dispatch/src/job-dispatch.ts +++ b/packages/job-dispatch/src/job-dispatch.ts @@ -4,6 +4,7 @@ import _ from "lodash"; import type { JobExecutionReason } from "./job-execution.js"; import { createJobExecutions } from "./job-execution.js"; +import { dispatchJobExecutionsQueue } from "./queue.js"; export type DispatchFilterFunc = ( db: Tx, @@ -53,6 +54,15 @@ class DispatchBuilder { for (const func of this._then) await func(this.db, t); + await dispatchJobExecutionsQueue.addBulk( + wfs.map((wf) => ({ + name: wf.id, + data: { + jobExecutionId: wf.id, + }, + })), + ); + return wfs; } } diff --git a/packages/job-dispatch/src/job-execution.ts b/packages/job-dispatch/src/job-execution.ts index a3eb1ce2..44f0829d 100644 --- a/packages/job-dispatch/src/job-execution.ts +++ b/packages/job-dispatch/src/job-execution.ts @@ -81,12 +81,7 @@ export const createJobExecutions = async ( if (insertJobExecutions.length === 0) return []; - const jobExecutions = await db - .insert(jobExecution) - .values(insertJobExecutions) - .returning(); - - return jobExecutions; + return db.insert(jobExecution).values(insertJobExecutions).returning(); }; export const onJobExecutionStatusChange = async (je: JobExecution) => { diff --git a/packages/job-dispatch/src/queue.ts b/packages/job-dispatch/src/queue.ts new file mode 100644 index 00000000..4f88edbf --- /dev/null +++ b/packages/job-dispatch/src/queue.ts @@ -0,0 +1,12 @@ +import { Queue } from "bullmq"; +import IORedis from "ioredis"; + +import { Channel } from "@ctrlplane/validators/events"; + +import { env } from "./config.js"; + +const connection = new IORedis(env.REDIS_URL, { maxRetriesPerRequest: null }); + +export const dispatchJobExecutionsQueue = new Queue(Channel.JobExecutionSync, { + connection, +}); diff --git a/packages/job-dispatch/src/queue/rabbitmq.ts b/packages/job-dispatch/src/queue/rabbitmq.ts deleted file mode 100644 index b5e44d49..00000000 --- a/packages/job-dispatch/src/queue/rabbitmq.ts +++ /dev/null @@ -1,56 +0,0 @@ -// import type { JobExecution } from "@ctrlplane/db/schema"; -// import type { Channel, Connection, ConsumeMessage } from "amqplib"; -// import amqp from "amqplib"; -// import ms from "ms"; - -// import type { JobQueue } from "./queue"; - -// /** -// * RabbitMQ implementation of the JobQueue interface. -// * -// * @deprecated This implementation is not yet complete. -// */ -// export class RabbitMQService implements JobQueue { -// private connection: Connection; -// private channel: Channel; -// private messageStore: Map; - -// constructor() { -// this.messageStore = new Map(); -// } - -// private async init() { -// this.connection = await amqp.connect("amqp://localhost"); -// this.channel = await this.connection.createChannel(); -// } - -// private async assertQueue(agentId: string) { -// const queueName = `agent:${agentId}:queue`; -// await this.channel.assertQueue(queueName, { -// durable: true, -// arguments: { -// "x-message-ttl": ms("5m"), -// "x-dead-letter-exchange": "", // Default exchange -// "x-dead-letter-routing-key": queueName, // Requeue to the same queue -// }, -// }); -// return queueName; -// } - -// enqueue(agentId: string, jobs: JobExecution[]): void { -// // Implementation -// } - -// async acknowledge(agentId: string, jobExcutionId: string): Promise { - -// } - -// async next(agentId: string): Promise { -// const queueName = await this.assertQueue(agentId); -// const message = await this.channel.get(queueName, { noAck: false }); -// if (message == false) { -// return []; -// } -// message.fields.deliveryTag -// } -// } diff --git a/packages/job-dispatch/src/queue/redis.ts b/packages/job-dispatch/src/queue/redis.ts deleted file mode 100644 index d05301a0..00000000 --- a/packages/job-dispatch/src/queue/redis.ts +++ /dev/null @@ -1,29 +0,0 @@ -// import type { JobExecution } from "@ctrlplane/db/schema"; -// import type { RedisClientType } from "redis"; -// import { createClient } from "redis"; - -// import type { JobQueue } from "./queue"; - -// export class RedisService implements JobQueue { -// private client: RedisClientType; - -// constructor() { -// this.client = createClient(); -// this.client.connect(); -// } - -// async enqueue(agentId: string, jobs: JobExecution[]): Promise { -// const queueKey = `agent:${agentId}:queue`; -// const pipeline = this.client.multi(); -// for (const job of jobs) pipeline.lPush(queueKey, JSON.stringify(job)); -// await pipeline.exec(); -// } - -// acknowledge(agentId: string, jobExcutionId: string): Promise { -// throw new Error("Method not implemented."); -// } - -// next(agentId: string): Promise { -// throw new Error("Method not implemented."); -// } -// } diff --git a/packages/validators/package.json b/packages/validators/package.json index 030f6832..200b6ea4 100644 --- a/packages/validators/package.json +++ b/packages/validators/package.json @@ -15,6 +15,14 @@ "./events": { "types": "./src/events/index.ts", "default": "./dist/events/index.js" + }, + "./jobs": { + "types": "./src/jobs/index.ts", + "default": "./dist/jobs/index.js" + }, + "./github": { + "types": "./src/github/index.ts", + "default": "./dist/github/index.js" } }, "license": "MIT", diff --git a/packages/validators/src/events/index.ts b/packages/validators/src/events/index.ts index ef8e4c1c..1785a7c3 100644 --- a/packages/validators/src/events/index.ts +++ b/packages/validators/src/events/index.ts @@ -2,15 +2,19 @@ import { z } from "zod"; export enum Channel { JobExecutionSync = "job-execution-sync", - DispatchJob = "dispatch-job", + DispatchJobExecution = "dispatch-job-execution", TargetScan = "target-scan", } export const targetScanEvent = z.object({ targetProviderId: z.string() }); export type TargetScanEvent = z.infer; -export const dispatchJobEvent = z.object({ jobConfigId: z.string() }); -export type DispatchJobEvent = z.infer; +export const dispatchJobExecutionEvent = z.object({ + jobExecutionId: z.string(), +}); +export type DispatchJobExecutionEvent = z.infer< + typeof dispatchJobExecutionEvent +>; export const jobExecutionSyncEvent = z.object({ jobExecutionId: z.string() }); export type JobExecutionSyncEvent = z.infer; diff --git a/packages/validators/src/github/index.ts b/packages/validators/src/github/index.ts new file mode 100644 index 00000000..6441efa2 --- /dev/null +++ b/packages/validators/src/github/index.ts @@ -0,0 +1,8 @@ +import { z } from "zod"; + +export const configSchema = z.object({ + installationId: z.number(), + owner: z.string().min(1), + repo: z.string().min(1), + workflowId: z.number(), +}); diff --git a/packages/validators/src/jobs/index.ts b/packages/validators/src/jobs/index.ts new file mode 100644 index 00000000..43fe7a47 --- /dev/null +++ b/packages/validators/src/jobs/index.ts @@ -0,0 +1,16 @@ +export enum JobAgentType { + GithubApp = "github-app", +} + +export enum JobExecutionStatus { + Completed = "completed", + Cancelled = "cancelled", + Skipped = "skipped", + InProgress = "in_progress", + ActionRequired = "action_required", + Pending = "pending", + Failure = "failure", + InvalidJobAgent = "invalid_job_agent", + InvalidIntegration = "invalid_integration", + ExternalRunNotFound = "external_run_not_found", +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5bacdd88..89183dde 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -58,70 +58,6 @@ importers: specifier: ^5.4.5 version: 5.5.4 - agents/github-app: - dependencies: - '@ctrlplane/api': - specifier: workspace:* - version: link:../../packages/api - '@ctrlplane/db': - specifier: workspace:* - version: link:../../packages/db - '@octokit/auth-app': - specifier: ^7.1.0 - version: 7.1.0 - '@octokit/rest': - specifier: ^20.1.1 - version: 20.1.1 - '@t3-oss/env-core': - specifier: ^0.10.1 - version: 0.10.1(typescript@5.5.4)(zod@3.23.8) - '@trpc/client': - specifier: 11.0.0-rc.364 - version: 11.0.0-rc.364(@trpc/server@11.0.0-rc.364) - cron: - specifier: ^3.1.7 - version: 3.1.7 - dotenv: - specifier: ^16.4.5 - version: 16.4.5 - lodash: - specifier: ^4.17.21 - version: 4.17.21 - p-retry: - specifier: ^6.2.0 - version: 6.2.0 - devDependencies: - '@ctrlplane/eslint-config': - specifier: workspace:^ - version: link:../../tooling/eslint - '@ctrlplane/prettier-config': - specifier: workspace:^ - version: link:../../tooling/prettier - '@ctrlplane/tailwind-config': - specifier: workspace:* - version: link:../../tooling/tailwind - '@ctrlplane/tsconfig': - specifier: workspace:* - version: link:../../tooling/typescript - '@types/lodash': - specifier: ^4.17.5 - version: 4.17.7 - '@types/node': - specifier: ^20.12.9 - version: 20.16.1 - eslint: - specifier: 'catalog:' - version: 9.9.1(jiti@1.21.6) - prettier: - specifier: 'catalog:' - version: 3.3.3 - tsx: - specifier: ^4.11.0 - version: 4.18.0 - typescript: - specifier: ^5.4.5 - version: 5.5.4 - apps/docs: dependencies: '@ctrlplane/ui': @@ -203,6 +139,12 @@ importers: '@kubernetes/client-node': specifier: ^0.21.0 version: 0.21.0 + '@octokit/auth-app': + specifier: ^7.1.0 + version: 7.1.0 + '@octokit/rest': + specifier: ^20.1.1 + version: 20.1.1 '@t3-oss/env-core': specifier: ^0.10.1 version: 0.10.1(typescript@5.5.4)(zod@3.23.8) @@ -227,6 +169,9 @@ importers: ms: specifier: ^2.1.3 version: 2.1.3 + p-retry: + specifier: ^6.2.0 + version: 6.2.0 semver: specifier: ^7.6.2 version: 7.6.3 @@ -246,6 +191,9 @@ importers: '@types/lodash': specifier: ^4.17.5 version: 4.17.7 + '@types/ms': + specifier: ^0.7.34 + version: 0.7.34 '@types/node': specifier: ^20.12.9 version: 20.16.1 @@ -273,6 +221,9 @@ importers: '@ctrlplane/job-dispatch': specifier: workspace:* version: link:../../packages/job-dispatch + '@ctrlplane/validators': + specifier: workspace:* + version: link:../../packages/validators '@t3-oss/env-core': specifier: ^0.10.1 version: 0.10.1(typescript@5.5.4)(zod@3.23.8) @@ -391,9 +342,6 @@ importers: '@xterm/xterm': specifier: ^5.5.0 version: 5.5.0 - amqplib: - specifier: ^0.10.4 - version: 0.10.4 change-case: specifier: ^5.4.4 version: 5.4.4 @@ -503,9 +451,6 @@ importers: '@ctrlplane/tsconfig': specifier: workspace:* version: link:../../tooling/typescript - '@types/amqplib': - specifier: ^0.10.5 - version: 0.10.5 '@types/dagre': specifier: ^0.7.52 version: 0.7.52 @@ -831,12 +776,21 @@ importers: '@ctrlplane/db': specifier: workspace:* version: link:../db + '@ctrlplane/validators': + specifier: workspace:* + version: link:../validators '@t3-oss/env-core': specifier: ^0.10.1 version: 0.10.1(typescript@5.5.4)(zod@3.23.8) + bullmq: + specifier: ^5.12.10 + version: 5.12.10 date-fns: specifier: ^3.6.0 version: 3.6.0 + ioredis: + specifier: ^5.4.1 + version: 5.4.1 lodash: specifier: ^4.17.21 version: 4.17.21 @@ -868,9 +822,6 @@ importers: '@ctrlplane/tsconfig': specifier: workspace:* version: link:../../tooling/typescript - '@types/amqplib': - specifier: ^0.10.5 - version: 0.10.5 '@types/lodash': specifier: ^4.17.5 version: 4.17.7 @@ -1294,10 +1245,6 @@ importers: packages: - '@acuminous/bitsyntax@0.1.2': - resolution: {integrity: sha512-29lUK80d1muEQqiUsSo+3A0yP6CdspgC95EnKBMi22Xlwt79i/En4Vr67+cXhU+cZjbti3TgGGC5wy1stIywVQ==} - engines: {node: '>=0.8'} - '@alloc/quick-lru@5.2.0': resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} @@ -3845,9 +3792,6 @@ packages: '@types/acorn@4.0.6': resolution: {integrity: sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==} - '@types/amqplib@0.10.5': - resolution: {integrity: sha512-/cSykxROY7BWwDoi4Y4/jLAuZTshZxd8Ey1QYa/VaXriMotBDoou7V/twJiOSHzU6t1Kp1AHAUXGCgqq+6DNeg==} - '@types/body-parser@1.19.5': resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} @@ -4304,10 +4248,6 @@ packages: ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} - amqplib@0.10.4: - resolution: {integrity: sha512-DMZ4eCEjAVdX1II2TfIUpJhfKAuoCeDIo/YyETbfAqehHTXxxs7WOOd+N1Xxr4cKhx12y23zk8/os98FxlZHrw==} - engines: {node: '>=10'} - ansi-escapes@4.3.2: resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} engines: {node: '>=8'} @@ -4504,9 +4444,6 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - buffer-more-ints@1.0.0: - resolution: {integrity: sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==} - buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} @@ -4789,9 +4726,6 @@ packages: core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} - core-util-is@1.0.3: - resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} - cors@2.8.5: resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==} engines: {node: '>= 0.10'} @@ -6320,9 +6254,6 @@ packages: resolution: {integrity: sha512-ZhMwEosbFJkA0YhFnNDgTM4ZxDRsS6HqTo7qsZM08fehyRYIYa0yHu5R6mgo1n/8MgaPBXiPimPD77baVFYg+A==} engines: {node: '>=12.13'} - isarray@0.0.1: - resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==} - isarray@2.0.5: resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} @@ -7551,9 +7482,6 @@ packages: resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==} engines: {node: '>=0.6'} - querystringify@2.2.0: - resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} - queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -7711,9 +7639,6 @@ packages: read-cache@1.0.0: resolution: {integrity: sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==} - readable-stream@1.1.14: - resolution: {integrity: sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==} - readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} @@ -7813,9 +7738,6 @@ packages: resolution: {integrity: sha512-X34iHADNbNDfr6OTStIAHWSAvvKQRYgLO6duASaVf7J2VA3lvmNYboAHOuLC2huav1IwgZJtyEcJCKVzFxOSMQ==} engines: {node: '>=8.6.0'} - requires-port@1.0.0: - resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} - resize-observer-polyfill@1.5.1: resolution: {integrity: sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==} @@ -7902,9 +7824,6 @@ packages: resolution: {integrity: sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==} engines: {node: '>=0.4'} - safe-buffer@5.1.2: - resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} - safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -8156,9 +8075,6 @@ packages: resolution: {integrity: sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg==} engines: {node: '>= 0.4'} - string_decoder@0.10.31: - resolution: {integrity: sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==} - string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} @@ -8640,9 +8556,6 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} - url-parse@1.5.10: - resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} - url-template@2.0.8: resolution: {integrity: sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==} @@ -8908,14 +8821,6 @@ packages: snapshots: - '@acuminous/bitsyntax@0.1.2': - dependencies: - buffer-more-ints: 1.0.0 - debug: 4.3.6 - safe-buffer: 5.1.2 - transitivePeerDependencies: - - supports-color - '@alloc/quick-lru@5.2.0': {} '@ampproject/remapping@2.3.0': @@ -11763,10 +11668,6 @@ snapshots: dependencies: '@types/estree': 1.0.5 - '@types/amqplib@0.10.5': - dependencies: - '@types/node': 20.16.1 - '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.38 @@ -12304,15 +12205,6 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 - amqplib@0.10.4: - dependencies: - '@acuminous/bitsyntax': 0.1.2 - buffer-more-ints: 1.0.0 - readable-stream: 1.1.14 - url-parse: 1.5.10 - transitivePeerDependencies: - - supports-color - ansi-escapes@4.3.2: dependencies: type-fest: 0.21.3 @@ -12536,8 +12428,6 @@ snapshots: buffer-from@1.1.2: {} - buffer-more-ints@1.0.0: {} - buffer@5.7.1: dependencies: base64-js: 1.5.1 @@ -12831,8 +12721,6 @@ snapshots: core-util-is@1.0.2: {} - core-util-is@1.0.3: {} - cors@2.8.5: dependencies: object-assign: 4.1.1 @@ -14742,8 +14630,6 @@ snapshots: is-what@4.1.16: {} - isarray@0.0.1: {} - isarray@2.0.5: {} isbinaryfile@4.0.10: {} @@ -16241,8 +16127,6 @@ snapshots: qs@6.5.3: {} - querystringify@2.2.0: {} - queue-microtask@1.2.3: {} randomcolor@0.6.2: {} @@ -16488,13 +16372,6 @@ snapshots: dependencies: pify: 2.3.0 - readable-stream@1.1.14: - dependencies: - core-util-is: 1.0.3 - inherits: 2.0.4 - isarray: 0.0.1 - string_decoder: 0.10.31 - readable-stream@3.6.2: dependencies: inherits: 2.0.4 @@ -16672,8 +16549,6 @@ snapshots: transitivePeerDependencies: - supports-color - requires-port@1.0.0: {} - resize-observer-polyfill@1.5.1: {} resolve-from@4.0.0: {} @@ -16777,8 +16652,6 @@ snapshots: has-symbols: 1.0.3 isarray: 2.0.5 - safe-buffer@5.1.2: {} - safe-buffer@5.2.1: {} safe-regex-test@1.0.3: @@ -17066,8 +16939,6 @@ snapshots: define-properties: 1.2.1 es-object-atoms: 1.0.0 - string_decoder@0.10.31: {} - string_decoder@1.3.0: dependencies: safe-buffer: 5.2.1 @@ -17603,11 +17474,6 @@ snapshots: dependencies: punycode: 2.3.1 - url-parse@1.5.10: - dependencies: - querystringify: 2.2.0 - requires-port: 1.0.0 - url-template@2.0.8: {} use-callback-ref@1.3.2(@types/react@18.3.4)(react@18.3.1): diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index d2dbc6e0..b1004821 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -2,7 +2,6 @@ packages: - apps/* - packages/* - tooling/* - - agents/* - providers/* catalog: