From e0e5bd571b0a0c761c8d0987ba2021e0992fa2ab Mon Sep 17 00:00:00 2001 From: michaeljguarino Date: Fri, 12 Jan 2024 11:47:24 -0500 Subject: [PATCH] Dry Run + On-Demand PR generation data models (#606) --- assets/src/generated/graphql.ts | 28 ++++- lib/console/commands/plural.ex | 2 + lib/console/deployments/git.ex | 118 +++++++++++++++++- lib/console/deployments/pr/config.ex | 23 ++++ lib/console/deployments/pr/dispatcher.ex | 29 +++++ lib/console/deployments/pr/git.ex | 46 +++++++ lib/console/deployments/pr/impl/github.ex | 5 + lib/console/deployments/pr/impl/gitlab.ex | 5 + lib/console/deployments/pr/impl/pass.ex | 5 + lib/console/graphql/deployments/cluster.ex | 6 +- lib/console/graphql/deployments/service.ex | 20 +++ lib/console/graphql/resolvers/deployments.ex | 2 + lib/console/schema/component_content.ex | 23 ++++ lib/console/schema/pr_automation.ex | 49 ++++++++ lib/console/schema/scm_connection.ex | 30 +++++ lib/console/schema/scm_webhook.ex | 25 ++++ lib/console/schema/service.ex | 5 +- lib/console/schema/service_component.ex | 4 +- mix.exs | 1 + mix.lock | 1 + .../20240111013712_add_pr_automation.exs | 63 ++++++++++ schema/schema.graphql | 35 +++++- .../queries/deployment_queries_test.exs | 31 +++++ 23 files changed, 544 insertions(+), 12 deletions(-) create mode 100644 lib/console/deployments/pr/config.ex create mode 100644 lib/console/deployments/pr/dispatcher.ex create mode 100644 lib/console/deployments/pr/git.ex create mode 100644 lib/console/deployments/pr/impl/github.ex create mode 100644 lib/console/deployments/pr/impl/gitlab.ex create mode 100644 lib/console/deployments/pr/impl/pass.ex create mode 100644 lib/console/schema/component_content.ex create mode 100644 lib/console/schema/pr_automation.ex create mode 100644 lib/console/schema/scm_connection.ex create mode 100644 lib/console/schema/scm_webhook.ex create mode 100644 priv/repo/migrations/20240111013712_add_pr_automation.exs diff --git a/assets/src/generated/graphql.ts b/assets/src/generated/graphql.ts index eb40a3fe09..b1d8caedcb 100644 --- a/assets/src/generated/graphql.ts +++ b/assets/src/generated/graphql.ts @@ -834,6 +834,7 @@ export type Component = { }; export type ComponentAttributes = { + content?: InputMaybe; group: Scalars['String']['input']; kind: Scalars['String']['input']; name: Scalars['String']['input']; @@ -843,6 +844,24 @@ export type ComponentAttributes = { version: Scalars['String']['input']; }; +/** dry run content of a service component */ +export type ComponentContent = { + __typename?: 'ComponentContent'; + /** the inferred desired state of this component */ + desired?: Maybe; + id: Scalars['ID']['output']; + insertedAt?: Maybe; + live?: Maybe; + updatedAt?: Maybe; +}; + +/** the content of a component when visualized in dry run state */ +export type ComponentContentAttributes = { + /** the desired state of a service component as determined from the configured manifests */ + desired?: InputMaybe; + live?: InputMaybe; +}; + export enum ComponentState { Failed = 'FAILED', Paused = 'PAUSED', @@ -3304,7 +3323,7 @@ export type RootQueryTypeClustersArgs = { after?: InputMaybe; before?: InputMaybe; first?: InputMaybe; - health?: InputMaybe; + healthy?: InputMaybe; last?: InputMaybe; q?: InputMaybe; tag?: InputMaybe; @@ -3940,6 +3959,8 @@ export type ServiceComponent = { __typename?: 'ServiceComponent'; /** any api deprecations discovered from this component */ apiDeprecations?: Maybe>>; + /** the live and desired states of this service component */ + content?: Maybe; /** api group of this resource */ group?: Maybe; /** internal id */ @@ -3982,6 +4003,8 @@ export type ServiceDeployment = { deletedAt?: Maybe; /** fetches the /docs directory within this services git tree. This is a heavy operation and should NOT be used in list queries */ docs?: Maybe>>; + /** whether this service should not actively reconcile state and instead simply report pending changes */ + dryRun?: Maybe; /** whether this service is editable */ editable?: Maybe; /** a list of errors generated by the deployment operator */ @@ -3996,6 +4019,8 @@ export type ServiceDeployment = { /** internal id of this service */ id: Scalars['ID']['output']; insertedAt?: Maybe; + /** the desired sync interval for this service */ + interval?: Maybe; /** kustomize related service metadata */ kustomize?: Maybe; /** the commit message currently in use */ @@ -4045,6 +4070,7 @@ export type ServiceDeploymentRevisionsArgs = { export type ServiceDeploymentAttributes = { configuration?: InputMaybe>>; docsPath?: InputMaybe; + dryRun?: InputMaybe; git?: InputMaybe; helm?: InputMaybe; kustomize?: InputMaybe; diff --git a/lib/console/commands/plural.ex b/lib/console/commands/plural.ex index 8f8da2136d..254efbe610 100644 --- a/lib/console/commands/plural.ex +++ b/lib/console/commands/plural.ex @@ -47,6 +47,8 @@ defmodule Console.Commands.Plural do def repair(), do: plural("repair", []) + def template(conf), do: plural("cd", ["template", "--file", conf]) + def plural_home(command, args, env \\ []), do: cmd("plural", [command | args], System.user_home(), env) diff --git a/lib/console/deployments/git.ex b/lib/console/deployments/git.ex index 86da8a16d3..46f87df7ca 100644 --- a/lib/console/deployments/git.ex +++ b/lib/console/deployments/git.ex @@ -3,17 +3,29 @@ defmodule Console.Deployments.Git do import Console.Deployments.Policies alias Console.PubSub alias Console.Deployments.Settings - alias Console.Schema.{GitRepository, User, DeploymentSettings} + alias Console.Schema.{GitRepository, User, DeploymentSettings, ScmConnection, ScmWebhook, PrAutomation} @type repository_resp :: {:ok, GitRepository.t} | Console.error + @type connection_resp :: {:ok, ScmConnection.t} | Console.error + @type webhook_resp :: {:ok, ScmWebhook.t} | Console.error + @type automation_resp :: {:ok, PrAutomation.t} | Console.error - def get_repository(id), do: Console.Repo.get(GitRepository, id) + def get_repository(id), do: Repo.get(GitRepository, id) - def get_repository!(id), do: Console.Repo.get!(GitRepository, id) + def get_repository!(id), do: Repo.get!(GitRepository, id) - def get_by_url!(url), do: Console.Repo.get_by!(GitRepository, url: url) + def get_by_url!(url), do: Repo.get_by!(GitRepository, url: url) - def get_by_url(url), do: Console.Repo.get_by(GitRepository, url: url) + def get_by_url(url), do: Repo.get_by(GitRepository, url: url) + + def get_scm_connection(id), do: Repo.get(ScmConnection, id) + def get_scm_connection!(id), do: Repo.get!(ScmConnection, id) + + def get_scm_webhook(id), do: Repo.get(ScmWebhook, id) + def get_scm_webhook!(id), do: Repo.get!(ScmWebhook, id) + + def get_pr_automation(id), do: Repo.get(PrAutomation, id) + def get_pr_automation!(id), do: Repo.get!(PrAutomation, id) def deploy_url(), do: "https://github.com/pluralsh/deployment-operator.git" @@ -73,6 +85,102 @@ defmodule Console.Deployments.Git do end end + @doc """ + + """ + @spec create_scm_connection(map, User.t) :: connection_resp + def create_scm_connection(attrs, %User{} = user) do + %ScmConnection{} + |> ScmConnection.changeset(attrs) + |> allow(user, :edit) + |> when_ok(:insert) + end + + @doc """ + + """ + @spec update_scm_connection(map, binary, User.t) :: connection_resp + def update_scm_connection(attrs, id, %User{} = user) do + get_scm_connection!(id) + |> ScmConnection.changeset(attrs) + |> allow(user, :edit) + |> when_ok(:update) + end + + @doc """ + + """ + @spec delete_scm_connection(binary, User.t) :: connection_resp + def delete_scm_connection(id, %User{} = user) do + get_scm_connection!(id) + |> allow(user, :edit) + |> when_ok(:delete) + end + + @doc """ + + """ + @spec create_scm_webhook(map, User.t) :: webhook_resp + def create_scm_webhook(attrs, %User{} = user) do + %ScmWebhook{} + |> ScmWebhook.changeset(attrs) + |> allow(user, :edit) + |> when_ok(:insert) + end + + @doc """ + + """ + @spec update_scm_webhook(map, binary, User.t) :: webhook_resp + def update_scm_webhook(attrs, id, %User{} = user) do + get_scm_webhook!(id) + |> ScmWebhook.changeset(attrs) + |> allow(user, :edit) + |> when_ok(:update) + end + + @doc """ + + """ + @spec delete_scm_webhook(binary, User.t) :: webhook_resp + def delete_scm_webhook(id, %User{} = user) do + get_scm_webhook!(id) + |> allow(user, :edit) + |> when_ok(:delete) + end + + @doc """ + + """ + @spec create_pr_automation(map, User.t) :: automation_resp + def create_pr_automation(attrs, %User{} = user) do + %PrAutomation{} + |> PrAutomation.changeset(attrs) + |> allow(user, :edit) + |> when_ok(:insert) + end + + @doc """ + + """ + @spec update_pr_automation(map, binary, User.t) :: automation_resp + def update_pr_automation(attrs, id, %User{} = user) do + get_pr_automation!(id) + |> PrAutomation.changeset(attrs) + |> allow(user, :edit) + |> when_ok(:update) + end + + @doc """ + + """ + @spec delete_pr_automation(binary, User.t) :: automation_resp + def delete_pr_automation(id, %User{} = user) do + get_pr_automation!(id) + |> allow(user, :edit) + |> when_ok(:delete) + end + @doc """ Fetches all helm repos registered in this cluster so far """ diff --git a/lib/console/deployments/pr/config.ex b/lib/console/deployments/pr/config.ex new file mode 100644 index 0000000000..901810ad66 --- /dev/null +++ b/lib/console/deployments/pr/config.ex @@ -0,0 +1,23 @@ +defmodule Console.Deployments.Pr.Config do + alias Console.Schema.PrAutomation + + @doc """ + Generate onfig for executing a pr template + """ + @spec config(PrAutomation.t, map) :: {:ok, binary} | Console.error + def config(%PrAutomation{} = pr, ctx) do + with {:ok, f} <- Briefly.create(), + {:ok, doc} <- Ymlr.document(structure(pr, ctx)), + :ok <- File.write(f, String.trim_leading(doc, "---\n")), + do: {:ok, f} + end + + defp structure(pr, ctx) do + %{ + apiVersion: "pr.plural.sh/v1alpha1", + kind: "PrTemplate", + spec: Console.mapify(pr.spec) |> Map.put(:message, pr.message), + context: ctx + } + end +end diff --git a/lib/console/deployments/pr/dispatcher.ex b/lib/console/deployments/pr/dispatcher.ex new file mode 100644 index 0000000000..a54d20c2bc --- /dev/null +++ b/lib/console/deployments/pr/dispatcher.ex @@ -0,0 +1,29 @@ +defmodule Console.Deployments.Pr.Dispatcher do + import Console.Deployments.Pr.Git + alias Console.Repo + alias Console.Deployments.Pr.Config + alias Console.Commands.Plural + alias Console.Deployments.Pr.Impl.{Github, Gitlab} + alias Console.Schema.{PrAutomation, ScmConnection} + + @type pr_resp :: {:ok, binary} | Console.error + + @callback create(pr :: PrAutomation.t, identifier :: binary, branch :: binary, context :: map) :: pr_resp + + @doc """ + Fully creates a pr against the working dispatcher implementation + """ + @spec create(PrAutomation.t, binary, binary, map) :: pr_resp + def create(%PrAutomation{} = pr, identifier, branch, ctx) do + %{scm_connection: conn} = pr = Repo.preload(pr, [:scm_connection]) + impl = dispatcher(conn) + with {:ok, conn} <- setup(conn, identifier, branch), + {:ok, f} <- Config.config(pr, ctx), + {:ok, _} <- Plural.template(f), + {:ok, _} <- push(conn, branch), + do: impl.create(pr, identifier, branch, ctx) + end + + defp dispatcher(%ScmConnection{type: :github}), do: Github + defp dispatcher(%ScmConnection{type: :gitlab}), do: Gitlab +end diff --git a/lib/console/deployments/pr/git.ex b/lib/console/deployments/pr/git.ex new file mode 100644 index 0000000000..059d5a122e --- /dev/null +++ b/lib/console/deployments/pr/git.ex @@ -0,0 +1,46 @@ +defmodule Console.Deployments.Pr.Git do + alias Console.Schema.ScmConnection + + @type git_resp :: {:ok, binary} | Console.error + + @spec setup(ScmConnection.t, binary, binary) :: {:ok, ScmConnection.t} | Console.error + def setup(%ScmConnection{} = conn, id, branch) do + with {:ok, dir} <- Briefly.create(directory: true), + conn = %{conn | dir: dir}, + {:ok, _} <- git(conn, "clone", [url(conn, id), dir]), + {:ok, _} <- git(conn, "checkout", ["-b", branch]), + do: {:ok, conn} + end + + @spec commit(ScmConnection.t, binary) :: git_resp + def commit(%ScmConnection{} = conn, msg), do: git(conn, "commit", ["-m", msg]) + + @spec push(ScmConnection.t, binary) :: git_resp + def push(%ScmConnection{} = conn, branch), do: git(conn, "push", ["--set-upstream", "origin", branch]) + + defp git(%ScmConnection{} = conn, cmd, args) when is_list(args) do + case System.cmd("git", [cmd | args], opts(conn)) do + {out, 0} -> {:ok, out} + {out, _} -> {:error, out} + end + end + + defp url(%ScmConnection{username: nil} = conn, id), do: url(%{conn | username: "apikey"}, id) + defp url(%ScmConnection{username: username} = conn, id) do + base = url(conn) + uri = URI.parse("#{base}/#{id}.git") + URI.to_string(%{uri | userinfo: username}) + end + + defp url(%ScmConnection{base_url: base}) when is_binary(base), do: base + defp url(%ScmConnection{type: :github}), do: "https://github.com/" + defp url(%ScmConnection{type: :gitlab}), do: "https://gitlab.com/" + + defp opts(%ScmConnection{dir: dir} = conn), do: [env: env(conn), cd: dir, stderr_to_stdout: true] + + defp env(%ScmConnection{token: password}) when is_binary(password), + do: [{"GIT_ACCESS_TOKEN", password}, {"GIT_ASKPASS", git_askpass()}] + defp env(_), do: [] + + defp git_askpass(), do: Console.conf(:git_askpass) +end diff --git a/lib/console/deployments/pr/impl/github.ex b/lib/console/deployments/pr/impl/github.ex new file mode 100644 index 0000000000..d345b3e847 --- /dev/null +++ b/lib/console/deployments/pr/impl/github.ex @@ -0,0 +1,5 @@ +defmodule Console.Deployments.Pr.Impl.Github do + @behaviour Console.Deployments.Pr.Dispatcher + + def create(_, _, _, _), do: {:ok, ""} +end diff --git a/lib/console/deployments/pr/impl/gitlab.ex b/lib/console/deployments/pr/impl/gitlab.ex new file mode 100644 index 0000000000..cc4f4871a4 --- /dev/null +++ b/lib/console/deployments/pr/impl/gitlab.ex @@ -0,0 +1,5 @@ +defmodule Console.Deployments.Pr.Impl.Gitlab do + @behaviour Console.Deployments.Pr.Dispatcher + + def create(_, _, _, _), do: {:ok, ""} +end diff --git a/lib/console/deployments/pr/impl/pass.ex b/lib/console/deployments/pr/impl/pass.ex new file mode 100644 index 0000000000..e6b2234192 --- /dev/null +++ b/lib/console/deployments/pr/impl/pass.ex @@ -0,0 +1,5 @@ +defmodule Console.Deployments.Pr.Impl.Pass do + @behaviour Console.Deployments.Pr.Dispatcher + + def create(_, _, _, _), do: {:ok, ""} +end diff --git a/lib/console/graphql/deployments/cluster.ex b/lib/console/graphql/deployments/cluster.ex index ce3bba785d..6133a1e229 100644 --- a/lib/console/graphql/deployments/cluster.ex +++ b/lib/console/graphql/deployments/cluster.ex @@ -475,9 +475,9 @@ defmodule Console.GraphQl.Deployments.Cluster do @desc "a relay connection of all clusters visible to the current user" connection field :clusters, node_type: :cluster do middleware Authenticated - arg :q, :string - arg :health, :boolean - arg :tag, :tag_input + arg :q, :string + arg :healthy, :boolean + arg :tag, :tag_input resolve &Deployments.list_clusters/2 end diff --git a/lib/console/graphql/deployments/service.ex b/lib/console/graphql/deployments/service.ex index 77d9a4deee..3282233fc2 100644 --- a/lib/console/graphql/deployments/service.ex +++ b/lib/console/graphql/deployments/service.ex @@ -15,6 +15,7 @@ defmodule Console.GraphQl.Deployments.Service do field :sync_config, :sync_config_attributes field :protect, :boolean field :repository_id, :id + field :dry_run, :boolean field :git, :git_ref_attributes field :helm, :helm_config_attributes field :kustomize, :kustomize_attributes @@ -73,6 +74,13 @@ defmodule Console.GraphQl.Deployments.Service do field :kind, non_null(:string) field :namespace, non_null(:string) field :name, non_null(:string) + field :content, :component_content_attributes + end + + @desc "the content of a component when visualized in dry run state" + input_object :component_content_attributes do + field :desired, :string, description: "the desired state of a service component as determined from the configured manifests" + field :live, :string end input_object :service_error_attributes do @@ -99,6 +107,7 @@ defmodule Console.GraphQl.Deployments.Service do field :namespace, non_null(:string), description: "kubernetes namespace this service will be deployed to" field :status, non_null(:service_deployment_status), description: "A summary status enum for the health of this service" field :version, non_null(:string), description: "semver of this service" + field :interval, :string, description: "the desired sync interval for this service" field :git, :git_ref, description: "description on where in git the service's manifests should be fetched" field :helm, :helm_spec, description: "description of how helm charts should be applied", resolve: fn %{helm: %{} = helm} = svc, _, _ -> @@ -114,6 +123,7 @@ defmodule Console.GraphQl.Deployments.Service do field :kustomize, :kustomize, description: "kustomize related service metadata" field :message, :string, description: "the commit message currently in use" field :deleted_at, :datetime, description: "the time this service was scheduled for deletion" + field :dry_run, :boolean, description: "whether this service should not actively reconcile state and instead simply report pending changes" @desc "fetches the /docs directory within this services git tree. This is a heavy operation and should NOT be used in list queries" field :docs, list_of(:git_file), resolve: &Deployments.docs/3 @@ -200,10 +210,20 @@ defmodule Console.GraphQl.Deployments.Service do field :namespace, :string, description: "kubernetes namespace of this resource" field :name, non_null(:string), description: "kubernetes name of this resource" + field :content, :component_content, resolve: dataloader(Deployments), description: "the live and desired states of this service component" field :service, :service_deployment, resolve: dataloader(Deployments), description: "the service this component belongs to" field :api_deprecations, list_of(:api_deprecation), resolve: dataloader(Deployments), description: "any api deprecations discovered from this component" end + @desc "dry run content of a service component" + object :component_content do + field :id, non_null(:id) + field :live, :string + field :desired, :string, description: "the inferred desired state of this component" + + timestamps() + end + @desc "a representation of a kubernetes api deprecation" object :api_deprecation do field :deprecated_in, :string, description: "the kubernetes version the deprecation was posted" diff --git a/lib/console/graphql/resolvers/deployments.ex b/lib/console/graphql/resolvers/deployments.ex index e35dbbca01..07339fba89 100644 --- a/lib/console/graphql/resolvers/deployments.ex +++ b/lib/console/graphql/resolvers/deployments.ex @@ -27,6 +27,7 @@ defmodule Console.GraphQl.Resolvers.Deployments do PipelinePromotion, PromotionCriteria, PromotionService, + ComponentContent } def query(Pipeline, _), do: Pipeline @@ -49,6 +50,7 @@ defmodule Console.GraphQl.Resolvers.Deployments do def query(Revision, _), do: Revision def query(ServiceComponent, _), do: ServiceComponent def query(GitRepository, _), do: GitRepository + def query(ComponentContent, _), do: ComponentContent def query(_, _), do: Cluster def list_clusters(args, %{context: %{current_user: user}}) do diff --git a/lib/console/schema/component_content.ex b/lib/console/schema/component_content.ex new file mode 100644 index 0000000000..a7f07af434 --- /dev/null +++ b/lib/console/schema/component_content.ex @@ -0,0 +1,23 @@ +defmodule Console.Schema.ComponentContent do + use Piazza.Ecto.Schema + alias Console.Schema.{ServiceComponent} + + schema "component_contents" do + field :desired, :binary + field :live, :binary + + belongs_to :component, ServiceComponent + + timestamps() + end + + @valid ~w(desired live component_id)a + + def changeset(model, attrs \\ %{}) do + model + |> cast(attrs, @valid) + |> foreign_key_constraint(:component_id) + |> unique_constraint(:component_id) + |> validate_required([:desired]) + end +end diff --git a/lib/console/schema/pr_automation.ex b/lib/console/schema/pr_automation.ex new file mode 100644 index 0000000000..3187a2837e --- /dev/null +++ b/lib/console/schema/pr_automation.ex @@ -0,0 +1,49 @@ +defmodule Console.Schema.PrAutomation do + use Piazza.Ecto.Schema + alias Console.Schema.{Cluster, Service, ScmConnection} + + defenum MatchStrategy, any: 0, all: 1, recursive: 2 + + schema "pr_automations" do + field :identifier, :string + field :name, :string + field :documentation, :binary + field :addon, :string + field :message, :binary + + embeds_one :spec, Spec, on_replace: :update do + field :regexes, {:array, :string} + field :files, {:array, :string} + field :replace_template, :string + field :yq, :string + field :match_strategy, MatchStrategy + end + + belongs_to :cluster, Cluster + belongs_to :service, Service + belongs_to :connection, ScmConnection + + timestamps() + end + + def ordered(query \\ __MODULE__, order \\ [asc: :name]) do + from(p in query, order_by: ^order) + end + + @valid ~w(name identifier documentation addon cluster_id service_id connection_id)a + + def changeset(model, attrs \\ %{}) do + model + |> cast(attrs, @valid) + |> cast_embed(:spec) + |> validate_required([:name, :documentation, :connection_id]) + |> unique_constraint(:name) + |> foreign_key_constraint(:cluster_id) + |> foreign_key_constraint(:service_id) + |> foreign_key_constraint(:connection_id) + end + + def spec_changeset(model, attrs \\ %{}) do + cast(model, attrs, ~w(regexes files yq)a) + end +end diff --git a/lib/console/schema/scm_connection.ex b/lib/console/schema/scm_connection.ex new file mode 100644 index 0000000000..3b83ec8889 --- /dev/null +++ b/lib/console/schema/scm_connection.ex @@ -0,0 +1,30 @@ +defmodule Console.Schema.ScmConnection do + use Piazza.Ecto.Schema + + defenum Type, github: 0, gitlab: 1 + + schema "scm_connections" do + field :name, :string + field :type, Type + field :base_url, :string + field :api_url, :string + field :username, :string + field :token, Piazza.Ecto.EncryptedString + field :dir, :string, virtual: true + + timestamps() + end + + def ordered(query \\ __MODULE__, order \\ [asc: :name]) do + from(scm in query, order_by: ^order) + end + + @valid ~w(name type base_url username token)a + + def changeset(model, attrs \\ %{}) do + model + |> cast(attrs, @valid) + |> unique_constraint(:name) + |> validate_required([:name, :type, :token]) + end +end diff --git a/lib/console/schema/scm_webhook.ex b/lib/console/schema/scm_webhook.ex new file mode 100644 index 0000000000..ca3a5c11ac --- /dev/null +++ b/lib/console/schema/scm_webhook.ex @@ -0,0 +1,25 @@ +defmodule Console.Schema.ScmWebhook do + use Piazza.Ecto.Schema + + schema "scm_webhooks" do + field :name, :string + field :type, Console.Schema.ScmConnection.Type + field :hmac, Piazza.Ecto.EncryptedString + field :external_id, :string + + timestamps() + end + + def ordered(query \\ __MODULE__, order \\ [asc: :name]) do + from(w in query, order_by: ^order) + end + + @valid ~w(name type hmac external_id)a + + def changeset(model, attrs \\ %{}) do + model + |> cast(attrs, @valid) + |> put_new_change(:external_id, fn -> Console.rand_alphanum(30) end) + |> validate_required(~w(name type hmac external_id)a) + end +end diff --git a/lib/console/schema/service.ex b/lib/console/schema/service.ex index c7f548e461..802cb89cfd 100644 --- a/lib/console/schema/service.ex +++ b/lib/console/schema/service.ex @@ -75,6 +75,8 @@ defmodule Console.Schema.Service do field :read_policy_id, :binary_id field :deleted_at, :utc_datetime_usec field :protect, :boolean + field :dry_run, :boolean + field :interval, :string embeds_one :git, Git, on_replace: :update embeds_one :helm, Helm, on_replace: :update @@ -185,13 +187,14 @@ defmodule Console.Schema.Service do def docs_path(%__MODULE__{docs_path: p}) when is_binary(p), do: p def docs_path(%__MODULE__{git: %{folder: p}}), do: Path.join(p, "docs") - @valid ~w(name protect docs_path component_status status version sha cluster_id repository_id namespace owner_id message)a + @valid ~w(name protect interval docs_path component_status dry_run interval status version sha cluster_id repository_id namespace owner_id message)a def changeset(model, attrs \\ %{}) do model |> cast(attrs, @valid) |> kubernetes_names([:name, :namespace]) |> semver(:version) + |> validate_format(:interval, ~r/\d+[mhs]/, message: "interval must be a valid go interval string") |> cast_embed(:git) |> cast_embed(:helm) |> cast_embed(:sync_config, with: &sync_config_changeset/2) diff --git a/lib/console/schema/service_component.ex b/lib/console/schema/service_component.ex index 00490a64ea..0eba9a0878 100644 --- a/lib/console/schema/service_component.ex +++ b/lib/console/schema/service_component.ex @@ -1,6 +1,6 @@ defmodule Console.Schema.ServiceComponent do use Piazza.Ecto.Schema - alias Console.Schema.{Service, ApiDeprecation} + alias Console.Schema.{Service, ApiDeprecation, Content} defenum State, running: 0, pending: 1, failed: 2, paused: 3 @@ -15,6 +15,7 @@ defmodule Console.Schema.ServiceComponent do belongs_to :service, Service has_many :api_deprecations, ApiDeprecation, foreign_key: :component_id + has_one :content, ComponentContent, foreign_key: :component_id timestamps() end @@ -28,6 +29,7 @@ defmodule Console.Schema.ServiceComponent do def changeset(model, attrs \\ %{}) do model |> cast(attrs, @valid) + |> cast_assoc(:content) |> foreign_key_constraint(:service) |> validate_required([:kind, :name]) end diff --git a/mix.exs b/mix.exs index 0391a10f28..11d0ad367c 100644 --- a/mix.exs +++ b/mix.exs @@ -72,6 +72,7 @@ defmodule Console.MixProject do {:distillery, "~> 2.1"}, {:libcluster, "~> 3.2"}, {:horde, "~> 0.8"}, + {:tentacat, "~> 2.0"}, {:postgrex, ">= 0.0.0"}, {:phoenix, "~> 1.5"}, {:phoenix_view, "~> 2.0"}, diff --git a/mix.lock b/mix.lock index b1dc017529..21632278c2 100644 --- a/mix.lock +++ b/mix.lock @@ -108,6 +108,7 @@ "sweet_xml": {:hex, :sweet_xml, "0.7.1", "a2cac8e2101237e617dfa9d427d44b8aff38ba6294f313ffb4667524d6b71b98", [:mix], [], "hexpm", "8bc7b7b584a6a87113071d0d2fd39fe2251cf2224ecaeed7093bdac1b9c1555f"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "telemetry_poller": {:hex, :telemetry_poller, "0.4.1", "50d03d976a3b8ab4898d9e873852e688840df47685a13af90af40e1ba43a758b", [:rebar3], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c5bacbbcd62c1fe4e4517485bd64312622e9b83683273dcf2627ff224d7d485b"}, + "tentacat": {:hex, :tentacat, "2.3.0", "2d252bf7526292d461fd799739f6b62587d46bf3ee53243d3e538199f285a9fc", [:mix], [{:httpoison, "~> 2.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "0570a962fd4b0d13a77e6b7002efd6d6c3c33b7277444c61904528b3c3956f96"}, "timex": {:hex, :timex, "3.7.5", "3eca56e23bfa4e0848f0b0a29a92fa20af251a975116c6d504966e8a90516dfd", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "a15608dca680f2ef663d71c95842c67f0af08a0f3b1d00e17bbd22872e2874e4"}, "tzdata": {:hex, :tzdata, "1.1.0", "72f5babaa9390d0f131465c8702fa76da0919e37ba32baa90d93c583301a8359", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "18f453739b48d3dc5bcf0e8906d2dc112bb40baafe2c707596d89f3c8dd14034"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, diff --git a/priv/repo/migrations/20240111013712_add_pr_automation.exs b/priv/repo/migrations/20240111013712_add_pr_automation.exs new file mode 100644 index 0000000000..7e48d414f3 --- /dev/null +++ b/priv/repo/migrations/20240111013712_add_pr_automation.exs @@ -0,0 +1,63 @@ +defmodule Console.Repo.Migrations.AddPrAutomation do + use Ecto.Migration + + def change do + create table(:scm_connections, primary_key: false) do + add :id, :uuid, primary_key: true + add :type, :integer + add :name, :string + add :base_url, :string + add :api_url, :string + add :token, :binary + add :username, :string + + timestamps() + end + + create table(:scm_webhooks, primary_key: false) do + add :id, :uuid, primary_key: true + add :type, :integer + add :hmac, :binary + add :external_id, :string + + timestamps() + end + + create table(:pr_automations, primary_key: false) do + add :id, :uuid, primary_key: true + add :name, :string + add :documentation, :binary + add :addon, :string + add :identifier, :string + add :message, :binary + add :cluster_id, references(:clusters, type: :uuid, on_delete: :nilify_all) + add :service_id, references(:services, type: :uuid, on_delete: :nilify_all) + add :connection_id, references(:scm_connections, type: :uuid, on_delete: :delete_all) + add :spec, :map + + timestamps() + end + + alter table(:services) do + add :dry_run, :boolean + add :interval, :string + end + + create table(:component_contents, primary_key: false) do + add :id, :uuid, primary_key: true + add :live, :binary + add :desired, :binary + add :component_id, references(:service_components, type: :uuid, on_delete: :delete_all) + + timestamps() + end + + create unique_index(:scm_connections, [:name]) + create unique_index(:pr_automations, [:name]) + create unique_index(:scm_webhooks, [:external_id]) + create index(:pr_automations, [:addon]) + create index(:pr_automations, [:cluster_id]) + create index(:pr_automations, [:service_id]) + create unique_index(:component_contents, [:component_id]) + end +end diff --git a/schema/schema.graphql b/schema/schema.graphql index 8c602b111e..5dfef57fd5 100644 --- a/schema/schema.graphql +++ b/schema/schema.graphql @@ -173,7 +173,7 @@ type RootQueryType { tokenExchange(token: String!): User "a relay connection of all clusters visible to the current user" - clusters(after: String, first: Int, before: String, last: Int, q: String, health: Boolean, tag: TagInput): ClusterConnection + clusters(after: String, first: Int, before: String, last: Int, q: String, healthy: Boolean, tag: TagInput): ClusterConnection "gets summary information for all healthy\/unhealthy clusters in your fleet" clusterStatuses(q: String, tag: TagInput): [ClusterStatusInfo] @@ -941,6 +941,7 @@ input ServiceDeploymentAttributes { syncConfig: SyncConfigAttributes protect: Boolean repositoryId: ID + dryRun: Boolean git: GitRefAttributes helm: HelmConfigAttributes kustomize: KustomizeAttributes @@ -999,6 +1000,15 @@ input ComponentAttributes { kind: String! namespace: String! name: String! + content: ComponentContentAttributes +} + +"the content of a component when visualized in dry run state" +input ComponentContentAttributes { + "the desired state of a service component as determined from the configured manifests" + desired: String + + live: String } input ServiceErrorAttributes { @@ -1043,6 +1053,9 @@ type ServiceDeployment { "semver of this service" version: String! + "the desired sync interval for this service" + interval: String + "description on where in git the service's manifests should be fetched" git: GitRef @@ -1076,6 +1089,9 @@ type ServiceDeployment { "the time this service was scheduled for deletion" deletedAt: DateTime + "whether this service should not actively reconcile state and instead simply report pending changes" + dryRun: Boolean + "fetches the \/docs directory within this services git tree. This is a heavy operation and should NOT be used in list queries" docs: [GitFile] @@ -1215,6 +1231,9 @@ type ServiceComponent { "kubernetes name of this resource" name: String! + "the live and desired states of this service component" + content: ComponentContent + "the service this component belongs to" service: ServiceDeployment @@ -1222,6 +1241,20 @@ type ServiceComponent { apiDeprecations: [ApiDeprecation] } +"dry run content of a service component" +type ComponentContent { + id: ID! + + live: String + + "the inferred desired state of this component" + desired: String + + insertedAt: DateTime + + updatedAt: DateTime +} + "a representation of a kubernetes api deprecation" type ApiDeprecation { "the kubernetes version the deprecation was posted" diff --git a/test/console/graphql/queries/deployment_queries_test.exs b/test/console/graphql/queries/deployment_queries_test.exs index 3f42189fd7..aba6a0e9db 100644 --- a/test/console/graphql/queries/deployment_queries_test.exs +++ b/test/console/graphql/queries/deployment_queries_test.exs @@ -91,6 +91,37 @@ defmodule Console.GraphQl.DeploymentQueriesTest do assert Enum.all?(found, & &1["name"]) end + test "it can list clusters by health in the system" do + clusters = insert_list(3, :cluster, pinged_at: Timex.now()) + others = insert_list(3, :cluster, pinged_at: Timex.now() |> Timex.shift(hours: -1)) + + {:ok, %{data: %{"clusters" => found}}} = run_query(""" + query { + clusters(first: 5, healthy: true) { + edges { node { id name } } + } + } + """, %{}, %{current_user: admin_user()}) + + found = from_connection(found) + + assert ids_equal(found, clusters) + assert Enum.all?(found, & &1["name"]) + + {:ok, %{data: %{"clusters" => found}}} = run_query(""" + query { + clusters(first: 5, healthy: false) { + edges { node { id name } } + } + } + """, %{}, %{current_user: admin_user()}) + + found = from_connection(found) + + assert ids_equal(found, others) + assert Enum.all?(found, & &1["name"]) + end + test "it will respect rbac" do user = insert(:user) %{group: group} = insert(:group_member, user: user)