From 0d73868dfea5ac2fc7b10cdc0e58df35183753c8 Mon Sep 17 00:00:00 2001 From: michaeljguarino Date: Wed, 18 Sep 2024 19:29:48 -0400 Subject: [PATCH] refactor to separate shared/dedicated workflows, and add test --- apps/core/lib/core/clients/console.ex | 5 + .../lib/core/services/cloud/configuration.ex | 2 +- apps/core/lib/core/services/cloud/utils.ex | 14 ++ apps/core/lib/core/services/cloud/workflow.ex | 209 ++---------------- .../core/services/cloud/workflow/dedicated.ex | 79 +++++++ .../core/services/cloud/workflow/shared.ex | 135 +++++++++++ .../test/services/cloud/workflow_test.exs | 59 +++++ 7 files changed, 310 insertions(+), 193 deletions(-) create mode 100644 apps/core/lib/core/services/cloud/utils.ex create mode 100644 apps/core/lib/core/services/cloud/workflow/dedicated.ex create mode 100644 apps/core/lib/core/services/cloud/workflow/shared.ex diff --git a/apps/core/lib/core/clients/console.ex b/apps/core/lib/core/clients/console.ex index 47612d0e9..a8050c7ad 100644 --- a/apps/core/lib/core/clients/console.ex +++ b/apps/core/lib/core/clients/console.ex @@ -90,6 +90,11 @@ defmodule Core.Clients.Console do } """ + def queries(:me_q), do: @me_q + def queries(:stack_q), do: @stack_q + def queries(:stack_create), do: @create_stack_q + def queries(:stack_delete), do: @delete_stack_q + def new(url, token) do Req.new(base_url: with_gql(url), auth: "Token #{token}") |> AbsintheClient.attach() diff --git a/apps/core/lib/core/services/cloud/configuration.ex b/apps/core/lib/core/services/cloud/configuration.ex index 3a547bedf..b5764bcc0 100644 --- a/apps/core/lib/core/services/cloud/configuration.ex +++ b/apps/core/lib/core/services/cloud/configuration.ex @@ -35,7 +35,7 @@ defmodule Core.Services.Cloud.Configuration do cluster_id: Core.conf(:mgmt_cluster), type: "TERRAFORM", manageState: true, - approval: true, + approval: false, configuration: %{version: "1.8"}, git: %{ref: "main", folder: "terraform/modules/dedicated/#{inst.cloud}"}, environment: Enum.map([ diff --git a/apps/core/lib/core/services/cloud/utils.ex b/apps/core/lib/core/services/cloud/utils.ex new file mode 100644 index 000000000..4e775e09b --- /dev/null +++ b/apps/core/lib/core/services/cloud/utils.ex @@ -0,0 +1,14 @@ +defmodule Core.Services.Cloud.Utils do + alias Core.Repo + alias Core.Clients.Console + alias Core.Schema.{ConsoleInstance} + + def mark_provisioned(inst) do + ConsoleInstance.changeset(inst, %{status: :provisioned}) + |> Repo.update() + end + + def console(), do: Console.new(Core.conf(:console_url), Core.conf(:console_token)) + + def dedicated_console(), do: Console.new(Core.conf(:console_url), Core.conf(:dedicated_console_token)) +end diff --git a/apps/core/lib/core/services/cloud/workflow.ex b/apps/core/lib/core/services/cloud/workflow.ex index e80bfe354..f88274c94 100644 --- a/apps/core/lib/core/services/cloud/workflow.ex +++ b/apps/core/lib/core/services/cloud/workflow.ex @@ -1,28 +1,19 @@ defmodule Core.Services.Cloud.Workflow do use Core.Services.Base - alias Core.Clients.Console - alias Core.Services.{Cloud, Users} - alias Core.Services.Cloud.{Poller, Configuration} - alias Core.Schema.{ConsoleInstance, PostgresCluster, User} alias Core.Repo + alias Core.Clients.Console + alias Core.Schema.{ConsoleInstance} + alias Core.Services.Cloud.Workflow.{Dedicated, Shared} require Logger - def sync(%ConsoleInstance{type: :dedicated, external_id: id} = inst) when is_binary(id) do - with {:ok, project_id} <- Poller.project(), - {:ok, repo_id} <- Poller.repository(), - {:ok, actor} <- Console.me(dedicated_console()), - attrs = %{actor_id: actor, project_id: project_id, repository_id: repo_id}, - do: Console.update_stack(dedicated_console(), id, Configuration.stack_attributes(inst, attrs)) - end + @type error :: {:error, term} + @type resp :: {:ok, ConsoleInstance.t} | error - def sync(%ConsoleInstance{external_id: id} = instance) when is_binary(id) do - instance = Repo.preload(instance, [:cluster, :postgres]) - Console.update_service(console(), id, %{ - configuration: Configuration.build(instance) - }) - end - def sync(_), do: :ok + @callback sync(inst :: ConsoleInstance.t) :: :ok | {:ok, term} | error + @callback up(inst :: ConsoleInstance.t) :: resp + @callback down(inst :: ConsoleInstance.t) :: resp + @callback finalize(inst :: ConsoleInstance.t, :up | :down) :: resp def provision(%ConsoleInstance{} = instance) do instance = Repo.preload(instance, [:postgres, :cluster]) @@ -58,181 +49,15 @@ defmodule Core.Services.Cloud.Workflow do |> finalize(:down) end - defp up(%ConsoleInstance{status: :pending, type: :dedicated} = inst) do - with {:ok, id} <- Poller.project(), - {:ok, repo_id} <- Poller.repository(), - {:ok, actor} <- Console.me(dedicated_console()), - attrs = %{actor_id: actor, project_id: id, repository_id: repo_id}, - {:ok, stack_id} <- Console.create_stack(dedicated_console(), Configuration.stack_attributes(inst, attrs)) do - ConsoleInstance.changeset(inst, %{ - instance_status: %{stack: true}, - status: :stack_created, - external_id: stack_id - }) - |> Repo.update() - end - end - - defp up(%ConsoleInstance{type: :dedicated, status: :stack_created, external_id: id} = inst) do - Enum.reduce_while(0..120, inst, fn _, inst -> - dedicated_console() - |> Console.stack(id) - |> case do - {:ok, %{"status" => "SUCCESSFUL"}} -> - ConsoleInstance.changeset(inst, %{status: :provisioned}) - |> Repo.update() - status -> - Logger.info "stack not ready yet, sleeping: #{inspect(status)}" - :timer.sleep(:timer.minutes(1)) - {:ok, inst} - end - end) - end - - defp up(%ConsoleInstance{status: :deployment_created, url: url} = inst) do - case {DNS.resolve(url), DNS.resolve(url, :cname)} do - {{:ok, [_ | _]}, _} -> mark_provisioned(inst) - {_, {:ok, [_ | _]}} -> mark_provisioned(inst) - {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"} - end - end - - defp up(%ConsoleInstance{status: :pending, postgres: pg, configuration: conf} = inst) do - with {:ok, pid} <- connect(pg), - {:ok, _} <- Postgrex.query(pid, "CREATE DATABASE #{conf.database}", []), - {:ok, _} <- Postgrex.transaction(pid, fn conn -> - Postgrex.query!(conn, "CREATE USER #{conf.dbuser} WITH PASSWORD '#{conf.dbpassword}'", []) - Postgrex.query!(conn, "GRANT ALL ON DATABASE #{conf.database} TO #{conf.dbuser}", []) - end) do - ConsoleInstance.changeset(inst, %{ - instance_status: %{db: true}, - status: :database_created, - }) - |> Repo.update() - end - end - - defp up(%ConsoleInstance{instance_status: %{db: true}, name: name, cluster: cluster} = inst) do - with {:ok, id} <- Poller.repository(), - {:ok, svc_id} <- Console.create_service(console(), cluster.external_id, %{ - name: "console-cloud-#{name}", - namespace: "plrl-cloud-#{name}", - helm: %{ - url: "https://pluralsh.github.io/console", - chart: "console-rapid", - release: "console", - version: "x.x.x", - valuesFiles: ["console.yaml.liquid"] - }, - repository_id: id, - git: %{ref: "main", folder: "helm"}, - configuration: Configuration.build(inst), - }) do - ConsoleInstance.changeset(inst, %{ - external_id: svc_id, - instance_status: %{svc: true}, - status: :deployment_created - }) - |> Repo.update() - end - end - - defp up(inst), do: {:ok, inst} - - defp down(%ConsoleInstance{type: :dedicated, instance_status: %{stack: true}, external_id: id} = inst) do - with {:ok, _} <- Console.delete_stack(dedicated_console(), id) do - ConsoleInstance.changeset(inst, %{status: :stack_deleted}) - |> Repo.update() - end - end - - defp down(%ConsoleInstance{instance_status: %{svc: false, db: true}, configuration: conf, postgres: pg} = inst) do - with {:ok, pid} <- connect(pg), - {:ok, _} <- Postgrex.query(pid, "DROP DATABASE IF EXISTS #{conf.database}", []), - {:ok, _} <- Postgrex.transaction(pid, fn conn -> - Postgrex.query!(conn, "DROP USER IF EXISTS #{conf.dbuser}", []) - end) do - ConsoleInstance.changeset(inst, %{ - instance_status: %{db: false}, - status: :database_deleted, - }) - |> Repo.update() - end - end + def sync(%ConsoleInstance{type: :dedicated} = inst), do: Dedicated.sync(inst) + def sync(inst), do: Shared.sync(inst) - defp down(%ConsoleInstance{instance_status: %{svc: true}} = inst) do - with {:ok, _} <- Console.delete_service(console(), inst.external_id) do - ConsoleInstance.changeset(inst, %{ - instance_status: %{svc: false, db: true}, - status: :deployment_deleted, - }) - |> Repo.update() - end - end - - defp down(inst), do: {:ok, inst} - - defp finalize(%ConsoleInstance{status: :provisioned} = inst, :up), do: {:ok, inst} - - defp finalize(%ConsoleInstance{status: :database_deleted, cluster: cluster, postgres: pg} = inst, :down) do - start_transaction() - |> add_operation(:inst, fn _ -> Repo.delete(inst) end) - |> add_operation(:cluster, fn _ -> Cloud.dec(cluster) end) - |> add_operation(:pg, fn _ -> Cloud.dec(pg) end) - |> add_operation(:sa, fn %{inst: %{name: name}} -> - case Users.get_user_by_email("#{name}-cloud-sa@srv.plural.sh") do - %User{} = u -> Repo.delete(u) - _ -> {:ok, nil} - end - end) - |> execute(extract: :inst) - end - - defp finalize(%ConsoleInstance{type: :dedicated} = inst, :down) do - start_transaction() - |> add_operation(:inst, fn _ -> Repo.delete(inst) end) - |> add_operation(:sa, fn %{inst: %{name: name}} -> - case Users.get_user_by_email("#{name}-cloud-sa@srv.plural.sh") do - %User{} = u -> Repo.delete(u) - _ -> {:ok, nil} - end - end) - |> execute(extract: :inst) - end - - defp finalize(inst, _) do - Logger.warn "failed to finalize console instance: #{inst.id}" - {:ok, inst} - end - - defp connect(%PostgresCluster{} = roach) do - uri = URI.parse(roach.url) - user = userinfo(uri) - Postgrex.start_link( - database: uri.path && String.trim_leading(uri.path, "/"), - username: user[:username], - password: user[:password], - hostname: uri.host, - port: uri.port, - ssl: Core.conf(:bootstrap_ssl) - ) - end - - defp userinfo(%URI{userinfo: info}) when is_binary(info) do - case String.split(info, ":") do - [user, pwd] -> %{username: user, password: pwd} - [user] -> %{username: user} - _ -> %{} - end - end - defp userinfo(_), do: %{} - - defp mark_provisioned(inst) do - ConsoleInstance.changeset(inst, %{status: :provisioned}) - |> Repo.update() - end + defp up(%ConsoleInstance{type: :dedicated} = inst), do: Dedicated.up(inst) + defp up(inst), do: Shared.up(inst) - defp console(), do: Console.new(Core.conf(:console_url), Core.conf(:console_token)) + defp down(%ConsoleInstance{type: :dedicated} = inst), do: Dedicated.down(inst) + defp down(inst), do: Shared.down(inst) - defp dedicated_console(), do: Console.new(Core.conf(:console_url), Core.conf(:dedicated_console_token)) + defp finalize(%ConsoleInstance{type: :dedicated} = inst, direction), do: Dedicated.finalize(inst, direction) + defp finalize(%ConsoleInstance{} = inst, direction), do: Shared.finalize(inst, direction) end diff --git a/apps/core/lib/core/services/cloud/workflow/dedicated.ex b/apps/core/lib/core/services/cloud/workflow/dedicated.ex new file mode 100644 index 000000000..c6736e864 --- /dev/null +++ b/apps/core/lib/core/services/cloud/workflow/dedicated.ex @@ -0,0 +1,79 @@ +defmodule Core.Services.Cloud.Workflow.Dedicated do + use Core.Services.Base + import Core.Services.Cloud.Utils + + alias Core.Clients.Console + alias Core.Services.{Users} + alias Core.Services.Cloud.{Poller, Configuration} + alias Core.Schema.{ConsoleInstance, User} + alias Core.Repo + + require Logger + + @behaviour Core.Services.Cloud.Workflow + + def sync(%ConsoleInstance{external_id: id} = inst) when is_binary(id) do + with {:ok, project_id} <- Poller.project(), + {:ok, repo_id} <- Poller.repository(), + {:ok, actor} <- Console.me(dedicated_console()), + attrs = %{actor_id: actor, project_id: project_id, repository_id: repo_id}, + do: Console.update_stack(dedicated_console(), id, Configuration.stack_attributes(inst, attrs)) + end + + def sync(_), do: :ok + + def up(%ConsoleInstance{status: :pending} = inst) do + with {:ok, id} <- Poller.project(), + {:ok, repo_id} <- Poller.repository(), + {:ok, actor} <- Console.me(dedicated_console()), + attrs = %{actor_id: actor, project_id: id, repository_id: repo_id}, + {:ok, stack_id} <- Console.create_stack(dedicated_console(), Configuration.stack_attributes(inst, attrs)) do + ConsoleInstance.changeset(inst, %{ + instance_status: %{stack: true}, + status: :stack_created, + external_id: stack_id + }) + |> Repo.update() + end + end + + def up(%ConsoleInstance{status: :stack_created, external_id: id} = inst) do + Enum.reduce_while(0..120, inst, fn _, inst -> + dedicated_console() + |> Console.stack(id) + |> case do + {:ok, %{"status" => "SUCCESSFUL"}} -> + {:halt, mark_provisioned(inst)} + status -> + Logger.info "stack not ready yet, sleeping: #{inspect(status)}" + :timer.sleep(:timer.minutes(1)) + {:cont, inst} + end + end) + end + + def up(inst), do: {:ok, inst} + + def down(%ConsoleInstance{instance_status: %{stack: true}, external_id: id} = inst) do + with {:ok, _} <- Console.delete_stack(dedicated_console(), id) do + ConsoleInstance.changeset(inst, %{status: :stack_deleted}) + |> Repo.update() + end + end + + def down(inst), do: {:ok, inst} + + def finalize(%ConsoleInstance{} = inst, :down) do + start_transaction() + |> add_operation(:inst, fn _ -> Repo.delete(inst) end) + |> add_operation(:sa, fn %{inst: %{name: name}} -> + case Users.get_user_by_email("#{name}-cloud-sa@srv.plural.sh") do + %User{} = u -> Repo.delete(u) + _ -> {:ok, nil} + end + end) + |> execute(extract: :inst) + end + + def finalize(inst, _), do: {:ok, inst} +end diff --git a/apps/core/lib/core/services/cloud/workflow/shared.ex b/apps/core/lib/core/services/cloud/workflow/shared.ex new file mode 100644 index 000000000..4cf042771 --- /dev/null +++ b/apps/core/lib/core/services/cloud/workflow/shared.ex @@ -0,0 +1,135 @@ +defmodule Core.Services.Cloud.Workflow.Shared do + use Core.Services.Base + import Core.Services.Cloud.Utils + + alias Core.Clients.Console + alias Core.Services.{Cloud, Users} + alias Core.Services.Cloud.{Poller, Configuration} + alias Core.Schema.{ConsoleInstance, PostgresCluster, User} + alias Core.Repo + + @behaviour Core.Services.Cloud.Workflow + + def sync(%ConsoleInstance{external_id: id} = instance) when is_binary(id) do + instance = Repo.preload(instance, [:cluster, :postgres]) + Console.update_service(console(), id, %{ + configuration: Configuration.build(instance) + }) + end + + def sync(_), do: :ok + + def up(%ConsoleInstance{status: :deployment_created, url: url} = inst) do + case {DNS.resolve(url), DNS.resolve(url, :cname)} do + {{:ok, [_ | _]}, _} -> mark_provisioned(inst) + {_, {:ok, [_ | _]}} -> mark_provisioned(inst) + {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"} + end + end + + def up(%ConsoleInstance{status: :pending, postgres: pg, configuration: conf} = inst) do + with {:ok, pid} <- connect(pg), + {:ok, _} <- Postgrex.query(pid, "CREATE DATABASE #{conf.database}", []), + {:ok, _} <- Postgrex.transaction(pid, fn conn -> + Postgrex.query!(conn, "CREATE USER #{conf.dbuser} WITH PASSWORD '#{conf.dbpassword}'", []) + Postgrex.query!(conn, "GRANT ALL ON DATABASE #{conf.database} TO #{conf.dbuser}", []) + end) do + ConsoleInstance.changeset(inst, %{ + instance_status: %{db: true}, + status: :database_created, + }) + |> Repo.update() + end + end + + def up(%ConsoleInstance{instance_status: %{db: true}, name: name, cluster: cluster} = inst) do + with {:ok, id} <- Poller.repository(), + {:ok, svc_id} <- Console.create_service(console(), cluster.external_id, %{ + name: "console-cloud-#{name}", + namespace: "plrl-cloud-#{name}", + helm: %{ + url: "https://pluralsh.github.io/console", + chart: "console-rapid", + release: "console", + version: "x.x.x", + valuesFiles: ["console.yaml.liquid"] + }, + repository_id: id, + git: %{ref: "main", folder: "helm"}, + configuration: Configuration.build(inst), + }) do + ConsoleInstance.changeset(inst, %{ + external_id: svc_id, + instance_status: %{svc: true}, + status: :deployment_created + }) + |> Repo.update() + end + end + + def up(inst), do: {:ok, inst} + + def down(%ConsoleInstance{instance_status: %{svc: false, db: true}, configuration: conf, postgres: pg} = inst) do + with {:ok, pid} <- connect(pg), + {:ok, _} <- Postgrex.query(pid, "DROP DATABASE IF EXISTS #{conf.database}", []), + {:ok, _} <- Postgrex.transaction(pid, fn conn -> + Postgrex.query!(conn, "DROP USER IF EXISTS #{conf.dbuser}", []) + end) do + ConsoleInstance.changeset(inst, %{ + instance_status: %{db: false}, + status: :database_deleted, + }) + |> Repo.update() + end + end + + def down(%ConsoleInstance{instance_status: %{svc: true}} = inst) do + with {:ok, _} <- Console.delete_service(console(), inst.external_id) do + ConsoleInstance.changeset(inst, %{ + instance_status: %{svc: false, db: true}, + status: :deployment_deleted, + }) + |> Repo.update() + end + end + + def down(inst), do: {:ok, inst} + + def finalize(%ConsoleInstance{status: :database_deleted, cluster: cluster, postgres: pg} = inst, :down) do + start_transaction() + |> add_operation(:inst, fn _ -> Repo.delete(inst) end) + |> add_operation(:cluster, fn _ -> Cloud.dec(cluster) end) + |> add_operation(:pg, fn _ -> Cloud.dec(pg) end) + |> add_operation(:sa, fn %{inst: %{name: name}} -> + case Users.get_user_by_email("#{name}-cloud-sa@srv.plural.sh") do + %User{} = u -> Repo.delete(u) + _ -> {:ok, nil} + end + end) + |> execute(extract: :inst) + end + + def finalize(inst, _), do: {:ok, inst} + + defp connect(%PostgresCluster{} = roach) do + uri = URI.parse(roach.url) + user = userinfo(uri) + Postgrex.start_link( + database: uri.path && String.trim_leading(uri.path, "/"), + username: user[:username], + password: user[:password], + hostname: uri.host, + port: uri.port, + ssl: Core.conf(:bootstrap_ssl) + ) + end + + defp userinfo(%URI{userinfo: info}) when is_binary(info) do + case String.split(info, ":") do + [user, pwd] -> %{username: user, password: pwd} + [user] -> %{username: user} + _ -> %{} + end + end + defp userinfo(_), do: %{} +end diff --git a/apps/core/test/services/cloud/workflow_test.exs b/apps/core/test/services/cloud/workflow_test.exs index 4c006841c..766108830 100644 --- a/apps/core/test/services/cloud/workflow_test.exs +++ b/apps/core/test/services/cloud/workflow_test.exs @@ -1,6 +1,7 @@ defmodule Core.Services.Cloud.WorkflowTest do use Core.SchemaCase, async: true use Mimic + alias Core.Clients.Console alias Core.Services.{Cloud, Cloud.Workflow} describe "up and down" do @@ -47,5 +48,63 @@ defmodule Core.Services.Cloud.WorkflowTest do assert refetch(roach).count == 0 assert refetch(cluster).count == 0 end + + test "it can handle setup and teardown of a dedicated cloud instance" do + account = insert(:account) + enterprise_plan(account) + user = admin_user(account) + insert(:repository, name: "console") + + expect(HTTPoison, :post, fn _, _, _ -> + {:ok, %{status_code: 200, body: Jason.encode!(%{client_id: "123", client_secret: "secret"})}} + end) + + {:ok, instance} = Cloud.create_instance(%{ + name: "plrltest", + cloud: :aws, + region: "us-east-1", + size: :small, + type: :dedicated + }, user) + + expect(Core.Services.Cloud.Poller, :repository, fn -> {:ok, "repo-id"} end) + expect(Core.Services.Cloud.Poller, :project, fn -> {:ok, "proj-id"} end) + stack_id = Ecto.UUID.generate() + stack_q = Console.queries(:stack_q) + me_q = Console.queries(:me_q) + expect(Req, :post, 3, fn + _, [graphql: {^me_q, _}] -> {:ok, %Req.Response{status: 200, body: %{"data" => %{"me" => %{"id" => "me-id"}}}}} + _, [graphql: {_, %{attributes: attrs}}] -> + send self(), {:attributes, attrs} + {:ok, %Req.Response{status: 200, body: %{"data" => %{"createStack" => %{"id" => stack_id}}}}} + _, [graphql: {^stack_q, %{id: ^stack_id}}] -> + {:ok, %Req.Response{ + status: 200, + body: %{"data" => %{"infrastructureStack" => %{"id" => stack_id, "status" => "SUCCESSFUL"}}} + }} + end) + + {:ok, %{external_id: stack_id} = instance} = Workflow.provision(instance) + + assert instance.status == :provisioned + assert instance.instance_status.stack + + assert_receive {:attributes, attrs} + + assert attrs.project_id == "proj-id" + assert attrs.actor_id == "me-id" + assert attrs.repository_id == "repo-id" + assert attrs.git.ref == "main" + assert attrs.git.folder == "terraform/modules/dedicated/aws" + + del_q = Console.queries(:stack_delete) + expect(Req, :post, fn _, [graphql: {^del_q, %{id: ^stack_id}}] -> + {:ok, %Req.Response{status: 200, body: %{"data" => %{"deleteStack" => %{"id" => stack_id}}}}} + end) + + {:ok, instance} = Workflow.deprovision(instance) + + refute refetch(instance) + end end end