From 05805bbfe762b8860cf9a78a165b2ad36be4fb2e Mon Sep 17 00:00:00 2001
From: michaeljguarino <mjg@plural.sh>
Date: Fri, 30 Aug 2024 17:08:39 -0400
Subject: [PATCH] Query DNS before finalizing cloud instances

There's some lag to externaldns registering for an ingress that could cause user-facing goofiness, just query dns until it resolves to wait for that to dissipate.
---
 apps/core/lib/core/schema/console_instance.ex |  4 ++++
 apps/core/lib/core/services/cloud/workflow.ex | 24 +++++++++++++------
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/apps/core/lib/core/schema/console_instance.ex b/apps/core/lib/core/schema/console_instance.ex
index 7df847d6c..80efe5903 100644
--- a/apps/core/lib/core/schema/console_instance.ex
+++ b/apps/core/lib/core/schema/console_instance.ex
@@ -86,6 +86,10 @@ defmodule Core.Schema.ConsoleInstance do
     )
   end
 
+  def provisioned(query \\ __MODULE__) do
+    from(c in query, where: c.status == ^:provisioned)
+  end
+
   def ordered(query \\ __MODULE__, order \\ [asc: :name]) do
     from(c in query, order_by: ^order)
   end
diff --git a/apps/core/lib/core/services/cloud/workflow.ex b/apps/core/lib/core/services/cloud/workflow.ex
index 4e50fe48c..98e6cd63c 100644
--- a/apps/core/lib/core/services/cloud/workflow.ex
+++ b/apps/core/lib/core/services/cloud/workflow.ex
@@ -21,7 +21,7 @@ defmodule Core.Services.Cloud.Workflow do
 
     Enum.reduce_while(0..10, instance, fn _, acc ->
       case up(acc) do
-        {:ok, %ConsoleInstance{status: :deployment_created} = inst} -> {:halt, inst}
+        {:ok, %ConsoleInstance{status: :provisioned} = inst} -> {:halt, inst}
         {:ok, inst} -> {:cont, inst}
         err ->
           :timer.sleep(:timer.seconds(1))
@@ -35,13 +35,13 @@ defmodule Core.Services.Cloud.Workflow do
   def deprovision(%ConsoleInstance{} = instance) do
     instance = Repo.preload(instance, [:postgres, :cluster])
 
-    Enum.reduce_while(0..10, instance, fn _, acc ->
+    Enum.reduce_while(0..20, instance, fn _, acc ->
       case down(acc) do
         {:ok, %ConsoleInstance{status: :pending} = inst} -> {:halt, inst}
         {:ok, %ConsoleInstance{status: :database_deleted} = inst} -> {:halt, inst}
         {:ok, inst} -> {:cont, inst}
         err ->
-          :timer.sleep(:timer.seconds(1))
+          :timer.sleep(:timer.seconds(10))
           Logger.error "failed to transition deprovisioning console: #{inspect(err)}"
           {:cont, acc}
       end
@@ -49,6 +49,14 @@ defmodule Core.Services.Cloud.Workflow do
     |> finalize(:down)
   end
 
+  defp up(%ConsoleInstance{status: :deployment_created, url: url} = inst) do
+    case {DNS.resolve(url), DNS.resolve(url, :cname)} do
+      {{:ok, [_ | _]}, _} -> mark_provisioned(inst)
+      {_, {:ok, [_ | _]}} -> mark_provisioned(inst)
+      {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"}
+    end
+  end
+
   defp up(%ConsoleInstance{status: :pending, postgres: pg, configuration: conf} = inst) do
     with {:ok, pid} <- connect(pg),
          {:ok, _} <- Postgrex.query(pid, "CREATE DATABASE #{conf.database}", []),
@@ -115,10 +123,7 @@ defmodule Core.Services.Cloud.Workflow do
 
   defp down(inst), do: {:ok, inst}
 
-  defp finalize(%ConsoleInstance{status: :deployment_created} = inst, :up) do
-    ConsoleInstance.changeset(inst, %{status: :provisioned})
-    |> Repo.update()
-  end
+  defp finalize(%ConsoleInstance{status: :provisioned} = inst, :up), do: {:ok, inst}
 
   defp finalize(%ConsoleInstance{status: :database_deleted, cluster: cluster, postgres: pg} = inst, :down) do
     start_transaction()
@@ -161,5 +166,10 @@ defmodule Core.Services.Cloud.Workflow do
   end
   defp userinfo(_), do: %{}
 
+  defp mark_provisioned(inst) do
+    ConsoleInstance.changeset(inst, %{status: :provisioned})
+    |> Repo.update()
+  end
+
   defp console(), do: Console.new(Core.conf(:console_url), Core.conf(:console_token))
 end