Better dns resolver retries

.github/workflows/www.yaml

@@ -4,15 +4,9 @@ on:
     branches:
       - master
       - "renovate/frontend/*"
-    paths:
-      - ".github/workflows/www.yaml"
-      - "www/**"
   pull_request:
     branches:
       - "**"
-    paths:
-      - ".github/workflows/www.yaml"
-      - "www/**"
 jobs:
   build:
     name: Build image

apps/core/lib/core/retry.ex

@@ -0,0 +1,22 @@
+defmodule Core.Retry do
+  require Logger
+
+  defstruct [wait: 200, attempts: 0, max: 3]
+
+  def retry(fun, args) when is_function(fun) and (is_list(args) or is_map(args)) do
+    struct(__MODULE__, Map.new(args))
+    |> retry(fun)
+  end
+
+  def retry(%__MODULE__{attempts: attempts, max: max, wait: wait} = conf, fun) do
+    case {attempts < max, fun.()} do
+      {_, :ok} -> :ok
+      {_, {:ok, res}} -> {:ok, res}
+      {true, {:error, _} = error} ->
+        Logger.info "failed to execute function, error: #{inspect(error)}"
+        :timer.sleep(wait)
+        retry(%{conf | attempts: attempts + 1}, fun)
+      {_, {:error, err}} -> {:error, err}
+    end
+  end
+end

apps/core/lib/core/services/cloud/workflow/shared.ex

@@ -20,11 +20,19 @@ defmodule Core.Services.Cloud.Workflow.Shared do
   def sync(_), do: :ok
 
   def up(%ConsoleInstance{status: :deployment_created, url: url} = inst) do
-    :timer.sleep(:timer.seconds(5))
-    case {DNS.resolve(url), DNS.resolve(url, :cname)} do
-      {{:ok, [_ | _]}, _} -> mark_provisioned(inst)
-      {_, {:ok, [_ | _]}} -> mark_provisioned(inst)
-      {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"}
+    :timer.sleep(:timer.seconds(10))
+    Core.Retry.retry(fn ->
+      case {DNS.resolve(url), DNS.resolve(url, :cname)} do
+        {{:ok, [_ | _]}, _} -> mark_provisioned(inst)
+        {_, {:ok, [_ | _]}} -> mark_provisioned(inst)
+        {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"}
+      end
+    end, wait: :timer.seconds(30), max: 4)
+    |> case do
+      {:ok, _} = res -> res
+      {:error, err} ->
+        Logger.info "failed to resolve dns, error: #{inspect(err)}, just going to mark anyways and assume it's a negative caching bug"
+        mark_provisioned(inst)
     end
   end