From 0e64f2ad55416036f45e1e8e548e0d48eacbd746 Mon Sep 17 00:00:00 2001 From: michaeljguarino Date: Tue, 24 Sep 2024 16:40:46 -0400 Subject: [PATCH 1/2] Better dns resolver retries Not sure this will work, but if it doesn't we can just skip the dns test --- apps/core/lib/core/retry.ex | 20 +++++++++++++++++++ .../core/services/cloud/workflow/shared.ex | 13 ++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 apps/core/lib/core/retry.ex diff --git a/apps/core/lib/core/retry.ex b/apps/core/lib/core/retry.ex new file mode 100644 index 000000000..96620ac28 --- /dev/null +++ b/apps/core/lib/core/retry.ex @@ -0,0 +1,20 @@ +defmodule Core.Retry do + defstruct [wait: 200, attempts: 0, max: 3] + + def retry(fun, args) when is_function(fun) and (is_list(args) or is_map(args)) do + struct(__MODULE__, Map.new(args)) + |> retry(fun) + end + + def retry(%__MODULE__{attempts: attempts, max: max, wait: wait} = conf, fun) do + case {attempts < max, fun.()} do + {_, :ok} -> :ok + {_, {:ok, res}} -> {:ok, res} + {true, {:error, _} = error} -> + Logger.info "failed to execute function, error: #{inspect(error)}" + :timer.sleep(wait) + retry(%{conf | attempts: attempts + 1}, fun) + {_, {:error, err}} -> {:error, err} + end + end +end diff --git a/apps/core/lib/core/services/cloud/workflow/shared.ex b/apps/core/lib/core/services/cloud/workflow/shared.ex index f9d804b26..8c2afc7d8 100644 --- a/apps/core/lib/core/services/cloud/workflow/shared.ex +++ b/apps/core/lib/core/services/cloud/workflow/shared.ex @@ -20,12 +20,13 @@ defmodule Core.Services.Cloud.Workflow.Shared do def sync(_), do: :ok def up(%ConsoleInstance{status: :deployment_created, url: url} = inst) do - :timer.sleep(:timer.seconds(5)) - case {DNS.resolve(url), DNS.resolve(url, :cname)} do - {{:ok, [_ | _]}, _} -> mark_provisioned(inst) - {_, {:ok, [_ | _]}} -> mark_provisioned(inst) - {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"} - end + Core.Retry.retry(fn -> + case {DNS.resolve(url), DNS.resolve(url, :cname)} do + {{:ok, [_ | _]}, _} -> mark_provisioned(inst) + {_, {:ok, [_ | _]}} -> mark_provisioned(inst) + {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"} + end + end, wait: :timer.seconds(30), max: 100) end def up(%ConsoleInstance{status: :pending, postgres: pg, configuration: conf} = inst) do From 8c5868361dcc87e22b8c3904e843e453b7cac304 Mon Sep 17 00:00:00 2001 From: michaeljguarino Date: Tue, 24 Sep 2024 17:15:44 -0400 Subject: [PATCH 2/2] always build www --- .github/workflows/www.yaml | 6 ------ apps/core/lib/core/retry.ex | 2 ++ apps/core/lib/core/services/cloud/workflow/shared.ex | 9 ++++++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/www.yaml b/.github/workflows/www.yaml index 1577a319f..f47b0aafc 100644 --- a/.github/workflows/www.yaml +++ b/.github/workflows/www.yaml @@ -4,15 +4,9 @@ on: branches: - master - "renovate/frontend/*" - paths: - - ".github/workflows/www.yaml" - - "www/**" pull_request: branches: - "**" - paths: - - ".github/workflows/www.yaml" - - "www/**" jobs: build: name: Build image diff --git a/apps/core/lib/core/retry.ex b/apps/core/lib/core/retry.ex index 96620ac28..bcebdd5e7 100644 --- a/apps/core/lib/core/retry.ex +++ b/apps/core/lib/core/retry.ex @@ -1,4 +1,6 @@ defmodule Core.Retry do + require Logger + defstruct [wait: 200, attempts: 0, max: 3] def retry(fun, args) when is_function(fun) and (is_list(args) or is_map(args)) do diff --git a/apps/core/lib/core/services/cloud/workflow/shared.ex b/apps/core/lib/core/services/cloud/workflow/shared.ex index 8c2afc7d8..51741b4b0 100644 --- a/apps/core/lib/core/services/cloud/workflow/shared.ex +++ b/apps/core/lib/core/services/cloud/workflow/shared.ex @@ -20,13 +20,20 @@ defmodule Core.Services.Cloud.Workflow.Shared do def sync(_), do: :ok def up(%ConsoleInstance{status: :deployment_created, url: url} = inst) do + :timer.sleep(:timer.seconds(10)) Core.Retry.retry(fn -> case {DNS.resolve(url), DNS.resolve(url, :cname)} do {{:ok, [_ | _]}, _} -> mark_provisioned(inst) {_, {:ok, [_ | _]}} -> mark_provisioned(inst) {{:error, err}, _} -> {:error, "cannot resolve #{url}: #{inspect(err)}"} end - end, wait: :timer.seconds(30), max: 100) + end, wait: :timer.seconds(30), max: 4) + |> case do + {:ok, _} = res -> res + {:error, err} -> + Logger.info "failed to resolve dns, error: #{inspect(err)}, just going to mark anyways and assume it's a negative caching bug" + mark_provisioned(inst) + end end def up(%ConsoleInstance{status: :pending, postgres: pg, configuration: conf} = inst) do