chore (sync service): setup sentry (#2099)

This PR configures Electric to track errors with sentry.
To enable sentry for the single tenant Electric app you need to pass a
`SENTRY_DSN` env var.

I've tested this with the cloud deploy and errors are tracked
successfully in sentry.

TODO:
- [ ] add stack_id and request_id meta data to the recorded error events
(*)

(*) The easy way is to configure sentry to add the `stack_id` and
`request_id` meta data to the event context. That way it will show up in
the error's details but it won't be searchable filterable on those
fields! So a better way would be to add that meta data as tags. But not
sure if that is configurable out of the box, might need to write a
custom LoggerHandler.

Adding the meta data as tags is possible by extending all
`Logger.metadata` calls (from @icehaunter's recent PR) with a `sentry:
[tags: %{stack_id: stack_id, shape_handle: shape_handle}]` key. Cf.
https://github.com/getsentry/sentry-elixir/blob/master/lib/sentry/logger_handler.ex#L205
for more info.

---------

Co-authored-by: Ilia Borovitinov <[email protected]>

.changeset/cuddly-toys-deliver.md

@@ -0,0 +1,5 @@
+---
+"@core/sync-service": patch
+---
+
+Configure sentry for error tracking.

packages/sync-service/Dockerfile

@@ -28,11 +28,12 @@ COPY rel /app/rel
 COPY lib /app/lib/
 
 COPY package.json /app/
-COPY config/*runtime.exs /app/config/
+COPY config/*.exs /app/config/
 
 ARG ELECTRIC_VERSION
 
 RUN mix compile
+RUN mix sentry.package_source_code
 RUN mix release
 
 FROM ${RUNNER_IMAGE} AS runner_setup

packages/sync-service/config/config.exs

@@ -0,0 +1,7 @@
+import Config
+
+# Sentry's source-context-related options need to be set in compile-time config files
+# cf. https://hexdocs.pm/sentry/Mix.Tasks.Sentry.PackageSourceCode.html
+config :sentry,
+  enable_source_code_context: true,
+  root_source_code_paths: [File.cwd!()]

packages/sync-service/config/runtime.exs

@@ -31,6 +31,17 @@ if config_env() == :test do
   config :logger, :default_handler, level: :error
 end
 
+config :sentry,
+  environment_name: config_env(),
+  client: Electric.Telemetry.SentryReqHTTPClient
+
+sentry_dsn = env!("SENTRY_DSN", :string, nil)
+
+if !is_nil(sentry_dsn) do
+  config :sentry,
+    dsn: sentry_dsn
+end
+
 service_name = env!("ELECTRIC_SERVICE_NAME", :string, "electric")
 instance_id = env!("ELECTRIC_INSTANCE_ID", :string, Electric.Utils.uuid4())
 version = Electric.version()

packages/sync-service/lib/electric/application.ex

@@ -12,6 +12,8 @@ defmodule Electric.Application do
   def start(_type, _args) do
     :erlang.system_flag(:backtrace_depth, 50)
 
+    Electric.Telemetry.Sentry.add_logger_handler()
+
     # We have "instance id" identifier as the node ID, however that's generated every runtime,
     # so isn't stable across restarts. Our storages however scope themselves based on this stack ID
     # so we're just hardcoding it here.

packages/sync-service/lib/electric/plug/router.ex

@@ -1,5 +1,6 @@
 defmodule Electric.Plug.Router do
   use Plug.Router, copy_opts_to_assign: :config
+  use Sentry.PlugCapture
 
   alias Electric.Plug.Utils.CORSHeaderPlug
   alias Electric.Plug.Utils.PassAssignToOptsPlug
@@ -15,6 +16,7 @@ defmodule Electric.Plug.Router do
   plug Electric.Plug.TraceContextPlug
   plug Plug.Telemetry, event_prefix: [:electric, :routing]
   plug Plug.Logger
+  plug Sentry.PlugContext
   plug :put_cors_headers
   plug :dispatch
 

packages/sync-service/lib/electric/telemetry/sentry.ex

@@ -0,0 +1,7 @@
+defmodule Electric.Telemetry.Sentry do
+  def add_logger_handler do
+    :logger.add_handler(:electric_sentry_handler, Sentry.LoggerHandler, %{
+      config: %{metadata: :all}
+    })
+  end
+end

packages/sync-service/lib/electric/telemetry/sentry_req_http_client.ex

@@ -0,0 +1,17 @@
+defmodule Electric.Telemetry.SentryReqHTTPClient do
+  @moduledoc """
+  A custom Sentry HTTP client implementation using Req.
+  """
+  @behaviour Sentry.HTTPClient
+
+  @impl true
+  def post(url, headers, body) do
+    case Req.post(url, headers: headers, body: body, decode_body: false) do
+      {:ok, %Req.Response{status: status, headers: headers, body: body}} ->
+        {:ok, status, headers |> Enum.into([], fn {k, [v]} -> {k, v} end), body}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+end

packages/sync-service/mix.exs

@@ -92,6 +92,7 @@ defmodule Electric.MixProject do
         {:retry, "~> 0.18"},
         {:remote_ip, "~> 1.2"},
         {:req, "~> 0.5"},
+        {:sentry, "~> 10.0"},
         {:telemetry_metrics_prometheus_core, "~> 1.1"},
         {:telemetry_metrics_statsd, "~> 0.7"},
         {:telemetry_poller, "~> 1.1"},

packages/sync-service/mix.lock

@@ -30,6 +30,7 @@
   "mint": {:hex, :mint, "1.6.2", "af6d97a4051eee4f05b5500671d47c3a67dac7386045d87a904126fd4bbcea2e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5ee441dffc1892f1ae59127f74afe8fd82fda6587794278d924e4d90ea3d63f9"},
   "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"},
   "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
+  "nimble_ownership": {:hex, :nimble_ownership, "1.0.0", "3f87744d42c21b2042a0aa1d48c83c77e6dd9dd357e425a038dd4b49ba8b79a1", [:mix], [], "hexpm", "7c16cc74f4e952464220a73055b557a273e8b1b7ace8489ec9d86e9ad56cb2cc"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
   "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
   "opentelemetry": {:hex, :opentelemetry, "1.5.0", "7dda6551edfc3050ea4b0b40c0d2570423d6372b97e9c60793263ef62c53c3c2", [:rebar3], [{:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "cdf4f51d17b592fc592b9a75f86a6f808c23044ba7cf7b9534debbcc5c23b0ee"},
@@ -45,6 +46,7 @@
   "remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"},
   "req": {:hex, :req, "0.5.7", "b722680e03d531a2947282adff474362a48a02aa54b131196fbf7acaff5e4cee", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "c6035374615120a8923e8089d0c21a3496cf9eda2d287b806081b8f323ceee29"},
   "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"},
+  "sentry": {:hex, :sentry, "10.8.0", "1e8cc0ef21401e5914e6fc2f37489d6c685d31a0556dbd8ab4709cc1587a7232", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0 or ~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "92549e7ba776b7ccfed4e74d58987272d37d99606b130e4141bc015a1a8e4235"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"},