feat(otel): setup otel

core/config/dev.exs

@@ -82,3 +82,5 @@ config :phoenix_live_view, :debug_heex_annotations, true
 config :swoosh, :api_client, false
 
 config :canary, :typesense, collection: "canary_dev"
+
+config :opentelemetry, traces_exporter: :none

core/config/prod.exs

@@ -25,6 +25,15 @@ config :canary, :github_app_url, "https://github.com/apps/getcanary/installation
 
 config :canary, :typesense, collection: "canary_prod"
 
+config :opentelemetry,
+  traces_exporter: :otlp,
+  span_processor: :batch,
+  resource: [
+    service: [name: "core", namespace: "canary"],
+    deployment: [environment: "prod"]
+  ],
+  sampler: {:parent_based, %{root: {:trace_id_ratio_based, 1.0}}}
+
 config :sentry,
   dsn:
     "https://f6f1f6ab1218132660f6bd80e8634712@o4506190168522752.ingest.us.sentry.io/4508073135833088",

core/config/runtime.exs

@@ -177,6 +177,9 @@ config :canary, :github,
   client_secret: System.get_env("GITHUB_CLIENT_SECRET"),
   redirect_uri: System.get_env("GITHUB_REDIRECT_URI")
 
+config :canary, :fly_api_key, System.get_env("FLY_API_KEY")
+config :canary, :fly_app_name, System.get_env("FLY_APP_NAME")
+
 if config_env() == :prod do
   config :canary, :typesense,
     base_url: System.get_env("TYPESENSE_BASE_URL"),
@@ -187,5 +190,9 @@ else
     api_key: "canary"
 end
 
-config :canary, :fly_api_key, System.get_env("FLY_API_KEY")
-config :canary, :fly_app_name, System.get_env("FLY_APP_NAME")
+if config_env() == :prod do
+  config :opentelemetry_exporter,
+    otlp_protocol: :http_protobuf,
+    otlp_endpoint: System.fetch_env!("OTEL_COLLECTOR_URL"),
+    otlp_headers: [{"Authorization", "Bearer #{System.fetch_env!("OTEL_COLLECTOR_URL_AUTH")}"}]
+end

core/lib/canary.ex

@@ -1,11 +1,21 @@
 defmodule Canary do
   def rest_client(opts \\ []) do
     Req.new(opts)
+    |> attach_otel()
   end
 
   def graphql_client(opts \\ []) do
     Req.new(opts)
+    |> attach_otel()
     |> AbsintheClient.attach()
     |> Req.Request.register_options([:graphql])
   end
+
+  defp attach_otel(req) do
+    req
+    |> OpentelemetryReq.attach(
+      propagate_trace_ctx: true,
+      no_path_params: true
+    )
+  end
 end

core/lib/canary/application.ex

@@ -9,6 +9,7 @@ defmodule Canary.Application do
   def start(_type, _args) do
     attach_oban_telemetry()
     add_sentry_logger()
+    OpentelemetryEcto.setup([:canary, :repo])
 
     :ok = Canary.Index.Collection.ensure(:webpage)
     :ok = Canary.Index.Collection.ensure(:github_issue)

core/mix.exs

@@ -10,7 +10,15 @@ defmodule Canary.MixProject do
       start_permanent: Mix.env() == :prod,
       aliases: aliases(),
       deps: deps(),
-      consolidate_protocols: Mix.env() == :prod
+      consolidate_protocols: Mix.env() == :prod,
+      releases: [
+        canary: [
+          applications: [
+            opentelemetry_exporter: :permanent,
+            opentelemetry: :temporary
+          ]
+        ]
+      ]
     ]
   end
 
@@ -92,7 +100,12 @@ defmodule Canary.MixProject do
       {:ex_json_schema, "~> 0.10"},
       {:yaml_elixir, "~> 2.11"},
       {:sentry, "~> 10.2.0"},
-      {:hackney, "~> 1.8"}
+      {:hackney, "~> 1.8"},
+      {:opentelemetry_exporter, "~> 1.2"},
+      {:opentelemetry, "~> 1.2"},
+      {:opentelemetry_api, "~> 1.2"},
+      {:opentelemetry_req, "~> 0.2.0"},
+      {:opentelemetry_ecto, "~> 1.0"}
     ] ++ deps_eval()
   end