check last_fetched_at in source processor

fastrepl · Oct 8, 2024 · 8ddd803 · 8ddd803
1 parent 46d641b
commit 8ddd803
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/core/lib/canary/workers/source_processor.ex b/core/lib/canary/workers/source_processor.ex
@@ -18,10 +18,14 @@ defmodule Canary.Workers.SourceProcessor do
   def perform(%Oban.Job{}) do
     sources =
       Source
-      |> Ash.Query.select([:id, :config])
+      |> Ash.Query.select([:id, :config, :last_fetched_at])
       |> Ash.read!()
 
     sources
+    |> Enum.filter(fn
+      %Source{last_fetched_at: nil} -> false
+      %Source{last_fetched_at: time} -> DateTime.diff(DateTime.utc_now(), time, :hour) > 18
+    end)
     |> Enum.map(fn %Source{id: id, config: %Ash.Union{type: type}} ->
       case type do
         :webpage -> Workers.WebpageProcessor.new(%{source_id: id})