Fix retry sorting and avoids loading dataclips (#2581)

CHANGELOG.md

@@ -26,6 +26,8 @@ and this project adheres to
 
 ### Fixed
 
+- Fix work order retry sorting and avoids loading dataclips
+  [#2581](https://github.com/OpenFn/lightning/issues/2581)
 - Fix editor panel overlays output panel when scrolled
   [#2291](https://github.com/OpenFn/lightning/issues/2291)
 

lib/lightning/work_orders.ex

@@ -447,8 +447,8 @@ defmodule Lightning.WorkOrders do
           | UsageLimiting.error()
           | {:error, :enqueue_error}
   def retry_many([%WorkOrder{} | _rest] = workorders, opts) do
-    {retriable_workorders, non_retriable_workorders} =
-      workorders |> Enum.map(& &1.id) |> fetch_and_split_workorders()
+    retriable_workorders =
+      workorders |> Enum.map(& &1.id) |> fetch_retriable_workorders()
 
     with project_id <- Keyword.fetch!(opts, :project_id),
          :ok <-
@@ -476,7 +476,7 @@ defmodule Lightning.WorkOrders do
       |> case do
         inserted_list when is_list(inserted_list) ->
           enqueued_count = length(retriable_workorders)
-          discarded_count = length(non_retriable_workorders)
+          discarded_count = length(workorders) - enqueued_count
 
           {:ok, enqueued_count, discarded_count}
 
@@ -669,11 +669,13 @@ defmodule Lightning.WorkOrders do
     run.starting_job || hd(workorder.trigger.edges).target_job
   end
 
-  defp fetch_and_split_workorders(workorder_ids) do
-    workorder_ids
-    |> workorders_with_dataclips_query()
+  defp fetch_retriable_workorders(workorder_ids) do
+    from(w in WorkOrder,
+      join: d in assoc(w, :dataclip),
+      where: w.id in ^workorder_ids and is_nil(d.wiped_at),
+      order_by: [asc: w.inserted_at]
+    )
     |> Repo.all()
-    |> Enum.split_with(&retriable?/1)
   end
 
   defp workorders_with_dataclips_query(workorder_ids) do
@@ -701,8 +703,4 @@ defmodule Lightning.WorkOrders do
       preload: [:starting_job, starting_trigger: [edges: :target_job]],
       limit: 1
   end
-
-  defp retriable?(workorder) do
-    workorder.dataclip != nil and is_nil(workorder.dataclip.wiped_at)
-  end
 end

mix.lock

@@ -147,7 +147,7 @@
   "swoosh": {:hex, :swoosh, "1.12.0", "ecc85ee12947932986243299b8d28e6cdfc192c8d9e24c4c64f6738efdf344cb", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "87db7ab0f35e358ba5eac3afc7422ed0c8c168a2d219d2a83ad8cb7a424f6cc9"},
   "table_rex": {:hex, :table_rex, "3.1.1", "0c67164d1714b5e806d5067c1e96ff098ba7ae79413cc075973e17c38a587caa", [:mix], [], "hexpm", "678a23aba4d670419c23c17790f9dcd635a4a89022040df7d5d772cb21012490"},
   "tailwind": {:hex, :tailwind, "0.2.2", "9e27288b568ede1d88517e8c61259bc214a12d7eed271e102db4c93fcca9b2cd", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "ccfb5025179ea307f7f899d1bb3905cd0ac9f687ed77feebc8f67bdca78565c4"},
-  "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
+  "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"},
   "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.1.0", "4e15f6d7dbedb3a4e3aed2262b7e1407f166fcb9c30ca3f96635dfbbef99965c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0dd10e7fe8070095df063798f82709b0a1224c31b8baf6278b423898d591a069"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"},