Limit entries count on term workorders search (#1499)

* Limit entries count on term workorders search

* Extra: declare pagination_bar attrs

* provide some help text for large string search

---------

Co-authored-by: Taylor Downs <[email protected]>

CHANGELOG.md

@@ -12,6 +12,9 @@ and this project adheres to
 
 ### Changed
 
+- Limit entries count on term work orders search
+  [#1461](https://github.com/OpenFn/Lightning/issues/1461)
+
 ### Fixed
 
 - Use checkbox on boolean credential fields

lib/lightning/invocation.ex

@@ -14,6 +14,9 @@ defmodule Lightning.Invocation do
   alias Lightning.Projects.Project
 
   @workorders_search_timeout 30_000
+  @workorders_count_limit 50
+
+  def get_workorders_count_limit, do: @workorders_count_limit
 
   @doc """
   Returns the list of dataclips.
@@ -312,16 +315,23 @@ defmodule Lightning.Invocation do
 
   def search_workorders(
         %Project{} = project,
-        %SearchParams{} = search_params,
+        %SearchParams{search_term: search_term} = search_params,
         params \\ %{}
       ) do
     params =
       update_in(
         params,
         [:options],
-        fn
-          nil -> [timeout: @workorders_search_timeout]
-          options -> options
+        fn options ->
+          [timeout: @workorders_search_timeout]
+          |> Keyword.merge(options || [])
+          |> then(fn options ->
+            if search_term do
+              Keyword.put(options, :limit, @workorders_count_limit)
+            else
+              options
+            end
+          end)
         end
       )
 

lib/lightning/scrivener/atom_paginater.ex

@@ -0,0 +1,10 @@
+defimpl Scrivener.Paginater, for: Atom do
+  @moduledoc false
+
+  @spec paginate(atom, Scrivener.Config.t()) :: Scrivener.Page.t()
+  def paginate(atom, config) do
+    atom
+    |> Ecto.Queryable.to_query()
+    |> Scrivener.Paginater.paginate(config)
+  end
+end

lib/lightning/scrivener/query_paginater.ex

@@ -0,0 +1,140 @@
+defimpl Scrivener.Paginater, for: Ecto.Query do
+  import Ecto.Query
+
+  alias Scrivener.{Config, Page}
+
+  @moduledoc false
+
+  @spec paginate(Ecto.Query.t(), Scrivener.Config.t()) :: Scrivener.Page.t()
+  def paginate(query, %Config{
+        page_size: page_size,
+        page_number: page_number,
+        module: repo,
+        caller: caller,
+        options: options
+      }) do
+    total_entries =
+      Keyword.get(options, :total_entries) ||
+        total_entries(query, repo, caller, options)
+
+    total_pages = total_pages(total_entries, page_size)
+
+    allow_overflow_page_number =
+      Keyword.get(options, :allow_overflow_page_number, false)
+
+    page_number =
+      if allow_overflow_page_number,
+        do: page_number,
+        else: min(total_pages, page_number)
+
+    %Page{
+      page_size: page_size,
+      page_number: page_number,
+      entries:
+        entries(
+          query,
+          repo,
+          page_number,
+          total_pages,
+          page_size,
+          caller,
+          options
+        ),
+      total_entries: total_entries,
+      total_pages: total_pages
+    }
+  end
+
+  defp entries(_, _, page_number, total_pages, _, _, _)
+       when page_number > total_pages,
+       do: []
+
+  defp entries(query, repo, page_number, _, page_size, caller, options) do
+    offset =
+      Keyword.get_lazy(options, :offset, fn -> page_size * (page_number - 1) end)
+
+    opts = Keyword.take(options, [:prefix, :timeout])
+
+    query
+    |> offset(^offset)
+    |> limit(^page_size)
+    |> all(repo, caller, opts)
+  end
+
+  defp total_entries(query, repo, caller, options) do
+    prefix = options[:prefix]
+    limit = options[:limit]
+
+    total_entries =
+      query
+      |> exclude(:preload)
+      |> exclude(:order_by)
+      |> limit_count(limit)
+      |> aggregate()
+      |> one(repo, caller, prefix)
+
+    total_entries || 0
+  end
+
+  defp limit_count(query, nil), do: query
+  defp limit_count(query, limit), do: query |> limit(^limit)
+
+  defp aggregate(%{distinct: %{expr: expr}} = query)
+       when expr == true or is_list(expr) do
+    query
+    |> exclude(:select)
+    |> count()
+  end
+
+  defp aggregate(
+         %{
+           group_bys: [
+             %Ecto.Query.QueryExpr{
+               expr: [
+                 {{:., [], [{:&, [], [source_index]}, field]}, [], []} | _
+               ]
+             }
+             | _
+           ]
+         } = query
+       ) do
+    query
+    |> exclude(:select)
+    |> select([{x, source_index}], struct(x, ^[field]))
+    |> count()
+  end
+
+  defp aggregate(query) do
+    query
+    |> exclude(:select)
+    |> select(count("*"))
+  end
+
+  defp count(query) do
+    query
+    |> subquery
+    |> select(count("*"))
+  end
+
+  defp total_pages(0, _), do: 1
+
+  defp total_pages(total_entries, page_size) do
+    (total_entries / page_size) |> Float.ceil() |> round
+  end
+
+  defp all(query, repo, caller, []) do
+    repo.all(query, caller: caller)
+  end
+
+  defp all(query, repo, caller, opts) do
+    repo.all(query, Keyword.put(opts, :caller, caller))
+  end
+
+  defp one(query, repo, caller, nil) do
+    repo.one(query, caller: caller)
+  end
+
+  defp one(query, repo, caller, prefix) do
+    repo.one(query, caller: caller, prefix: prefix)
+  end
+end

lib/lightning_web/live/run_live/index.ex

@@ -92,6 +92,7 @@ defmodule LightningWeb.RunLive.Index do
        workflows: workflows,
        statuses: statuses,
        search_fields: search_fields,
+       string_search_limit: Invocation.get_workorders_count_limit(),
        active_menu_item: :runs,
        work_orders: [],
        selected_work_orders: [],

lib/lightning_web/live/run_live/index.html.heex

@@ -447,7 +447,14 @@
               <div>
                 <p class="text-xl text-gray-500 font-medium">
                   <%= if search = get_change(@filters_changeset, :search_term) do %>
-                    <%= @page.total_entries %> Work Orders with runs matching "<%= search %>"
+                    <%= @page.total_entries %><%= if @page.total_entries >=
+                                                       @string_search_limit,
+                                                     do: "+" %> work orders with runs matching "<%= search %>"
+                    <%= if @page.total_entries >= @string_search_limit do %>
+                      <span class="text-xs text-gray">
+                        (try more specific filters)
+                      </span>
+                    <% end %>
                   <% else %>
                     Work Orders
                   <% end %>

lib/lightning_web/pagination.ex

@@ -90,6 +90,8 @@ defmodule LightningWeb.Pagination do
   end
 
   attr :async_page, Phoenix.LiveView.AsyncResult, default: nil
+  attr :page, :map, required: true
+  attr :url, :any, required: true
 
   def pagination_bar(assigns) do
     ~H"""

mix.exs

@@ -105,8 +105,7 @@ defmodule Lightning.MixProject do
       {:postgrex, ">= 0.0.0"},
       {:prom_ex, "~> 1.8.0"},
       {:rambo, "~> 0.3.4"},
-      {:scrivener_ecto,
-       git: "https://github.com/OpenFn/scrivener_ecto.git", ref: "444b7505"},
+      {:scrivener, "~> 2.7"},
       {:sentry, "~> 8.0"},
       {:sobelow, "~> 0.13.0", only: [:test, :dev]},
       {:sweet_xml, "~> 0.7.1", only: [:test]},