From ed8d5b2f4570a124dc7cf05b7d7ee23f2ad3ab07 Mon Sep 17 00:00:00 2001 From: Rogerio Pontual <44991200+jyeshe@users.noreply.github.com> Date: Sun, 3 Dec 2023 16:18:06 +0000 Subject: [PATCH] Limit entries count on term workorders search (#1499) * Limit entries count on term workorders search * Extra: declare pagination_bar attrs * provide some help text for large string search --------- Co-authored-by: Taylor Downs --- CHANGELOG.md | 3 + lib/lightning/invocation.ex | 18 ++- lib/lightning/scrivener/atom_paginater.ex | 10 ++ lib/lightning/scrivener/query_paginater.ex | 140 ++++++++++++++++++ lib/lightning_web/live/run_live/index.ex | 1 + .../live/run_live/index.html.heex | 9 +- lib/lightning_web/pagination.ex | 2 + mix.exs | 3 +- 8 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 lib/lightning/scrivener/atom_paginater.ex create mode 100644 lib/lightning/scrivener/query_paginater.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index b9d7030965..f8471afe83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ and this project adheres to ### Changed +- Limit entries count on term work orders search + [#1461](https://github.com/OpenFn/Lightning/issues/1461) + ### Fixed - Use checkbox on boolean credential fields diff --git a/lib/lightning/invocation.ex b/lib/lightning/invocation.ex index 344b5704c2..d0a5cc654e 100644 --- a/lib/lightning/invocation.ex +++ b/lib/lightning/invocation.ex @@ -14,6 +14,9 @@ defmodule Lightning.Invocation do alias Lightning.Projects.Project @workorders_search_timeout 30_000 + @workorders_count_limit 50 + + def get_workorders_count_limit, do: @workorders_count_limit @doc """ Returns the list of dataclips. @@ -312,16 +315,23 @@ defmodule Lightning.Invocation do def search_workorders( %Project{} = project, - %SearchParams{} = search_params, + %SearchParams{search_term: search_term} = search_params, params \\ %{} ) do params = update_in( params, [:options], - fn - nil -> [timeout: @workorders_search_timeout] - options -> options + fn options -> + [timeout: @workorders_search_timeout] + |> Keyword.merge(options || []) + |> then(fn options -> + if search_term do + Keyword.put(options, :limit, @workorders_count_limit) + else + options + end + end) end ) diff --git a/lib/lightning/scrivener/atom_paginater.ex b/lib/lightning/scrivener/atom_paginater.ex new file mode 100644 index 0000000000..b498f529d4 --- /dev/null +++ b/lib/lightning/scrivener/atom_paginater.ex @@ -0,0 +1,10 @@ +defimpl Scrivener.Paginater, for: Atom do + @moduledoc false + + @spec paginate(atom, Scrivener.Config.t()) :: Scrivener.Page.t() + def paginate(atom, config) do + atom + |> Ecto.Queryable.to_query() + |> Scrivener.Paginater.paginate(config) + end +end diff --git a/lib/lightning/scrivener/query_paginater.ex b/lib/lightning/scrivener/query_paginater.ex new file mode 100644 index 0000000000..b6e3f722dc --- /dev/null +++ b/lib/lightning/scrivener/query_paginater.ex @@ -0,0 +1,140 @@ +defimpl Scrivener.Paginater, for: Ecto.Query do + import Ecto.Query + + alias Scrivener.{Config, Page} + + @moduledoc false + + @spec paginate(Ecto.Query.t(), Scrivener.Config.t()) :: Scrivener.Page.t() + def paginate(query, %Config{ + page_size: page_size, + page_number: page_number, + module: repo, + caller: caller, + options: options + }) do + total_entries = + Keyword.get(options, :total_entries) || + total_entries(query, repo, caller, options) + + total_pages = total_pages(total_entries, page_size) + + allow_overflow_page_number = + Keyword.get(options, :allow_overflow_page_number, false) + + page_number = + if allow_overflow_page_number, + do: page_number, + else: min(total_pages, page_number) + + %Page{ + page_size: page_size, + page_number: page_number, + entries: + entries( + query, + repo, + page_number, + total_pages, + page_size, + caller, + options + ), + total_entries: total_entries, + total_pages: total_pages + } + end + + defp entries(_, _, page_number, total_pages, _, _, _) + when page_number > total_pages, + do: [] + + defp entries(query, repo, page_number, _, page_size, caller, options) do + offset = + Keyword.get_lazy(options, :offset, fn -> page_size * (page_number - 1) end) + + opts = Keyword.take(options, [:prefix, :timeout]) + + query + |> offset(^offset) + |> limit(^page_size) + |> all(repo, caller, opts) + end + + defp total_entries(query, repo, caller, options) do + prefix = options[:prefix] + limit = options[:limit] + + total_entries = + query + |> exclude(:preload) + |> exclude(:order_by) + |> limit_count(limit) + |> aggregate() + |> one(repo, caller, prefix) + + total_entries || 0 + end + + defp limit_count(query, nil), do: query + defp limit_count(query, limit), do: query |> limit(^limit) + + defp aggregate(%{distinct: %{expr: expr}} = query) + when expr == true or is_list(expr) do + query + |> exclude(:select) + |> count() + end + + defp aggregate( + %{ + group_bys: [ + %Ecto.Query.QueryExpr{ + expr: [ + {{:., [], [{:&, [], [source_index]}, field]}, [], []} | _ + ] + } + | _ + ] + } = query + ) do + query + |> exclude(:select) + |> select([{x, source_index}], struct(x, ^[field])) + |> count() + end + + defp aggregate(query) do + query + |> exclude(:select) + |> select(count("*")) + end + + defp count(query) do + query + |> subquery + |> select(count("*")) + end + + defp total_pages(0, _), do: 1 + + defp total_pages(total_entries, page_size) do + (total_entries / page_size) |> Float.ceil() |> round + end + + defp all(query, repo, caller, []) do + repo.all(query, caller: caller) + end + + defp all(query, repo, caller, opts) do + repo.all(query, Keyword.put(opts, :caller, caller)) + end + + defp one(query, repo, caller, nil) do + repo.one(query, caller: caller) + end + + defp one(query, repo, caller, prefix) do + repo.one(query, caller: caller, prefix: prefix) + end +end diff --git a/lib/lightning_web/live/run_live/index.ex b/lib/lightning_web/live/run_live/index.ex index e728f8fb2f..78bc63145d 100644 --- a/lib/lightning_web/live/run_live/index.ex +++ b/lib/lightning_web/live/run_live/index.ex @@ -92,6 +92,7 @@ defmodule LightningWeb.RunLive.Index do workflows: workflows, statuses: statuses, search_fields: search_fields, + string_search_limit: Invocation.get_workorders_count_limit(), active_menu_item: :runs, work_orders: [], selected_work_orders: [], diff --git a/lib/lightning_web/live/run_live/index.html.heex b/lib/lightning_web/live/run_live/index.html.heex index e579c7919e..5567521536 100644 --- a/lib/lightning_web/live/run_live/index.html.heex +++ b/lib/lightning_web/live/run_live/index.html.heex @@ -447,7 +447,14 @@

<%= if search = get_change(@filters_changeset, :search_term) do %> - <%= @page.total_entries %> Work Orders with runs matching "<%= search %>" + <%= @page.total_entries %><%= if @page.total_entries >= + @string_search_limit, + do: "+" %> work orders with runs matching "<%= search %>" + <%= if @page.total_entries >= @string_search_limit do %> + + (try more specific filters) + + <% end %> <% else %> Work Orders <% end %> diff --git a/lib/lightning_web/pagination.ex b/lib/lightning_web/pagination.ex index e50c4e3000..eb9d9e1d54 100644 --- a/lib/lightning_web/pagination.ex +++ b/lib/lightning_web/pagination.ex @@ -90,6 +90,8 @@ defmodule LightningWeb.Pagination do end attr :async_page, Phoenix.LiveView.AsyncResult, default: nil + attr :page, :map, required: true + attr :url, :any, required: true def pagination_bar(assigns) do ~H""" diff --git a/mix.exs b/mix.exs index 41f7aa4474..3b4ddd47ce 100644 --- a/mix.exs +++ b/mix.exs @@ -105,8 +105,7 @@ defmodule Lightning.MixProject do {:postgrex, ">= 0.0.0"}, {:prom_ex, "~> 1.8.0"}, {:rambo, "~> 0.3.4"}, - {:scrivener_ecto, - git: "https://github.com/OpenFn/scrivener_ecto.git", ref: "444b7505"}, + {:scrivener, "~> 2.7"}, {:sentry, "~> 8.0"}, {:sobelow, "~> 0.13.0", only: [:test, :dev]}, {:sweet_xml, "~> 0.7.1", only: [:test]},