From 81ecd7cbaa257b896cc1314cfe3d60ca6a812a71 Mon Sep 17 00:00:00 2001 From: Miu Razvan Date: Wed, 17 May 2023 16:24:40 +0200 Subject: [PATCH 01/14] Add facet count endpoint --- src/plone/restapi/services/configure.zcml | 1 + .../restapi/services/facet_count/__init__.py | 0 .../services/facet_count/configure.zcml | 16 ++++++ src/plone/restapi/services/facet_count/get.py | 54 +++++++++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 src/plone/restapi/services/facet_count/__init__.py create mode 100644 src/plone/restapi/services/facet_count/configure.zcml create mode 100644 src/plone/restapi/services/facet_count/get.py diff --git a/src/plone/restapi/services/configure.zcml b/src/plone/restapi/services/configure.zcml index 0af1402ec3..f5901d8670 100644 --- a/src/plone/restapi/services/configure.zcml +++ b/src/plone/restapi/services/configure.zcml @@ -19,6 +19,7 @@ + diff --git a/src/plone/restapi/services/facet_count/__init__.py b/src/plone/restapi/services/facet_count/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/plone/restapi/services/facet_count/configure.zcml b/src/plone/restapi/services/facet_count/configure.zcml new file mode 100644 index 0000000000..98ea43b660 --- /dev/null +++ b/src/plone/restapi/services/facet_count/configure.zcml @@ -0,0 +1,16 @@ + + + + + diff --git a/src/plone/restapi/services/facet_count/get.py b/src/plone/restapi/services/facet_count/get.py new file mode 100644 index 0000000000..09a60a65ac --- /dev/null +++ b/src/plone/restapi/services/facet_count/get.py @@ -0,0 +1,54 @@ +from plone.restapi.deserializer import json_body +from plone.restapi.services import Service +from plone.restapi.services.querystringsearch.get import QuerystringSearch +from Products.CMFCore.utils import getToolByName +import json + + +class FacetCountGet(Service): + """Returns facet count.""" + + def reply(self): + facet_count = {} + ctool = getToolByName(self.context, "portal_catalog") + body = json_body(self.request) + + facet = body.get("facet", None) + query = body.get("query", None) + + try: + index = ctool._catalog.getIndex(facet) + except: + index = None + + if query: + body["query"] = [qs for qs in query if qs["i"] != facet] + self.request.set("BODY", json.dumps(body)) + + brains = QuerystringSearch(self.context, self.request).getResults() + + brains_rids = set(brain.getRID() for brain in brains) + index_rids = [rid for rid in index.documentToKeyMap()] if index else [] + + rids = brains_rids.intersection(index_rids) + + for rid in rids: + for key in index.keyForDocument(rid): + if key not in facet_count: + facet_count[key] = 0 + facet_count[key] += 1 + + return { + "@id": "%s/@facet-count" % self.context.absolute_url(), + "facets": { + facet: { + "count": len(rids), + "data": [ + {"value": key, "count": value} + for key, value in facet_count.items() + ], + } + } + if facet + else {}, + } From 8812b837a63049cc603db46ded9dbaff5aed381e Mon Sep 17 00:00:00 2001 From: Miu Razvan Date: Thu, 18 May 2023 20:18:08 +0200 Subject: [PATCH 02/14] optimization for facet count --- .../facet_count/{get.py => facet_count.py} | 21 +- .../services/querystringsearch/facet_count.py | 61 +++++ .../restapi/services/querystringsearch/get.py | 43 +++- src/plone/restapi/utils.py | 232 ++++++++++++++++++ 4 files changed, 341 insertions(+), 16 deletions(-) rename src/plone/restapi/services/facet_count/{get.py => facet_count.py} (72%) create mode 100644 src/plone/restapi/services/querystringsearch/facet_count.py create mode 100644 src/plone/restapi/utils.py diff --git a/src/plone/restapi/services/facet_count/get.py b/src/plone/restapi/services/facet_count/facet_count.py similarity index 72% rename from src/plone/restapi/services/facet_count/get.py rename to src/plone/restapi/services/facet_count/facet_count.py index 09a60a65ac..54765e04e4 100644 --- a/src/plone/restapi/services/facet_count/get.py +++ b/src/plone/restapi/services/facet_count/facet_count.py @@ -1,18 +1,28 @@ from plone.restapi.deserializer import json_body from plone.restapi.services import Service -from plone.restapi.services.querystringsearch.get import QuerystringSearch from Products.CMFCore.utils import getToolByName import json +from plone.restapi.utils import get_query, searchResults + class FacetCountGet(Service): """Returns facet count.""" + def __init__(self, context, request): + self.context = context + self.request = request + def reply(self): facet_count = {} ctool = getToolByName(self.context, "portal_catalog") body = json_body(self.request) + body["b_start"] = int(body.get("b_start", 0)) + body["b_size"] = int(body.get("b_size", 25)) + body["limit"] = int(body.get("limit", 1000)) + body["rids"] = True + facet = body.get("facet", None) query = body.get("query", None) @@ -25,12 +35,8 @@ def reply(self): body["query"] = [qs for qs in query if qs["i"] != facet] self.request.set("BODY", json.dumps(body)) - brains = QuerystringSearch(self.context, self.request).getResults() - - brains_rids = set(brain.getRID() for brain in brains) - index_rids = [rid for rid in index.documentToKeyMap()] if index else [] - - rids = brains_rids.intersection(index_rids) + brains_rids = set(searchResults(get_query(self.context, **body))) + rids = brains_rids.intersection(index.documentToKeyMap()) for rid in rids: for key in index.keyForDocument(rid): @@ -52,3 +58,4 @@ def reply(self): if facet else {}, } + diff --git a/src/plone/restapi/services/querystringsearch/facet_count.py b/src/plone/restapi/services/querystringsearch/facet_count.py new file mode 100644 index 0000000000..4f1b60a5a2 --- /dev/null +++ b/src/plone/restapi/services/querystringsearch/facet_count.py @@ -0,0 +1,61 @@ +from plone.restapi.deserializer import json_body +from plone.restapi.services import Service +from Products.CMFCore.utils import getToolByName +import json + +from plone.restapi.utils import get_query, searchResults + + +class FacetCount(Service): + """Returns facet count.""" + + def __init__(self, context, request): + self.context = context + self.request = request + + def reply(self): + facet_count = {} + ctool = getToolByName(self.context, "portal_catalog") + body = json_body(self.request) + + body["b_start"] = int(body.get("b_start", 0)) + body["b_size"] = int(body.get("b_size", 25)) + body["limit"] = int(body.get("limit", 1000)) + body["rids"] = True + + facet = body.get("facet", None) + query = body.get("query", None) + + try: + index = ctool._catalog.getIndex(facet) + except: + index = None + + if query: + body["query"] = [qs for qs in query if qs["i"] != facet] + self.request.set("BODY", json.dumps(body)) + + brains_rids = set(searchResults(get_query(self.context, **body))) + rids = brains_rids.intersection(index.documentToKeyMap()) + + for rid in rids: + for key in index.keyForDocument(rid): + if key not in facet_count: + facet_count[key] = 0 + facet_count[key] += 1 + + return { + "@id": "%s/@facet-count" % self.context.absolute_url(), + "facets": { + facet: { + "count": len(rids), + "data": [ + {"value": key, "count": value} + for key, value in facet_count.items() + ], + } + } + if facet + else {}, + } + diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index e59c9b5c84..3bf40008ae 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -6,6 +6,9 @@ from plone.restapi.services import Service from urllib import parse from zope.component import getMultiAdapter +from zope.interface import implementer +from zope.publisher.interfaces import IPublishTraverse +from facet_count import FacetCount zcatalog_version = get_distribution("Products.ZCatalog").version @@ -18,11 +21,27 @@ class QuerystringSearch: """Returns the querystring search results given a p.a.querystring data.""" - def __init__(self, context, request): + def __init__(self, context, request, params): self.context = context self.request = request + self.params = params + def __call__(self): + data = json_body(self.request) + fullobjects = data.get("fullobjects", False) + + if len(self.params) > 0: + return FacetCount(self.context, self.request).reply() + + results = self.getResults() + + results = getMultiAdapter((results, self.request), ISerializeToJson)( + fullobjects=fullobjects + ) + return results + + def getResults(self): data = json_body(self.request) query = data.get("query", None) b_start = int(data.get("b_start", 0)) @@ -30,7 +49,7 @@ def __call__(self): sort_on = data.get("sort_on", None) sort_order = data.get("sort_order", None) limit = int(data.get("limit", 1000)) - fullobjects = data.get("fullobjects", False) + rids = data.get("rids", False) if query is None: raise Exception("No query supplied") @@ -45,6 +64,7 @@ def __call__(self): querybuilder_parameters = dict( query=query, brains=True, + rids=rids, b_start=b_start, b_size=b_size, sort_on=sort_on, @@ -59,19 +79,24 @@ def __call__(self): dict(custom_query={"UID": {"not": self.context.UID()}}) ) - results = querybuilder(**querybuilder_parameters) - - results = getMultiAdapter((results, self.request), ISerializeToJson)( - fullobjects=fullobjects - ) - return results + return querybuilder(**querybuilder_parameters) +@implementer(IPublishTraverse) class QuerystringSearchPost(Service): """Returns the querystring search results given a p.a.querystring data.""" + def __init__(self, context, request): + super().__init__(context, request) + self.params = [] + + def publishTraverse(self, request, name): + # Treat any path segments after /@types as parameters + self.params.append(name) + return self + def reply(self): - querystring_search = QuerystringSearch(self.context, self.request) + querystring_search = QuerystringSearch(self.context, self.request, self.params) return querystring_search() diff --git a/src/plone/restapi/utils.py b/src/plone/restapi/utils.py new file mode 100644 index 0000000000..ddaa7462b7 --- /dev/null +++ b/src/plone/restapi/utils.py @@ -0,0 +1,232 @@ +from AccessControl import ClassSecurityInfo +from operator import itemgetter +from plone.app.querystring import queryparser +from plone.app.querystring.interfaces import IParsedQueryIndexModifier +from AccessControl.Permissions import search_zcatalog as SearchZCatalog +from zope.component import getUtilitiesFor, getUtility +from AccessControl.SecurityManagement import getSecurityManager +from plone.app.querystring.interfaces import IQueryModifier +from Products.CMFCore.indexing import processQueue +from Products.CMFCore.interfaces import ICatalogTool +from DateTime import DateTime + + +import logging +import re + +logger = logging.getLogger("plone.app.querystring") + +# We should accept both a simple space, unicode u'\u0020 but also a +# multi-space, so called 'waji-kankaku', unicode u'\u3000' +_MULTISPACE = "\u3000" +_BAD_CHARS = ("?", "-", "+", "*", _MULTISPACE) + +security = ClassSecurityInfo() + +def _quote_chars(s): + # We need to quote parentheses when searching text indices + if "(" in s: + s = s.replace("(", '"("') + if ")" in s: + s = s.replace(")", '")"') + if _MULTISPACE in s: + s = s.replace(_MULTISPACE, " ") + return s + +def _quote(term): + # The terms and, or and not must be wrapped in quotes to avoid + # being parsed as logical query atoms. + if term.lower() in ("and", "or", "not"): + term = '"%s"' % term + return term + + +def munge_search_term(query): + for char in _BAD_CHARS: + query = query.replace(char, " ") + + # extract quoted phrases first + quoted_phrases = re.findall(r'"([^"]*)"', query) + r = [] + for qp in quoted_phrases: + # remove from original query + query = query.replace(f'"{qp}"', "") + # replace with cleaned leading/trailing whitespaces + # and skip empty phrases + clean_qp = qp.strip() + if not clean_qp: + continue + r.append(f'"{clean_qp}"') + + r += map(_quote, query.strip().split()) + r = " AND ".join(r) + r = _quote_chars(r) + ("*" if r and not r.endswith('"') else "") + return r + + +def filter_query(query): + text = query.get("SearchableText", None) + if isinstance(text, dict): + text = text.get("query", "") + if text: + query["SearchableText"] = munge_search_term(text) + return query + +def get_query( + context, + query=None, + batch=False, + b_start=0, + b_size=30, + sort_on=None, + sort_order=None, + limit=0, + custom_query=None, + **kw +): + """Parse the (form)query and return using multi-adapter""" + query_modifiers = getUtilitiesFor(IQueryModifier) + for name, modifier in sorted(query_modifiers, key=itemgetter(0)): + query = modifier(query) + + parsedquery = queryparser.parseFormquery( + context, query, sort_on, sort_order + ) + + index_modifiers = getUtilitiesFor(IParsedQueryIndexModifier) + for name, modifier in index_modifiers: + if name in parsedquery: + new_name, query = modifier(parsedquery[name]) + parsedquery[name] = query + # if a new index name has been returned, we need to replace + # the native ones + if name != new_name: + del parsedquery[name] + parsedquery[new_name] = query + + # Check for valid indexes + ctool = getUtility(ICatalogTool) + valid_indexes = [index for index in parsedquery if index in ctool.indexes()] + + # We'll ignore any invalid index, but will return an empty set if none + # of the indexes are valid. + if not valid_indexes: + logger.warning("Using empty query because there are no valid indexes used.") + parsedquery = {} + + if batch: + parsedquery["b_start"] = b_start + parsedquery["b_size"] = b_size + elif limit: + parsedquery["sort_limit"] = limit + + if "path" not in parsedquery: + parsedquery["path"] = {"query": ""} + + if isinstance(custom_query, dict) and custom_query: + # Update the parsed query with an extra query dictionary. This may + # override the parsed query. The custom_query is a dictionary of + # index names and their associated query values. + for key in custom_query: + if isinstance(parsedquery.get(key), dict) and isinstance( + custom_query.get(key), dict + ): + parsedquery[key].update(custom_query[key]) + continue + parsedquery[key] = custom_query[key] + + # filter bad term and operator in query + parsedquery = filter_query(parsedquery) + return parsedquery + +@security.protected(SearchZCatalog) +def searchResults(query=None, **kw): + # =================== CatalogTool + # Calls ZCatalog.searchResults with extra arguments that + # limit the results to what the user is allowed to see. + # + # This version uses the 'effectiveRange' DateRangeIndex. + # + # It also accepts a keyword argument show_inactive to disable + # effectiveRange checking entirely even for those without portal + # wide AccessInactivePortalContent permission. + + # Make sure any pending index tasks have been processed + processQueue() + + ctool = getUtility(ICatalogTool) + + kw = kw.copy() + show_inactive = kw.get('show_inactive', False) + if isinstance(query, dict) and not show_inactive: + show_inactive = 'show_inactive' in query + + user = getSecurityManager().getUser() + kw['allowedRolesAndUsers'] = ctool._listAllowedRolesAndUsers(user) + + if not show_inactive and not ctool.allow_inactive(kw): + kw['effectiveRange'] = DateTime() + + # filter out invalid sort_on indexes + sort_on = kw.get('sort_on') or [] + if isinstance(sort_on, str): + sort_on = [sort_on] + valid_indexes = ctool.indexes() + try: + sort_on = [idx for idx in sort_on if idx in valid_indexes] + except TypeError: + # sort_on is not iterable + sort_on = [] + if not sort_on: + kw.pop('sort_on', None) + else: + kw['sort_on'] = sort_on + # ==================== Catalog searchresults + # You should pass in a simple dictionary as the first argument, + # which only contains the relevant query. + query = ctool._catalog.merge_query_args(query, **kw) + sort_indexes = ctool._catalog._getSortIndex(query) + reverse = False + if sort_indexes is not None: + order = ctool._catalog._get_sort_attr("order", query) + reverse = [] + if order is None: + order = [''] + elif isinstance(order, str): + order = [order] + for o in order: + reverse.append(o.lower() in ('reverse', 'descending')) + if len(reverse) == 1: + # be nice and keep the old API intact for single sort_order + reverse = reverse[0] + # ===================== Catalog search + # Indexes fulfill a fairly large contract here. We hand each + # index the query mapping we are given (which may be composed + # of some combination of web request, kw mappings or plain old dicts) + # and the index decides what to do with it. If the index finds work + # for itself in the query, it returns the results and a tuple of + # the attributes that were used. If the index finds nothing for it + # to do then it returns None. + + # Canonicalize the request into a sensible query before passing it on + query = ctool._catalog.make_query(query) + + cr = ctool._catalog.getCatalogPlan(query) + cr.start() + + plan = cr.plan() + if not plan: + plan = ctool._catalog._sorted_search_indexes(query) + + rs = None # result set + for index_id in plan: + # The actual core loop over all indices. + if index_id not in ctool._catalog.indexes: + # We can have bogus keys or the plan can contain index names + # that have been removed in the meantime. + continue + + rs = ctool._catalog._search_index(cr, index_id, query, rs) + if not rs: + break + return rs \ No newline at end of file From c6b9900c78ec9119163a9c1585f6f8cca9ecb970 Mon Sep 17 00:00:00 2001 From: Miu Razvan Date: Thu, 18 May 2023 20:49:46 +0200 Subject: [PATCH 03/14] clean up facet --- src/plone/restapi/services/configure.zcml | 1 - .../restapi/services/facet_count/__init__.py | 0 .../services/facet_count/configure.zcml | 16 ----- .../services/facet_count/facet_count.py | 61 ------------------- .../services/querystringsearch/facet.py | 56 +++++++++++++++++ .../services/querystringsearch/facet_count.py | 61 ------------------- .../restapi/services/querystringsearch/get.py | 60 +++++++++++------- 7 files changed, 93 insertions(+), 162 deletions(-) delete mode 100644 src/plone/restapi/services/facet_count/__init__.py delete mode 100644 src/plone/restapi/services/facet_count/configure.zcml delete mode 100644 src/plone/restapi/services/facet_count/facet_count.py create mode 100644 src/plone/restapi/services/querystringsearch/facet.py delete mode 100644 src/plone/restapi/services/querystringsearch/facet_count.py diff --git a/src/plone/restapi/services/configure.zcml b/src/plone/restapi/services/configure.zcml index f5901d8670..0af1402ec3 100644 --- a/src/plone/restapi/services/configure.zcml +++ b/src/plone/restapi/services/configure.zcml @@ -19,7 +19,6 @@ - diff --git a/src/plone/restapi/services/facet_count/__init__.py b/src/plone/restapi/services/facet_count/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/plone/restapi/services/facet_count/configure.zcml b/src/plone/restapi/services/facet_count/configure.zcml deleted file mode 100644 index 98ea43b660..0000000000 --- a/src/plone/restapi/services/facet_count/configure.zcml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - diff --git a/src/plone/restapi/services/facet_count/facet_count.py b/src/plone/restapi/services/facet_count/facet_count.py deleted file mode 100644 index 54765e04e4..0000000000 --- a/src/plone/restapi/services/facet_count/facet_count.py +++ /dev/null @@ -1,61 +0,0 @@ -from plone.restapi.deserializer import json_body -from plone.restapi.services import Service -from Products.CMFCore.utils import getToolByName -import json - -from plone.restapi.utils import get_query, searchResults - - -class FacetCountGet(Service): - """Returns facet count.""" - - def __init__(self, context, request): - self.context = context - self.request = request - - def reply(self): - facet_count = {} - ctool = getToolByName(self.context, "portal_catalog") - body = json_body(self.request) - - body["b_start"] = int(body.get("b_start", 0)) - body["b_size"] = int(body.get("b_size", 25)) - body["limit"] = int(body.get("limit", 1000)) - body["rids"] = True - - facet = body.get("facet", None) - query = body.get("query", None) - - try: - index = ctool._catalog.getIndex(facet) - except: - index = None - - if query: - body["query"] = [qs for qs in query if qs["i"] != facet] - self.request.set("BODY", json.dumps(body)) - - brains_rids = set(searchResults(get_query(self.context, **body))) - rids = brains_rids.intersection(index.documentToKeyMap()) - - for rid in rids: - for key in index.keyForDocument(rid): - if key not in facet_count: - facet_count[key] = 0 - facet_count[key] += 1 - - return { - "@id": "%s/@facet-count" % self.context.absolute_url(), - "facets": { - facet: { - "count": len(rids), - "data": [ - {"value": key, "count": value} - for key, value in facet_count.items() - ], - } - } - if facet - else {}, - } - diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py new file mode 100644 index 0000000000..e570c01db3 --- /dev/null +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -0,0 +1,56 @@ +from plone.restapi.deserializer import json_body +from zope.component import getUtility +from Products.CMFCore.interfaces import ICatalogTool +import json + +from plone.restapi.utils import get_query, searchResults + + +class Facet: + """Returns facet count.""" + + def __init__(self, context, name, querybuilder_params): + self.context = context + self.name = name + self.querybuilder_params = querybuilder_params + + def reply(self): + facet_count = {} + ctool = getUtility(ICatalogTool) + + query = self.querybuilder_params.get("query", None) + + try: + index = ctool._catalog.getIndex(self.name) + except: + index = None + + if query: + self.querybuilder_params["query"] = [ + qs for qs in query if qs["i"] != self.name + ] + + brains_rids = set( + searchResults(get_query(self.context, **self.querybuilder_params)) + ) + rids = brains_rids.intersection(index.documentToKeyMap()) + + for rid in rids: + for key in index.keyForDocument(rid): + if key not in facet_count: + facet_count[key] = 0 + facet_count[key] += 1 + + return { + "facets": { + self.name: { + "count": len(rids), + "data": [ + {"value": key, "count": value} + for key, value in facet_count.items() + ], + } + } + if self.name + else {}, + } diff --git a/src/plone/restapi/services/querystringsearch/facet_count.py b/src/plone/restapi/services/querystringsearch/facet_count.py deleted file mode 100644 index 4f1b60a5a2..0000000000 --- a/src/plone/restapi/services/querystringsearch/facet_count.py +++ /dev/null @@ -1,61 +0,0 @@ -from plone.restapi.deserializer import json_body -from plone.restapi.services import Service -from Products.CMFCore.utils import getToolByName -import json - -from plone.restapi.utils import get_query, searchResults - - -class FacetCount(Service): - """Returns facet count.""" - - def __init__(self, context, request): - self.context = context - self.request = request - - def reply(self): - facet_count = {} - ctool = getToolByName(self.context, "portal_catalog") - body = json_body(self.request) - - body["b_start"] = int(body.get("b_start", 0)) - body["b_size"] = int(body.get("b_size", 25)) - body["limit"] = int(body.get("limit", 1000)) - body["rids"] = True - - facet = body.get("facet", None) - query = body.get("query", None) - - try: - index = ctool._catalog.getIndex(facet) - except: - index = None - - if query: - body["query"] = [qs for qs in query if qs["i"] != facet] - self.request.set("BODY", json.dumps(body)) - - brains_rids = set(searchResults(get_query(self.context, **body))) - rids = brains_rids.intersection(index.documentToKeyMap()) - - for rid in rids: - for key in index.keyForDocument(rid): - if key not in facet_count: - facet_count[key] = 0 - facet_count[key] += 1 - - return { - "@id": "%s/@facet-count" % self.context.absolute_url(), - "facets": { - facet: { - "count": len(rids), - "data": [ - {"value": key, "count": value} - for key, value in facet_count.items() - ], - } - } - if facet - else {}, - } - diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index 3bf40008ae..789d82f6f2 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -8,7 +8,7 @@ from zope.component import getMultiAdapter from zope.interface import implementer from zope.publisher.interfaces import IPublishTraverse -from facet_count import FacetCount +from plone.restapi.services.querystringsearch.facet import Facet zcatalog_version = get_distribution("Products.ZCatalog").version @@ -26,22 +26,30 @@ def __init__(self, context, request, params): self.request = request self.params = params - def __call__(self): data = json_body(self.request) fullobjects = data.get("fullobjects", False) - if len(self.params) > 0: - return FacetCount(self.context, self.request).reply() - - results = self.getResults() + self.setQuerybuilderParams() - results = getMultiAdapter((results, self.request), ISerializeToJson)( - fullobjects=fullobjects - ) + if len(self.params) > 0: + results = Facet( + self.context, + name=self.params[0], + querybuilder_params=self.querybuilder_params, + ).reply() + results["@id"] = ( + "%s/@querystring-search/%s" + % (self.context.absolute_url(), self.params[0]), + ) + else: + results = self.getResults() + results = getMultiAdapter( + (results, self.request), ISerializeToJson + )(fullobjects=fullobjects) return results - - def getResults(self): + + def setQuerybuilderParams(self): data = json_body(self.request) query = data.get("query", None) b_start = int(data.get("b_start", 0)) @@ -49,22 +57,18 @@ def getResults(self): sort_on = data.get("sort_on", None) sort_order = data.get("sort_order", None) limit = int(data.get("limit", 1000)) - rids = data.get("rids", False) if query is None: raise Exception("No query supplied") if sort_order: - sort_order = "descending" if sort_order == "descending" else "ascending" - - querybuilder = getMultiAdapter( - (self.context, self.request), name="querybuilderresults" - ) + sort_order = ( + "descending" if sort_order == "descending" else "ascending" + ) - querybuilder_parameters = dict( + self.querybuilder_params = dict( query=query, brains=True, - rids=rids, b_start=b_start, b_size=b_size, sort_on=sort_on, @@ -72,14 +76,22 @@ def getResults(self): limit=limit, ) + def getResults(self): + querybuilder = getMultiAdapter( + (self.context, self.request), name="querybuilderresults" + ) + # Exclude "self" content item from the results when ZCatalog supports NOT UUID # queries and it is called on a content object. - if not IPloneSiteRoot.providedBy(self.context) and SUPPORT_NOT_UUID_QUERIES: - querybuilder_parameters.update( + if ( + not IPloneSiteRoot.providedBy(self.context) + and SUPPORT_NOT_UUID_QUERIES + ): + self.querybuilder_params.update( dict(custom_query={"UID": {"not": self.context.UID()}}) ) - return querybuilder(**querybuilder_parameters) + return querybuilder(**self.querybuilder_params) @implementer(IPublishTraverse) @@ -96,7 +108,9 @@ def publishTraverse(self, request, name): return self def reply(self): - querystring_search = QuerystringSearch(self.context, self.request, self.params) + querystring_search = QuerystringSearch( + self.context, self.request, self.params + ) return querystring_search() From f0977be5d207837bec3be43a76a2f30ca6153520 Mon Sep 17 00:00:00 2001 From: Miu Razvan Date: Thu, 18 May 2023 20:54:56 +0200 Subject: [PATCH 04/14] run black formatter --- .../services/querystringsearch/facet.py | 2 -- .../restapi/services/querystringsearch/get.py | 19 ++++-------- src/plone/restapi/utils.py | 30 ++++++++++--------- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index e570c01db3..3527785097 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -1,7 +1,5 @@ -from plone.restapi.deserializer import json_body from zope.component import getUtility from Products.CMFCore.interfaces import ICatalogTool -import json from plone.restapi.utils import get_query, searchResults diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index 789d82f6f2..116183be1e 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -44,9 +44,9 @@ def __call__(self): ) else: results = self.getResults() - results = getMultiAdapter( - (results, self.request), ISerializeToJson - )(fullobjects=fullobjects) + results = getMultiAdapter((results, self.request), ISerializeToJson)( + fullobjects=fullobjects + ) return results def setQuerybuilderParams(self): @@ -62,9 +62,7 @@ def setQuerybuilderParams(self): raise Exception("No query supplied") if sort_order: - sort_order = ( - "descending" if sort_order == "descending" else "ascending" - ) + sort_order = "descending" if sort_order == "descending" else "ascending" self.querybuilder_params = dict( query=query, @@ -83,10 +81,7 @@ def getResults(self): # Exclude "self" content item from the results when ZCatalog supports NOT UUID # queries and it is called on a content object. - if ( - not IPloneSiteRoot.providedBy(self.context) - and SUPPORT_NOT_UUID_QUERIES - ): + if not IPloneSiteRoot.providedBy(self.context) and SUPPORT_NOT_UUID_QUERIES: self.querybuilder_params.update( dict(custom_query={"UID": {"not": self.context.UID()}}) ) @@ -108,9 +103,7 @@ def publishTraverse(self, request, name): return self def reply(self): - querystring_search = QuerystringSearch( - self.context, self.request, self.params - ) + querystring_search = QuerystringSearch(self.context, self.request, self.params) return querystring_search() diff --git a/src/plone/restapi/utils.py b/src/plone/restapi/utils.py index ddaa7462b7..d3c0f0c9ee 100644 --- a/src/plone/restapi/utils.py +++ b/src/plone/restapi/utils.py @@ -23,6 +23,7 @@ security = ClassSecurityInfo() + def _quote_chars(s): # We need to quote parentheses when searching text indices if "(" in s: @@ -33,6 +34,7 @@ def _quote_chars(s): s = s.replace(_MULTISPACE, " ") return s + def _quote(term): # The terms and, or and not must be wrapped in quotes to avoid # being parsed as logical query atoms. @@ -72,6 +74,7 @@ def filter_query(query): query["SearchableText"] = munge_search_term(text) return query + def get_query( context, query=None, @@ -82,16 +85,14 @@ def get_query( sort_order=None, limit=0, custom_query=None, - **kw + **kw, ): """Parse the (form)query and return using multi-adapter""" query_modifiers = getUtilitiesFor(IQueryModifier) for name, modifier in sorted(query_modifiers, key=itemgetter(0)): query = modifier(query) - parsedquery = queryparser.parseFormquery( - context, query, sort_on, sort_order - ) + parsedquery = queryparser.parseFormquery(context, query, sort_on, sort_order) index_modifiers = getUtilitiesFor(IParsedQueryIndexModifier) for name, modifier in index_modifiers: @@ -139,6 +140,7 @@ def get_query( parsedquery = filter_query(parsedquery) return parsedquery + @security.protected(SearchZCatalog) def searchResults(query=None, **kw): # =================== CatalogTool @@ -157,18 +159,18 @@ def searchResults(query=None, **kw): ctool = getUtility(ICatalogTool) kw = kw.copy() - show_inactive = kw.get('show_inactive', False) + show_inactive = kw.get("show_inactive", False) if isinstance(query, dict) and not show_inactive: - show_inactive = 'show_inactive' in query + show_inactive = "show_inactive" in query user = getSecurityManager().getUser() - kw['allowedRolesAndUsers'] = ctool._listAllowedRolesAndUsers(user) + kw["allowedRolesAndUsers"] = ctool._listAllowedRolesAndUsers(user) if not show_inactive and not ctool.allow_inactive(kw): - kw['effectiveRange'] = DateTime() + kw["effectiveRange"] = DateTime() # filter out invalid sort_on indexes - sort_on = kw.get('sort_on') or [] + sort_on = kw.get("sort_on") or [] if isinstance(sort_on, str): sort_on = [sort_on] valid_indexes = ctool.indexes() @@ -178,9 +180,9 @@ def searchResults(query=None, **kw): # sort_on is not iterable sort_on = [] if not sort_on: - kw.pop('sort_on', None) + kw.pop("sort_on", None) else: - kw['sort_on'] = sort_on + kw["sort_on"] = sort_on # ==================== Catalog searchresults # You should pass in a simple dictionary as the first argument, # which only contains the relevant query. @@ -191,11 +193,11 @@ def searchResults(query=None, **kw): order = ctool._catalog._get_sort_attr("order", query) reverse = [] if order is None: - order = [''] + order = [""] elif isinstance(order, str): order = [order] for o in order: - reverse.append(o.lower() in ('reverse', 'descending')) + reverse.append(o.lower() in ("reverse", "descending")) if len(reverse) == 1: # be nice and keep the old API intact for single sort_order reverse = reverse[0] @@ -229,4 +231,4 @@ def searchResults(query=None, **kw): rs = ctool._catalog._search_index(cr, index_id, query, rs) if not rs: break - return rs \ No newline at end of file + return rs From ad03ea3780461723a6317ad15bcfb4909b3f27ac Mon Sep 17 00:00:00 2001 From: Razvan Date: Thu, 29 Feb 2024 17:24:01 +0200 Subject: [PATCH 05/14] get facet count in one request --- .../services/querystringsearch/facet.py | 84 ++++--- .../restapi/services/querystringsearch/get.py | 44 +++- src/plone/restapi/utils.py | 234 ------------------ 3 files changed, 75 insertions(+), 287 deletions(-) delete mode 100644 src/plone/restapi/utils.py diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index 3527785097..0b2abf2816 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -1,54 +1,56 @@ -from zope.component import getUtility +from BTrees.IIBTree import intersection from Products.CMFCore.interfaces import ICatalogTool - -from plone.restapi.utils import get_query, searchResults +from zope.component import getUtility +from zope.component import getMultiAdapter class Facet: """Returns facet count.""" - def __init__(self, context, name, querybuilder_params): + def __init__(self, context, request, name, querybuilder_parameters): self.context = context + self.request = request self.name = name - self.querybuilder_params = querybuilder_params - - def reply(self): - facet_count = {} - ctool = getUtility(ICatalogTool) - - query = self.querybuilder_params.get("query", None) + self.querybuilder_parameters = querybuilder_parameters + self.querybuilder_parameters["query"] = [ + qs for qs in querybuilder_parameters.get("query", []) if qs["i"] != self.name + ] + self.querybuilder_parameters["rids"] = True + def getFacet(self): try: + ctool = getUtility(ICatalogTool) + count = {} index = ctool._catalog.getIndex(self.name) - except: - index = None - - if query: - self.querybuilder_params["query"] = [ - qs for qs in query if qs["i"] != self.name - ] - - brains_rids = set( - searchResults(get_query(self.context, **self.querybuilder_params)) - ) - rids = brains_rids.intersection(index.documentToKeyMap()) + # Get the brains for the query without the facet + querybuilder = getMultiAdapter( + (self.context, self.request), name="querybuilderresults" + ) + brains_rids = querybuilder(**self.querybuilder_parameters) + # Get the rids for the brains that have the facet index set to the value we are interested in + rids = intersection(brains_rids, index.documentToKeyMap()) + + for rid in rids: + keys = index.keyForDocument(rid) + if isinstance(keys, str): + keys = [keys] + if not isinstance(keys, list): + continue + for key in keys: + if key not in count: + count[key] = 0 + count[key] += 1 + + results = { + "name": self.name, + "count": len(rids), + "data": {}, + } + + for key, value in count.items(): + results["data"][key] = value - for rid in rids: - for key in index.keyForDocument(rid): - if key not in facet_count: - facet_count[key] = 0 - facet_count[key] += 1 + return results - return { - "facets": { - self.name: { - "count": len(rids), - "data": [ - {"value": key, "count": value} - for key, value in facet_count.items() - ], - } - } - if self.name - else {}, - } + except: + return None \ No newline at end of file diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index e46f586719..1a6dd5d81f 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -30,13 +30,16 @@ def __init__(self, context, request, params): def __call__(self): self.setQuerybuilderParams() - + if len(self.params) > 0: results = Facet( self.context, + self.request, name=self.params[0], querybuilder_parameters=self.querybuilder_parameters, - ).reply() + ).getFacet() + if results is None: + raise BadRequest("Invalid facet") results["@id"] = ( "%s/@querystring-search/%s" % (self.context.absolute_url(), self.params[0]), @@ -46,6 +49,16 @@ def __call__(self): results = getMultiAdapter((results, self.request), ISerializeToJson)( fullobjects=self.fullobjects ) + results["facets_count"] = {} + for facet in self.facets: + facet_results = Facet( + self.context, + self.request, + name=facet, + querybuilder_parameters=self.querybuilder_parameters, + ).getFacet() + if facet_results: + results["facets_count"][facet] = facet_results return results def setQuerybuilderParams(self): @@ -71,6 +84,7 @@ def setQuerybuilderParams(self): raise BadRequest("Invalid limit") self.fullobjects = bool(data.get("fullobjects", False)) + self.facets = data.get("facets", []) if not query: raise BadRequest("No query supplied") @@ -87,19 +101,16 @@ def setQuerybuilderParams(self): sort_order=sort_order, limit=limit, ) - - def getResults(self): - querybuilder = getMultiAdapter( - (self.context, self.request), name="querybuilderresults" - ) - - # Exclude "self" content item from the results when ZCatalog supports NOT UUID - # queries and it is called on a content object. + if not IPloneSiteRoot.providedBy(self.context) and SUPPORT_NOT_UUID_QUERIES: self.querybuilder_parameters.update( dict(custom_query={"UID": {"not": self.context.UID()}}) ) + def getResults(self): + querybuilder = getMultiAdapter( + (self.context, self.request), name="querybuilderresults" + ) return querybuilder(**self.querybuilder_parameters) @@ -120,9 +131,18 @@ def reply(self): querystring_search = QuerystringSearch(self.context, self.request, self.params) return querystring_search() - +@implementer(IPublishTraverse) class QuerystringSearchGet(Service): """Returns the querystring search results given a p.a.querystring data.""" + + def __init__(self, context, request): + super().__init__(context, request) + self.params = [] + + def publishTraverse(self, request, name): + # Treat any path segments after /@types as parameters + self.params.append(name) + return self def reply(self): # We need to copy the JSON query parameters from the querystring @@ -133,5 +153,5 @@ def reply(self): # unset the get parameters self.request.form = {} - querystring_search = QuerystringSearch(self.context, self.request) + querystring_search = QuerystringSearch(self.context, self.request, self.params) return querystring_search() \ No newline at end of file diff --git a/src/plone/restapi/utils.py b/src/plone/restapi/utils.py deleted file mode 100644 index d3c0f0c9ee..0000000000 --- a/src/plone/restapi/utils.py +++ /dev/null @@ -1,234 +0,0 @@ -from AccessControl import ClassSecurityInfo -from operator import itemgetter -from plone.app.querystring import queryparser -from plone.app.querystring.interfaces import IParsedQueryIndexModifier -from AccessControl.Permissions import search_zcatalog as SearchZCatalog -from zope.component import getUtilitiesFor, getUtility -from AccessControl.SecurityManagement import getSecurityManager -from plone.app.querystring.interfaces import IQueryModifier -from Products.CMFCore.indexing import processQueue -from Products.CMFCore.interfaces import ICatalogTool -from DateTime import DateTime - - -import logging -import re - -logger = logging.getLogger("plone.app.querystring") - -# We should accept both a simple space, unicode u'\u0020 but also a -# multi-space, so called 'waji-kankaku', unicode u'\u3000' -_MULTISPACE = "\u3000" -_BAD_CHARS = ("?", "-", "+", "*", _MULTISPACE) - -security = ClassSecurityInfo() - - -def _quote_chars(s): - # We need to quote parentheses when searching text indices - if "(" in s: - s = s.replace("(", '"("') - if ")" in s: - s = s.replace(")", '")"') - if _MULTISPACE in s: - s = s.replace(_MULTISPACE, " ") - return s - - -def _quote(term): - # The terms and, or and not must be wrapped in quotes to avoid - # being parsed as logical query atoms. - if term.lower() in ("and", "or", "not"): - term = '"%s"' % term - return term - - -def munge_search_term(query): - for char in _BAD_CHARS: - query = query.replace(char, " ") - - # extract quoted phrases first - quoted_phrases = re.findall(r'"([^"]*)"', query) - r = [] - for qp in quoted_phrases: - # remove from original query - query = query.replace(f'"{qp}"', "") - # replace with cleaned leading/trailing whitespaces - # and skip empty phrases - clean_qp = qp.strip() - if not clean_qp: - continue - r.append(f'"{clean_qp}"') - - r += map(_quote, query.strip().split()) - r = " AND ".join(r) - r = _quote_chars(r) + ("*" if r and not r.endswith('"') else "") - return r - - -def filter_query(query): - text = query.get("SearchableText", None) - if isinstance(text, dict): - text = text.get("query", "") - if text: - query["SearchableText"] = munge_search_term(text) - return query - - -def get_query( - context, - query=None, - batch=False, - b_start=0, - b_size=30, - sort_on=None, - sort_order=None, - limit=0, - custom_query=None, - **kw, -): - """Parse the (form)query and return using multi-adapter""" - query_modifiers = getUtilitiesFor(IQueryModifier) - for name, modifier in sorted(query_modifiers, key=itemgetter(0)): - query = modifier(query) - - parsedquery = queryparser.parseFormquery(context, query, sort_on, sort_order) - - index_modifiers = getUtilitiesFor(IParsedQueryIndexModifier) - for name, modifier in index_modifiers: - if name in parsedquery: - new_name, query = modifier(parsedquery[name]) - parsedquery[name] = query - # if a new index name has been returned, we need to replace - # the native ones - if name != new_name: - del parsedquery[name] - parsedquery[new_name] = query - - # Check for valid indexes - ctool = getUtility(ICatalogTool) - valid_indexes = [index for index in parsedquery if index in ctool.indexes()] - - # We'll ignore any invalid index, but will return an empty set if none - # of the indexes are valid. - if not valid_indexes: - logger.warning("Using empty query because there are no valid indexes used.") - parsedquery = {} - - if batch: - parsedquery["b_start"] = b_start - parsedquery["b_size"] = b_size - elif limit: - parsedquery["sort_limit"] = limit - - if "path" not in parsedquery: - parsedquery["path"] = {"query": ""} - - if isinstance(custom_query, dict) and custom_query: - # Update the parsed query with an extra query dictionary. This may - # override the parsed query. The custom_query is a dictionary of - # index names and their associated query values. - for key in custom_query: - if isinstance(parsedquery.get(key), dict) and isinstance( - custom_query.get(key), dict - ): - parsedquery[key].update(custom_query[key]) - continue - parsedquery[key] = custom_query[key] - - # filter bad term and operator in query - parsedquery = filter_query(parsedquery) - return parsedquery - - -@security.protected(SearchZCatalog) -def searchResults(query=None, **kw): - # =================== CatalogTool - # Calls ZCatalog.searchResults with extra arguments that - # limit the results to what the user is allowed to see. - # - # This version uses the 'effectiveRange' DateRangeIndex. - # - # It also accepts a keyword argument show_inactive to disable - # effectiveRange checking entirely even for those without portal - # wide AccessInactivePortalContent permission. - - # Make sure any pending index tasks have been processed - processQueue() - - ctool = getUtility(ICatalogTool) - - kw = kw.copy() - show_inactive = kw.get("show_inactive", False) - if isinstance(query, dict) and not show_inactive: - show_inactive = "show_inactive" in query - - user = getSecurityManager().getUser() - kw["allowedRolesAndUsers"] = ctool._listAllowedRolesAndUsers(user) - - if not show_inactive and not ctool.allow_inactive(kw): - kw["effectiveRange"] = DateTime() - - # filter out invalid sort_on indexes - sort_on = kw.get("sort_on") or [] - if isinstance(sort_on, str): - sort_on = [sort_on] - valid_indexes = ctool.indexes() - try: - sort_on = [idx for idx in sort_on if idx in valid_indexes] - except TypeError: - # sort_on is not iterable - sort_on = [] - if not sort_on: - kw.pop("sort_on", None) - else: - kw["sort_on"] = sort_on - # ==================== Catalog searchresults - # You should pass in a simple dictionary as the first argument, - # which only contains the relevant query. - query = ctool._catalog.merge_query_args(query, **kw) - sort_indexes = ctool._catalog._getSortIndex(query) - reverse = False - if sort_indexes is not None: - order = ctool._catalog._get_sort_attr("order", query) - reverse = [] - if order is None: - order = [""] - elif isinstance(order, str): - order = [order] - for o in order: - reverse.append(o.lower() in ("reverse", "descending")) - if len(reverse) == 1: - # be nice and keep the old API intact for single sort_order - reverse = reverse[0] - # ===================== Catalog search - # Indexes fulfill a fairly large contract here. We hand each - # index the query mapping we are given (which may be composed - # of some combination of web request, kw mappings or plain old dicts) - # and the index decides what to do with it. If the index finds work - # for itself in the query, it returns the results and a tuple of - # the attributes that were used. If the index finds nothing for it - # to do then it returns None. - - # Canonicalize the request into a sensible query before passing it on - query = ctool._catalog.make_query(query) - - cr = ctool._catalog.getCatalogPlan(query) - cr.start() - - plan = cr.plan() - if not plan: - plan = ctool._catalog._sorted_search_indexes(query) - - rs = None # result set - for index_id in plan: - # The actual core loop over all indices. - if index_id not in ctool._catalog.indexes: - # We can have bogus keys or the plan can contain index names - # that have been removed in the meantime. - continue - - rs = ctool._catalog._search_index(cr, index_id, query, rs) - if not rs: - break - return rs From 4f4c64b1a91f36f433071c7fd7580799add804b2 Mon Sep 17 00:00:00 2001 From: Razvan Date: Thu, 29 Feb 2024 17:41:54 +0200 Subject: [PATCH 06/14] run black and flake8 --- .../services/querystringsearch/facet.py | 73 ++++++++++--------- .../restapi/services/querystringsearch/get.py | 11 +-- 2 files changed, 44 insertions(+), 40 deletions(-) diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index 0b2abf2816..b1998d819e 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -13,44 +13,47 @@ def __init__(self, context, request, name, querybuilder_parameters): self.name = name self.querybuilder_parameters = querybuilder_parameters self.querybuilder_parameters["query"] = [ - qs for qs in querybuilder_parameters.get("query", []) if qs["i"] != self.name + qs + for qs in querybuilder_parameters.get("query", []) + if qs["i"] != self.name ] self.querybuilder_parameters["rids"] = True def getFacet(self): + ctool = getUtility(ICatalogTool) + count = {} + index = None try: - ctool = getUtility(ICatalogTool) - count = {} index = ctool._catalog.getIndex(self.name) - # Get the brains for the query without the facet - querybuilder = getMultiAdapter( - (self.context, self.request), name="querybuilderresults" - ) - brains_rids = querybuilder(**self.querybuilder_parameters) - # Get the rids for the brains that have the facet index set to the value we are interested in - rids = intersection(brains_rids, index.documentToKeyMap()) - - for rid in rids: - keys = index.keyForDocument(rid) - if isinstance(keys, str): - keys = [keys] - if not isinstance(keys, list): - continue - for key in keys: - if key not in count: - count[key] = 0 - count[key] += 1 - - results = { - "name": self.name, - "count": len(rids), - "data": {}, - } - - for key, value in count.items(): - results["data"][key] = value - - return results - - except: - return None \ No newline at end of file + finally: + if index is None: + return None + # Get the brains for the query without the facet + querybuilder = getMultiAdapter( + (self.context, self.request), name="querybuilderresults" + ) + brains_rids = querybuilder(**self.querybuilder_parameters) + # Get the rids for the brains that have the facet index set to the value we are interested in + rids = intersection(brains_rids, index.documentToKeyMap()) + + for rid in rids: + keys = index.keyForDocument(rid) + if isinstance(keys, str): + keys = [keys] + if not isinstance(keys, list): + continue + for key in keys: + if key not in count: + count[key] = 0 + count[key] += 1 + + results = { + "name": self.name, + "count": len(rids), + "data": {}, + } + + for key, value in count.items(): + results["data"][key] = value + + return results diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index 1a6dd5d81f..2cfdb10c42 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -30,7 +30,7 @@ def __init__(self, context, request, params): def __call__(self): self.setQuerybuilderParams() - + if len(self.params) > 0: results = Facet( self.context, @@ -101,7 +101,7 @@ def setQuerybuilderParams(self): sort_order=sort_order, limit=limit, ) - + if not IPloneSiteRoot.providedBy(self.context) and SUPPORT_NOT_UUID_QUERIES: self.querybuilder_parameters.update( dict(custom_query={"UID": {"not": self.context.UID()}}) @@ -131,14 +131,15 @@ def reply(self): querystring_search = QuerystringSearch(self.context, self.request, self.params) return querystring_search() + @implementer(IPublishTraverse) class QuerystringSearchGet(Service): """Returns the querystring search results given a p.a.querystring data.""" - + def __init__(self, context, request): super().__init__(context, request) self.params = [] - + def publishTraverse(self, request, name): # Treat any path segments after /@types as parameters self.params.append(name) @@ -154,4 +155,4 @@ def reply(self): # unset the get parameters self.request.form = {} querystring_search = QuerystringSearch(self.context, self.request, self.params) - return querystring_search() \ No newline at end of file + return querystring_search() From 5ba67228b8a3d5651e88f419c17ab4a13baf0f40 Mon Sep 17 00:00:00 2001 From: Dobricean Ioan Dorian Date: Wed, 6 Mar 2024 13:17:16 +0200 Subject: [PATCH 07/14] return facet_count_criteria that stores how many results are for a facet based just on the query criteria set on the block --- .../services/querystringsearch/facet.py | 37 ++++++++++++++++--- .../restapi/services/querystringsearch/get.py | 2 +- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index b1998d819e..f81c7880db 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -3,7 +3,6 @@ from zope.component import getUtility from zope.component import getMultiAdapter - class Facet: """Returns facet count.""" @@ -11,17 +10,25 @@ def __init__(self, context, request, name, querybuilder_parameters): self.context = context self.request = request self.name = name - self.querybuilder_parameters = querybuilder_parameters + self.querybuilder_parameters = querybuilder_parameters.copy() + self.querybuilder_criteria_parameters = querybuilder_parameters.copy() self.querybuilder_parameters["query"] = [ qs for qs in querybuilder_parameters.get("query", []) - if qs["i"] != self.name + if qs["i"] != self.name or ('criteria' in qs and qs["criteria"] is True) ] self.querybuilder_parameters["rids"] = True + self.querybuilder_criteria_parameters["rids"] = True + self.querybuilder_criteria_parameters["query"] =[ + qs + for qs in querybuilder_parameters.get("query", []) + if 'criteria' in qs and qs["criteria"] is True + ] def getFacet(self): ctool = getUtility(ICatalogTool) count = {} + count_criteria = {} index = None try: index = ctool._catalog.getIndex(self.name) @@ -32,10 +39,15 @@ def getFacet(self): querybuilder = getMultiAdapter( (self.context, self.request), name="querybuilderresults" ) + querybuilder_criteria = getMultiAdapter( + (self.context, self.request), name="querybuilderresults" + ) + brains_rids = querybuilder(**self.querybuilder_parameters) + brains_rids_criteria = querybuilder_criteria(**self.querybuilder_criteria_parameters) # Get the rids for the brains that have the facet index set to the value we are interested in rids = intersection(brains_rids, index.documentToKeyMap()) - + rids_criteria = intersection(brains_rids_criteria, index.documentToKeyMap()) for rid in rids: keys = index.keyForDocument(rid) if isinstance(keys, str): @@ -46,14 +58,29 @@ def getFacet(self): if key not in count: count[key] = 0 count[key] += 1 + for rid in rids_criteria: + keys = index.keyForDocument(rid) + if isinstance(keys, str): + keys = [keys] + if not isinstance(keys, list): + continue + for key in keys: + if key not in count_criteria: + count_criteria[key] = 0 + count_criteria[key] += 1 results = { "name": self.name, "count": len(rids), "data": {}, + "count_criteria":len(rids_criteria) } for key, value in count.items(): - results["data"][key] = value + results["data"][key] = {'count':value, 'count_criteria': count_criteria[key] if (key in count_criteria) else 0} + + for key,value in count_criteria.items(): + if key not in results["data"]: + results["data"][key] = {'count':0, 'count_criteria': value} return results diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index 2cfdb10c42..caf8447c69 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -155,4 +155,4 @@ def reply(self): # unset the get parameters self.request.form = {} querystring_search = QuerystringSearch(self.context, self.request, self.params) - return querystring_search() + return querystring_search() \ No newline at end of file From 1e22583f27b6ce08b4327f7203e5c617198545c5 Mon Sep 17 00:00:00 2001 From: Dobricean Ioan Dorian Date: Wed, 6 Mar 2024 13:28:09 +0200 Subject: [PATCH 08/14] return just results that have count_criteria > 0 --- src/plone/restapi/services/querystringsearch/facet.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index f81c7880db..83ed12f37f 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -77,10 +77,11 @@ def getFacet(self): } for key, value in count.items(): - results["data"][key] = {'count':value, 'count_criteria': count_criteria[key] if (key in count_criteria) else 0} + if key in count_criteria and count_criteria[key] > 0: + results["data"][key] = value for key,value in count_criteria.items(): if key not in results["data"]: - results["data"][key] = {'count':0, 'count_criteria': value} + results["data"][key] = 0 return results From 880c28c2fb04a219ab1943f49bf50f3ece4a40a1 Mon Sep 17 00:00:00 2001 From: Dobricean Ioan Dorian Date: Wed, 6 Mar 2024 16:40:13 +0200 Subject: [PATCH 09/14] filter values after criteria --- .../restapi/services/querystringsearch/facet.py | 16 ++++++---------- .../restapi/services/querystringsearch/get.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index 83ed12f37f..ab7430005d 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -6,7 +6,7 @@ class Facet: """Returns facet count.""" - def __init__(self, context, request, name, querybuilder_parameters): + def __init__(self, context, request, name, querybuilder_parameters,brains_rids_criteria): self.context = context self.request = request self.name = name @@ -24,6 +24,7 @@ def __init__(self, context, request, name, querybuilder_parameters): for qs in querybuilder_parameters.get("query", []) if 'criteria' in qs and qs["criteria"] is True ] + self.brain_rids_criteria = brains_rids_criteria def getFacet(self): ctool = getUtility(ICatalogTool) @@ -39,12 +40,9 @@ def getFacet(self): querybuilder = getMultiAdapter( (self.context, self.request), name="querybuilderresults" ) - querybuilder_criteria = getMultiAdapter( - (self.context, self.request), name="querybuilderresults" - ) - + brains_rids = querybuilder(**self.querybuilder_parameters) - brains_rids_criteria = querybuilder_criteria(**self.querybuilder_criteria_parameters) + brains_rids_criteria = self.brain_rids_criteria # Get the rids for the brains that have the facet index set to the value we are interested in rids = intersection(brains_rids, index.documentToKeyMap()) rids_criteria = intersection(brains_rids_criteria, index.documentToKeyMap()) @@ -73,13 +71,11 @@ def getFacet(self): "name": self.name, "count": len(rids), "data": {}, - "count_criteria":len(rids_criteria) } for key, value in count.items(): - if key in count_criteria and count_criteria[key] > 0: - results["data"][key] = value - + if key in count_criteria and count_criteria[key]>0: + results["data"][key] = value for key,value in count_criteria.items(): if key not in results["data"]: results["data"][key] = 0 diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index caf8447c69..a8227ae959 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -30,13 +30,25 @@ def __init__(self, context, request, params): def __call__(self): self.setQuerybuilderParams() + querybuilder_criteria_parameters = self.querybuilder_parameters.copy() + querybuilder_criteria_parameters["query"] =[ + qs + for qs in self.querybuilder_parameters.get("query", []) + if 'criteria' in qs and qs["criteria"] is True + ] + querybuilder_criteria_parameters["rids"] = True + querybuilder = getMultiAdapter( + (self.context, self.request), name="querybuilderresults" + ) + brains_rids_criteria = querybuilder(**querybuilder_criteria_parameters) if len(self.params) > 0: results = Facet( self.context, self.request, name=self.params[0], querybuilder_parameters=self.querybuilder_parameters, + brains_rids_criteria=brains_rids_criteria ).getFacet() if results is None: raise BadRequest("Invalid facet") @@ -56,6 +68,7 @@ def __call__(self): self.request, name=facet, querybuilder_parameters=self.querybuilder_parameters, + brains_rids_criteria=brains_rids_criteria ).getFacet() if facet_results: results["facets_count"][facet] = facet_results From 16255d9d0552d1f3d56bfba97a686da6460093e6 Mon Sep 17 00:00:00 2001 From: Dobricean Ioan Dorian Date: Thu, 7 Mar 2024 13:04:30 +0200 Subject: [PATCH 10/14] format --- .../services/querystringsearch/facet.py | 23 +++++++++++-------- .../restapi/services/querystringsearch/get.py | 10 ++++---- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index ab7430005d..5277540529 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -3,10 +3,13 @@ from zope.component import getUtility from zope.component import getMultiAdapter + class Facet: """Returns facet count.""" - def __init__(self, context, request, name, querybuilder_parameters,brains_rids_criteria): + def __init__( + self, context, request, name, querybuilder_parameters, brains_rids_criteria + ): self.context = context self.request = request self.name = name @@ -15,14 +18,14 @@ def __init__(self, context, request, name, querybuilder_parameters,brains_rids_c self.querybuilder_parameters["query"] = [ qs for qs in querybuilder_parameters.get("query", []) - if qs["i"] != self.name or ('criteria' in qs and qs["criteria"] is True) + if qs["i"] != self.name or ("criteria" in qs and qs["criteria"] is True) ] self.querybuilder_parameters["rids"] = True self.querybuilder_criteria_parameters["rids"] = True - self.querybuilder_criteria_parameters["query"] =[ + self.querybuilder_criteria_parameters["query"] = [ qs for qs in querybuilder_parameters.get("query", []) - if 'criteria' in qs and qs["criteria"] is True + if "criteria" in qs and qs["criteria"] is True ] self.brain_rids_criteria = brains_rids_criteria @@ -40,7 +43,7 @@ def getFacet(self): querybuilder = getMultiAdapter( (self.context, self.request), name="querybuilderresults" ) - + brains_rids = querybuilder(**self.querybuilder_parameters) brains_rids_criteria = self.brain_rids_criteria # Get the rids for the brains that have the facet index set to the value we are interested in @@ -74,10 +77,10 @@ def getFacet(self): } for key, value in count.items(): - if key in count_criteria and count_criteria[key]>0: - results["data"][key] = value - for key,value in count_criteria.items(): - if key not in results["data"]: - results["data"][key] = 0 + if key in count_criteria and count_criteria[key] > 0: + results["data"][key] = value + for key, value in count_criteria.items(): + if key not in results["data"]: + results["data"][key] = 0 return results diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index a8227ae959..3a8946c220 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -31,10 +31,10 @@ def __init__(self, context, request, params): def __call__(self): self.setQuerybuilderParams() querybuilder_criteria_parameters = self.querybuilder_parameters.copy() - querybuilder_criteria_parameters["query"] =[ + querybuilder_criteria_parameters["query"] = [ qs for qs in self.querybuilder_parameters.get("query", []) - if 'criteria' in qs and qs["criteria"] is True + if "criteria" in qs and qs["criteria"] is True ] querybuilder_criteria_parameters["rids"] = True querybuilder = getMultiAdapter( @@ -48,7 +48,7 @@ def __call__(self): self.request, name=self.params[0], querybuilder_parameters=self.querybuilder_parameters, - brains_rids_criteria=brains_rids_criteria + brains_rids_criteria=brains_rids_criteria, ).getFacet() if results is None: raise BadRequest("Invalid facet") @@ -68,7 +68,7 @@ def __call__(self): self.request, name=facet, querybuilder_parameters=self.querybuilder_parameters, - brains_rids_criteria=brains_rids_criteria + brains_rids_criteria=brains_rids_criteria, ).getFacet() if facet_results: results["facets_count"][facet] = facet_results @@ -168,4 +168,4 @@ def reply(self): # unset the get parameters self.request.form = {} querystring_search = QuerystringSearch(self.context, self.request, self.params) - return querystring_search() \ No newline at end of file + return querystring_search() From 6c83e27b883771a98539ba6971e50fa19e4dcd7e Mon Sep 17 00:00:00 2001 From: Dobricean Ioan Dorian Date: Thu, 7 Mar 2024 13:08:52 +0200 Subject: [PATCH 11/14] changelog --- news/1637.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/1637.feature diff --git a/news/1637.feature b/news/1637.feature new file mode 100644 index 0000000000..c54d2beb29 --- /dev/null +++ b/news/1637.feature @@ -0,0 +1 @@ +feat(search): show facets count and delete the facets results that don't meet the criterias @razvanMiu @dobri1408 \ No newline at end of file From c695556fb9276179cc4c9a7128e17f26c095294e Mon Sep 17 00:00:00 2001 From: Dobricean Ioan Dorian Date: Fri, 8 Mar 2024 10:40:05 +0200 Subject: [PATCH 12/14] make search work on PloneRoot --- .../services/querystringsearch/facet.py | 53 +++++++++++-------- .../restapi/services/querystringsearch/get.py | 18 ++++--- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index 5277540529..64e705a0dc 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -2,37 +2,50 @@ from Products.CMFCore.interfaces import ICatalogTool from zope.component import getUtility from zope.component import getMultiAdapter +from pkg_resources import get_distribution +from pkg_resources import parse_version +zcatalog_version = get_distribution("Products.ZCatalog").version +if parse_version(zcatalog_version) >= parse_version("5.1"): + SUPPORT_NOT_UUID_QUERIES = True +else: + SUPPORT_NOT_UUID_QUERIES = False class Facet: """Returns facet count.""" def __init__( - self, context, request, name, querybuilder_parameters, brains_rids_criteria + self, context, request, name, querybuilder_parameters, brains_rids_mandatory ): + self.context = context self.request = request self.name = name self.querybuilder_parameters = querybuilder_parameters.copy() - self.querybuilder_criteria_parameters = querybuilder_parameters.copy() + self.querybuilder_mandatory_parameters = querybuilder_parameters.copy() self.querybuilder_parameters["query"] = [ qs for qs in querybuilder_parameters.get("query", []) - if qs["i"] != self.name or ("criteria" in qs and qs["criteria"] is True) + if qs["i"] != self.name or ("mandatory" in qs and qs["mandatory"] is True) ] self.querybuilder_parameters["rids"] = True - self.querybuilder_criteria_parameters["rids"] = True - self.querybuilder_criteria_parameters["query"] = [ + self.querybuilder_mandatory_parameters["rids"] = True + self.querybuilder_mandatory_parameters["query"] = [ qs for qs in querybuilder_parameters.get("query", []) - if "criteria" in qs and qs["criteria"] is True + if "mandatory" in qs and qs["mandatory"] is True ] - self.brain_rids_criteria = brains_rids_criteria + self.brain_rids_mandatory = brains_rids_mandatory + if SUPPORT_NOT_UUID_QUERIES: + self.querybuilder_parameters.update( + dict(custom_query={"UID": {"not": self.context.UID()}}) + ) + def getFacet(self): ctool = getUtility(ICatalogTool) count = {} - count_criteria = {} + count_mandatory = {} index = None try: index = ctool._catalog.getIndex(self.name) @@ -45,10 +58,12 @@ def getFacet(self): ) brains_rids = querybuilder(**self.querybuilder_parameters) - brains_rids_criteria = self.brain_rids_criteria + brains_rids_mandatory = self.brain_rids_mandatory # Get the rids for the brains that have the facet index set to the value we are interested in - rids = intersection(brains_rids, index.documentToKeyMap()) - rids_criteria = intersection(brains_rids_criteria, index.documentToKeyMap()) + index_rids = index.documentToKeyMap() + rids = intersection(brains_rids, index_rids) + rids_mandatory = intersection(brains_rids_mandatory, index_rids) + for rid in rids: keys = index.keyForDocument(rid) if isinstance(keys, str): @@ -59,16 +74,16 @@ def getFacet(self): if key not in count: count[key] = 0 count[key] += 1 - for rid in rids_criteria: + for rid in rids_mandatory: keys = index.keyForDocument(rid) if isinstance(keys, str): keys = [keys] if not isinstance(keys, list): continue for key in keys: - if key not in count_criteria: - count_criteria[key] = 0 - count_criteria[key] += 1 + if key not in count_mandatory: + count_mandatory[key] = 0 + count_mandatory[key] += 1 results = { "name": self.name, @@ -76,11 +91,7 @@ def getFacet(self): "data": {}, } - for key, value in count.items(): - if key in count_criteria and count_criteria[key] > 0: - results["data"][key] = value - for key, value in count_criteria.items(): - if key not in results["data"]: - results["data"][key] = 0 + for key, _ in count_mandatory.items(): + results["data"][key] = count[key] if key in count else 0 return results diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index 3a8946c220..2501d6a703 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -30,25 +30,29 @@ def __init__(self, context, request, params): def __call__(self): self.setQuerybuilderParams() - querybuilder_criteria_parameters = self.querybuilder_parameters.copy() - querybuilder_criteria_parameters["query"] = [ + querybuilder_mandatory_parameters = self.querybuilder_parameters.copy() + querybuilder_mandatory_parameters["query"] = [ qs for qs in self.querybuilder_parameters.get("query", []) - if "criteria" in qs and qs["criteria"] is True + if "mandatory" in qs and qs["mandatory"] is True ] - querybuilder_criteria_parameters["rids"] = True + querybuilder_mandatory_parameters["rids"] = True + if SUPPORT_NOT_UUID_QUERIES: + querybuilder_mandatory_parameters.update( + dict(custom_query={"UID": {"not": self.context.UID()}}) + ) querybuilder = getMultiAdapter( (self.context, self.request), name="querybuilderresults" ) - brains_rids_criteria = querybuilder(**querybuilder_criteria_parameters) + brains_rids_mandatory = querybuilder(**querybuilder_mandatory_parameters) if len(self.params) > 0: results = Facet( self.context, self.request, name=self.params[0], querybuilder_parameters=self.querybuilder_parameters, - brains_rids_criteria=brains_rids_criteria, + brains_rids_mandatory=brains_rids_mandatory, ).getFacet() if results is None: raise BadRequest("Invalid facet") @@ -68,7 +72,7 @@ def __call__(self): self.request, name=facet, querybuilder_parameters=self.querybuilder_parameters, - brains_rids_criteria=brains_rids_criteria, + brains_rids_mandatory=brains_rids_mandatory, ).getFacet() if facet_results: results["facets_count"][facet] = facet_results From 8b80005572f2b538d811f4211c5e3401fc3ab208 Mon Sep 17 00:00:00 2001 From: Dobricean Ioan Dorian Date: Fri, 8 Mar 2024 10:42:34 +0200 Subject: [PATCH 13/14] make search work on PloneRoot --- src/plone/restapi/services/querystringsearch/facet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index 64e705a0dc..96f16e665c 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -11,6 +11,7 @@ else: SUPPORT_NOT_UUID_QUERIES = False + class Facet: """Returns facet count.""" @@ -41,7 +42,6 @@ def __init__( dict(custom_query={"UID": {"not": self.context.UID()}}) ) - def getFacet(self): ctool = getUtility(ICatalogTool) count = {} @@ -92,6 +92,6 @@ def getFacet(self): } for key, _ in count_mandatory.items(): - results["data"][key] = count[key] if key in count else 0 + results["data"][key] = count[key] if key in count else 0 return results From 59090a418952bcdc0c85e02e4f3a4c479cdde1ad Mon Sep 17 00:00:00 2001 From: Dobricean Ioan Dorian Date: Fri, 8 Mar 2024 10:43:58 +0200 Subject: [PATCH 14/14] make search work on PloneRoot --- src/plone/restapi/services/querystringsearch/facet.py | 2 ++ src/plone/restapi/services/querystringsearch/get.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/plone/restapi/services/querystringsearch/facet.py b/src/plone/restapi/services/querystringsearch/facet.py index 96f16e665c..4133ecb68f 100644 --- a/src/plone/restapi/services/querystringsearch/facet.py +++ b/src/plone/restapi/services/querystringsearch/facet.py @@ -37,6 +37,8 @@ def __init__( if "mandatory" in qs and qs["mandatory"] is True ] self.brain_rids_mandatory = brains_rids_mandatory + + # make serch work also on Plone Root if SUPPORT_NOT_UUID_QUERIES: self.querybuilder_parameters.update( dict(custom_query={"UID": {"not": self.context.UID()}}) diff --git a/src/plone/restapi/services/querystringsearch/get.py b/src/plone/restapi/services/querystringsearch/get.py index 2501d6a703..a4709e5608 100644 --- a/src/plone/restapi/services/querystringsearch/get.py +++ b/src/plone/restapi/services/querystringsearch/get.py @@ -37,6 +37,8 @@ def __call__(self): if "mandatory" in qs and qs["mandatory"] is True ] querybuilder_mandatory_parameters["rids"] = True + + # make serch work also on Plone Root if SUPPORT_NOT_UUID_QUERIES: querybuilder_mandatory_parameters.update( dict(custom_query={"UID": {"not": self.context.UID()}}) @@ -46,6 +48,7 @@ def __call__(self): ) brains_rids_mandatory = querybuilder(**querybuilder_mandatory_parameters) + if len(self.params) > 0: results = Facet( self.context,