AfO Register (#470)

* Implement AfO Regster serialiation & repos (WiP)

* Update resources & bind to context

* Add tests & update

* Fix test

* Update search, factory & tests

* Implement afo register suggestions & tests

* 'Refactored by Sourcery' (#484)

Co-authored-by: Sourcery AI <>

* Update query & sorting

* Update tests & format

* Clean up

* Update queries & sorting

* Update & refactor query collation, use in afo register queries

* Refactor

* Add text + textNumber queries (search by traditionalReferences)

* Add route & test

* Use post

* Update test

* Extend fragment query for `traditionalReferences`

---------

Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com>

ebl/afo_register/application/afo_register_repository.py

@@ -0,0 +1,29 @@
+from typing import Sequence
+from abc import ABC, abstractmethod
+
+from ebl.afo_register.domain.afo_register_record import (
+    AfoRegisterRecord,
+    AfoRegisterRecordSuggestion,
+)
+
+
+class AfoRegisterRepository(ABC):
+    @abstractmethod
+    def create(self, afo_register_record: AfoRegisterRecord) -> str:
+        ...
+
+    @abstractmethod
+    def search(self, query, *args, **kwargs) -> Sequence[AfoRegisterRecord]:
+        ...
+
+    @abstractmethod
+    def search_by_texts_and_numbers(
+        self, query_list: Sequence[str], *args, **kwargs
+    ) -> Sequence[AfoRegisterRecord]:
+        ...
+
+    @abstractmethod
+    def search_suggestions(
+        self, text_query: str, *args, **kwargs
+    ) -> Sequence[AfoRegisterRecordSuggestion]:
+        ...

ebl/afo_register/domain/afo_register_record.py

@@ -0,0 +1,19 @@
+import attr
+from typing import Sequence
+
+
+@attr.s(frozen=True, auto_attribs=True)
+class AfoRegisterRecord:
+    afo_number: str = ""
+    page: str = ""
+    text: str = ""
+    text_number: str = ""
+    lines_discussed: str = ""
+    discussed_by: str = ""
+    discussed_by_notes: str = ""
+
+
+@attr.s(frozen=True, auto_attribs=True)
+class AfoRegisterRecordSuggestion:
+    text: str = ""
+    text_numbers: Sequence[str] = tuple()

ebl/afo_register/infrastructure/mongo_afo_register_repository.py

@@ -0,0 +1,123 @@
+from marshmallow import Schema, fields, post_load, EXCLUDE
+from typing import cast, Sequence
+from pymongo.database import Database
+from natsort import natsorted
+from ebl.mongo_collection import MongoCollection
+from ebl.afo_register.domain.afo_register_record import (
+    AfoRegisterRecord,
+    AfoRegisterRecordSuggestion,
+)
+from ebl.afo_register.application.afo_register_repository import AfoRegisterRepository
+from ebl.common.query.query_collation import (
+    make_query_params,
+)
+
+
+COLLECTION = "afo_register"
+
+
+def create_search_query(query):
+    if "textNumber" not in query:
+        return query
+    text_number = query["textNumber"]
+    text_number_stripped = text_number.strip('"')
+    if text_number != text_number_stripped:
+        query["textNumber"] = text_number_stripped
+    else:
+        query["textNumber"] = {"$regex": f"^{text_number}.*", "$options": "i"}
+    return query
+
+
+def cast_with_sorting(
+    records: Sequence[AfoRegisterRecord],
+) -> Sequence[AfoRegisterRecord]:
+    return cast(
+        Sequence[AfoRegisterRecord],
+        natsorted(records, key=lambda record: f"${record.text} ${record.text_number}"),
+    )
+
+
+class AfoRegisterRecordSchema(Schema):
+    class Meta:
+        unknown = EXCLUDE
+
+    afo_number = fields.String(required=True, data_key="afoNumber")
+    page = fields.String(required=True)
+    text = fields.String(required=True)
+    text_number = fields.String(required=True, data_key="textNumber")
+    lines_discussed = fields.String(data_key="linesDiscussed")
+    discussed_by = fields.String(data_key="discussedBy")
+    discussed_by_notes = fields.String(data_key="discussedByNotes")
+
+    @post_load
+    def make_record(self, data, **kwargs):
+        return AfoRegisterRecord(**data)
+
+
+class AfoRegisterRecordSuggestionSchema(Schema):
+    text = fields.String(required=True)
+    text_numbers = fields.List(fields.String(), required=True, data_key="textNumbers")
+
+    @post_load
+    def make_suggestion(self, data, **kwargs):
+        data["text_numbers"] = natsorted(data["text_numbers"])
+        return AfoRegisterRecordSuggestion(**data)
+
+
+class MongoAfoRegisterRepository(AfoRegisterRepository):
+    def __init__(self, database: Database):
+        self._afo_register = MongoCollection(database, COLLECTION)
+
+    def create(self, afo_register_record: AfoRegisterRecord) -> str:
+        return self._afo_register.insert_one(
+            AfoRegisterRecordSchema().dump(afo_register_record)
+        )
+
+    def search(self, query, *args, **kwargs) -> Sequence[AfoRegisterRecord]:
+        data = self._afo_register.find_many(create_search_query(query))
+        records = AfoRegisterRecordSchema().load(data, many=True)
+        return cast_with_sorting(records)
+
+    def search_by_texts_and_numbers(
+        self, query_list: Sequence[str], *args, **kwargs
+    ) -> Sequence[AfoRegisterRecord]:
+        pipeline = [
+            {
+                "$addFields": {
+                    "combined_field": {"$concat": ["$text", " ", "$textNumber"]}
+                }
+            },
+            {"$match": {"combined_field": {"$in": query_list}}},
+            {"$group": {"_id": "$_id", "document": {"$first": "$$ROOT"}}},
+            {"$replaceRoot": {"newRoot": "$document"}},
+            {"$project": {"combined_field": 0}},
+        ]
+        data = self._afo_register.aggregate(pipeline)
+        records = AfoRegisterRecordSchema().load(data, many=True)
+        return cast_with_sorting(records)
+
+    def search_suggestions(
+        self, text_query: str, *args, **kwargs
+    ) -> Sequence[AfoRegisterRecordSuggestion]:
+        collated_query = list(make_query_params({"text": text_query}, "afo-register"))[
+            0
+        ]
+        pipeline = [
+            {"$match": {"text": {"$regex": collated_query.value, "$options": "i"}}},
+            {"$group": {"_id": "$text", "textNumbers": {"$addToSet": "$textNumber"}}},
+            {
+                "$project": {
+                    "text": "$_id",
+                    "_id": 0,
+                    "textNumbers": {"$setUnion": ["$textNumbers", []]},
+                }
+            },
+            {"$unwind": "$textNumbers"},
+            {"$sort": {"textNumbers": 1}},
+            {"$group": {"_id": "$text", "textNumbers": {"$push": "$textNumbers"}}},
+            {"$project": {"text": "$_id", "textNumbers": "$textNumbers", "_id": 0}},
+        ]
+        suggestions = AfoRegisterRecordSuggestionSchema().load(
+            self._afo_register.aggregate(pipeline), many=True
+        )
+        return cast(Sequence[AfoRegisterRecordSuggestion], suggestions)

ebl/afo_register/web/afo_register_records.py

@@ -0,0 +1,49 @@
+from falcon import Request, Response
+from ebl.errors import NotFoundError
+
+from ebl.afo_register.application.afo_register_repository import AfoRegisterRepository
+from ebl.afo_register.infrastructure.mongo_afo_register_repository import (
+    AfoRegisterRecordSchema,
+    AfoRegisterRecordSuggestionSchema,
+)
+
+
+class AfoRegisterResource:
+    def __init__(self, afoRegisterRepository: AfoRegisterRepository):
+        self._afoRegisterRepository = afoRegisterRepository
+
+    def on_get(self, req: Request, resp: Response) -> None:
+        try:
+            response = self._afoRegisterRepository.search(req.params)
+        except ValueError as error:
+            raise NotFoundError(
+                f"No AfO registry entries matching {str(req.params)} found."
+            ) from error
+        resp.media = AfoRegisterRecordSchema().dump(response, many=True)
+
+
+class AfoRegisterTextsAndNumbersResource:
+    def __init__(self, afoRegisterRepository: AfoRegisterRepository):
+        self._afoRegisterRepository = afoRegisterRepository
+
+    def on_post(self, req: Request, resp: Response) -> None:
+        try:
+            response = self._afoRegisterRepository.search_by_texts_and_numbers(
+                req.media
+            )
+        except ValueError as error:
+            raise NotFoundError(
+                f"No AfO registry entries matching {str(req.media)} found."
+            ) from error
+        resp.media = AfoRegisterRecordSchema().dump(response, many=True)
+
+
+class AfoRegisterSuggestionsResource:
+    def __init__(self, afoRegisterRepository: AfoRegisterRepository):
+        self._afoRegisterRepository = afoRegisterRepository
+
+    def on_get(self, req: Request, resp: Response) -> None:
+        response = self._afoRegisterRepository.search_suggestions(
+            req.params["text_query"]
+        )
+        resp.media = AfoRegisterRecordSuggestionSchema().dump(response, many=True)

ebl/afo_register/web/bootstrap.py

@@ -0,0 +1,21 @@
+import falcon
+from ebl.context import Context
+
+from ebl.afo_register.web.afo_register_records import (
+    AfoRegisterResource,
+    AfoRegisterTextsAndNumbersResource,
+    AfoRegisterSuggestionsResource,
+)
+
+
+def create_afo_register_routes(api: falcon.App, context: Context):
+    afo_register_search = AfoRegisterResource(context.afo_register_repository)
+    afo_register_search_texts_and_numbers = AfoRegisterTextsAndNumbersResource(
+        context.afo_register_repository
+    )
+    afo_register_suggestions_search = AfoRegisterSuggestionsResource(
+        context.afo_register_repository
+    )
+    api.add_route("/afo-register", afo_register_search)
+    api.add_route("/afo-register/texts-numbers", afo_register_search_texts_and_numbers)
+    api.add_route("/afo-register/suggestions", afo_register_suggestions_search)

ebl/app.py

@@ -42,10 +42,14 @@
 from ebl.lemmatization.web.bootstrap import create_lemmatization_routes
 from ebl.signs.infrastructure.mongo_sign_repository import MongoSignRepository
 from ebl.signs.web.bootstrap import create_signs_routes
+from ebl.afo_register.web.bootstrap import create_afo_register_routes
 from ebl.transliteration.application.parallel_line_injector import ParallelLineInjector
 from ebl.transliteration.infrastructure.mongo_parallel_repository import (
     MongoParallelRepository,
 )
+from ebl.afo_register.infrastructure.mongo_afo_register_repository import (
+    MongoAfoRegisterRepository,
+)
 from ebl.users.domain.user import Guest
 from ebl.users.infrastructure.auth0 import Auth0Backend
 from ebl.fragmentarium.infrastructure.mongo_findspot_repository import (
@@ -93,6 +97,7 @@ def create_context():
         text_repository=MongoTextRepository(database),
         annotations_repository=MongoAnnotationsRepository(database),
         lemma_repository=MongoLemmaRepository(database),
+        afo_register_repository=MongoAfoRegisterRepository(database),
         findspot_repository=MongoFindspotRepository(database),
         custom_cache=custom_cache,
         cache=cache,
@@ -121,12 +126,12 @@ def create_app(context: Context, issuer: str = "", audience: str = ""):
     create_fragmentarium_routes(api, context)
     create_lemmatization_routes(api, context)
     create_markup_route(api, context)
+    create_afo_register_routes(api, context)
 
     return api
 
 
 def get_app():
     sentry_sdk.init(dsn=os.environ["SENTRY_DSN"], integrations=[FalconIntegration()])
     context = create_context()
-
     return create_app(context, os.environ["AUTH0_ISSUER"], os.environ["AUTH0_AUDIENCE"])