From b94ab86d2c237f8dba00d58316157ee00a086779 Mon Sep 17 00:00:00 2001 From: Wim De Clercq Date: Tue, 10 Jan 2023 11:12:32 +0100 Subject: [PATCH] Sqlalchemy 2.0 changes part 2 Issue #90 --- skosprovider_sqlalchemy/providers.py | 274 ++++++++++++++++----------- 1 file changed, 160 insertions(+), 114 deletions(-) diff --git a/skosprovider_sqlalchemy/providers.py b/skosprovider_sqlalchemy/providers.py index c3d7bd1..c625281 100644 --- a/skosprovider_sqlalchemy/providers.py +++ b/skosprovider_sqlalchemy/providers.py @@ -8,8 +8,9 @@ from skosprovider.skos import Note from skosprovider.skos import Source from skosprovider.uri import DefaultUrnGenerator +from sqlalchemy import select +from sqlalchemy.exc import NoResultFound from sqlalchemy.orm import joinedload -from sqlalchemy.orm.exc import NoResultFound from skosprovider_sqlalchemy.models import Collection as CollectionModel from skosprovider_sqlalchemy.models import Concept as ConceptModel @@ -56,7 +57,7 @@ def __init__(self, metadata, session, **kwargs): :param :class:`sqlachemy.orm.session.Session` session: The database session. This can also be a callable that returns a Session. ''' - if not 'subject' in metadata: + if 'subject' not in metadata: metadata['subject'] = [] self.metadata = metadata if 'uri_generator' in kwargs: @@ -65,7 +66,7 @@ def __init__(self, metadata, session, **kwargs): self.uri_generator = DefaultUrnGenerator(self.metadata.get('id')) try: self.session = session() - except TypeError as e: + except TypeError: self.session = session try: self.conceptscheme_id = int(metadata.get( @@ -94,7 +95,6 @@ def _get_concept_scheme(self): ''' Find a :class:`skosprovider.skos.ConceptScheme` for this provider. - :param id: Id of a conceptscheme. :rtype: :class:`skosprovider.skos.ConceptScheme` ''' csm = ( @@ -113,15 +113,15 @@ def _get_concept_scheme(self): return ConceptScheme( uri=csm.uri, labels=[ - Label(l.label, l.labeltype_id, l.language_id) - for l in csm.labels + Label(label.label, label.labeltype_id, label.language_id) + for label in csm.labels ], notes=[ Note(n.note, n.notetype_id, n.language_id, n.markup) for n in csm.notes ], languages=[ - l.id for l in csm.languages + language.id for language in csm.languages ], sources=[ Source(s.citation, s.markup) for s in csm.sources @@ -136,40 +136,51 @@ def _from_thing(self, thing): to load. ''' if thing.type and thing.type == 'collection': + if thing.uri is not None: + uri = thing.uri + else: + uri = self.uri_generator.generate(type='collection', id=thing.concept_id) return Collection( id=thing.concept_id, - uri=thing.uri if thing.uri is not None else self.uri_generator.generate(type='collection', id=thing.concept_id), + uri=uri, concept_scheme=self.concept_scheme, labels=[ - Label(l.label, l.labeltype_id, l.language_id) - for l in thing.labels + Label(label.label, label.labeltype_id, label.language_id) + for label in thing.labels ], notes=[ Note(n.note, n.notetype_id, n.language_id, n.markup) for n in thing.notes ], sources=[ - Source(s.citation, s.markup ) for s in thing.sources + Source(s.citation, s.markup) for s in thing.sources ], - members=[member.concept_id for member in thing.members] if hasattr(thing, 'members') else [], + members=[member.concept_id for member in getattr(thing, 'members', [])], member_of=[member_of.concept_id for member_of in thing.member_of], - superordinates=[broader_concept.concept_id for broader_concept in thing.broader_concepts], + superordinates=[ + broader_concept.concept_id + for broader_concept in thing.broader_concepts + ], infer_concept_relations=thing.infer_concept_relations ) else: + if thing.uri is not None: + uri = thing.uri + else: + uri = self.uri_generator.generate(type='concept', id=thing.concept_id) matches = {} for m in thing.matches: key = m.matchtype.name[:m.matchtype.name.find('Match')] - if not key in matches: + if key not in matches: matches[key] = [] matches[key].append(m.uri) return Concept( id=thing.concept_id, - uri=thing.uri if thing.uri is not None else self.uri_generator.generate(type='concept', id=thing.concept_id), + uri=uri, concept_scheme=self.concept_scheme, labels=[ - Label(l.label, l.labeltype_id, l.language_id) - for l in thing.labels + Label(label.label, label.labeltype_id, label.language_id) + for label in thing.labels ], notes=[ Note(n.note, n.notetype_id, n.language_id, n.markup) @@ -182,21 +193,25 @@ def _from_thing(self, thing): narrower=[c.concept_id for c in thing.narrower_concepts], related=[c.concept_id for c in thing.related_concepts], member_of=[member_of.concept_id for member_of in thing.member_of], - subordinate_arrays=[narrower_collection.concept_id for narrower_collection in thing.narrower_collections], + subordinate_arrays=[ + narrower_collection.concept_id + for narrower_collection in thing.narrower_collections + ], matches=matches ) def get_by_id(self, concept_id): try: - thing = self.session\ - .query(Thing)\ - .options(joinedload(Thing.labels))\ - .options(joinedload(Thing.notes))\ - .options(joinedload(Thing.sources))\ - .filter( - Thing.concept_id == str(concept_id), - Thing.conceptscheme_id == self.conceptscheme_id - ).one() + thing = self.session.execute( + select(Thing) + .options(joinedload(Thing.labels)) + .options(joinedload(Thing.notes)) + .options(joinedload(Thing.sources)) + .filter( + Thing.concept_id == str(concept_id), + Thing.conceptscheme_id == self.conceptscheme_id + ) + ).unique().scalar_one() except NoResultFound: return False return self._from_thing(thing) @@ -215,15 +230,16 @@ def get_by_uri(self, uri): collection is unknown to the provider. ''' try: - thing = self.session\ - .query(Thing)\ - .options(joinedload(Thing.labels))\ - .options(joinedload(Thing.notes))\ - .options(joinedload(Thing.sources))\ - .filter( - Thing.uri == uri, - Thing.conceptscheme_id == self.conceptscheme_id - ).one() + thing = self.session.execute( + select(Thing) + .options(joinedload(Thing.labels)) + .options(joinedload(Thing.notes)) + .options(joinedload(Thing.sources)) + .filter( + Thing.uri == uri, + Thing.conceptscheme_id == self.conceptscheme_id + ) + ).unique().scalar_one() except NoResultFound: return False return self._from_thing(thing) @@ -233,12 +249,12 @@ def _get_id_and_label(self, c, lan): :param skosprovider_sqlalchemy.models.Thing c: A concept or collection. :param string lan: A language (eg. "en", "nl", "la", "fr") ''' - l = c.label(lan) + label = c.label(lan) return { 'id': c.concept_id, 'uri': c.uri, 'type': c.type, - 'label': l.label if l is not None else None + 'label': label.label if label is not None else None } def find(self, query, **kwargs): @@ -251,11 +267,12 @@ def find(self, query, **kwargs): 'Please provide a URI to match with.' ) model = ConceptModel - q = self.session\ - .query(model)\ - .options(joinedload(model.labels))\ - .join(MatchModel)\ - .filter(model.conceptscheme_id == self.conceptscheme_id) + q = ( + select(model) + .options(joinedload(model.labels)) + .join(MatchModel) + .filter(model.conceptscheme_id == self.conceptscheme_id) + ) mtype = query['matches'].get('type') if mtype and mtype in Concept.matchtypes: mtype += 'Match' @@ -269,10 +286,11 @@ def find(self, query, **kwargs): else: q = q.filter(MatchModel.uri == match_uri) else: - q = self.session\ - .query(model)\ - .options(joinedload(model.labels))\ - .filter(model.conceptscheme_id == self.conceptscheme_id) + q = ( + select(model) + .options(joinedload(model.labels)) + .filter(model.conceptscheme_id == self.conceptscheme_id) + ) if 'type' in query and query['type'] in ['concept', 'collection']: q = q.filter(model.type == query['type']) if 'label' in query: @@ -292,51 +310,67 @@ def find(self, query, **kwargs): else: members = coll.members q = q.filter(model.concept_id.in_(members)) - all = q.all() + things = self.session.execute(q).unique().scalars().all() sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(all, sort, lan, sort_order=='desc')] + return [ + self._get_id_and_label(c, lan) + for c in self._sort(things, sort, lan, sort_order == 'desc') + ] def get_all(self, **kwargs): - all = self.session\ - .query(Thing)\ - .options(joinedload(Thing.labels))\ - .filter(Thing.conceptscheme_id == self.conceptscheme_id)\ - .all() + things = self.session.execute( + select(Thing) + .options(joinedload(Thing.labels)) + .filter(Thing.conceptscheme_id == self.conceptscheme_id) + ).unique().scalars().all() lan = self._get_language(**kwargs) sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(all, sort, lan, sort_order=='desc')] + return [ + self._get_id_and_label(c, lan) + for c in self._sort(things, sort, lan, sort_order == 'desc') + ] def get_top_concepts(self, **kwargs): # get the concepts that have no direct broader concept - top = self.session\ - .query(ConceptModel)\ - .options(joinedload(ConceptModel.labels))\ - .filter( - ConceptModel.conceptscheme_id == self.conceptscheme_id, - ConceptModel.broader_concepts == None - ).all() + top = self.session.execute( + select(ConceptModel) + .options(joinedload(ConceptModel.labels)) + .filter( + ConceptModel.conceptscheme_id == self.conceptscheme_id, + ~ConceptModel.broader_concepts.any() + ) + ).unique().scalars().all() + # check if they have an indirect broader concept def _has_higher_concept(c): for coll in c.member_of: - if coll.infer_concept_relations and (coll.broader_concepts or _has_higher_concept(coll)): + if ( + coll.infer_concept_relations + and coll.broader_concepts or _has_higher_concept(coll) + ): return True return False top = [c for c in top if not _has_higher_concept(c)] lan = self._get_language(**kwargs) sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(top, sort, lan, sort_order=='desc')] - def expand(self, id): + return [ + self._get_id_and_label(c, lan) + for c in self._sort(top, sort, lan, sort_order == 'desc') + ] + + def expand(self, concept_id): try: - thing = self.session\ - .query(Thing)\ - .filter( - Thing.concept_id == str(id), - Thing.conceptscheme_id == self.conceptscheme_id - ).one() + thing = self.session.execute( + select(Thing) + .filter( + Thing.concept_id == str(concept_id), + Thing.conceptscheme_id == self.conceptscheme_id + ) + ).scalar_one() except NoResultFound: return False @@ -361,27 +395,30 @@ def _expand_recurse(self, thing): def _expand_visit(self, thing): if thing.type == 'collection': - ret = [] + concept_ids = [] for m in thing.members: - ret += self._expand_visit(m) + concept_ids += self._expand_visit(m) else: try: - cov = self.session\ - .query(Visitation.lft, Visitation.rght)\ - .filter(Visitation.conceptscheme_id == self.conceptscheme_id)\ - .filter(Visitation.concept_id == thing.id)\ - .one() + visitation = self.session.execute( + select(Visitation.lft, Visitation.rght) + .filter( + Visitation.conceptscheme_id == self.conceptscheme_id, + Visitation.concept_id == thing.id + ) + ).one() except NoResultFound: return self._expand_recurse(thing) - ids = self.session\ - .query(Thing.concept_id)\ - .join(Visitation)\ - .filter(Thing.conceptscheme_id == self.conceptscheme_id)\ - .filter(Visitation.lft.between(cov[0], cov[1]))\ - .all() - ret = [id[0] for id in ids] - return list(set(ret)) + concept_ids = self.session.execute( + select(Thing.concept_id) + .join(Visitation) + .filter( + Thing.conceptscheme_id == self.conceptscheme_id, + Visitation.lft.between(visitation.lft, visitation.rght) + ) + ).scalars().all() + return list(set(concept_ids)) def get_top_display(self, **kwargs): ''' @@ -396,34 +433,39 @@ def get_top_display(self, **kwargs): the `**kwargs` parameter, the default language of the provider and falls back to `en` if nothing is present. ''' - tco = self.session\ - .query(ConceptModel)\ - .options(joinedload(ConceptModel.labels))\ - .filter( - ConceptModel.conceptscheme_id == self.conceptscheme_id, - ConceptModel.broader_concepts == None, - ConceptModel.member_of == None - ).all() - tcl = self.session\ - .query(CollectionModel)\ - .options(joinedload(CollectionModel.labels))\ - .filter( - CollectionModel.conceptscheme_id == self.conceptscheme_id, - CollectionModel.broader_concepts == None, - CollectionModel.member_of == None - ).all() - res = tco + tcl + top_concepts = self.session.execute( + select(ConceptModel) + .options(joinedload(ConceptModel.labels)) + .filter( + ConceptModel.conceptscheme_id == self.conceptscheme_id, + ~ConceptModel.broader_concepts.any(), + ~ConceptModel.member_of.any() + ) + ).unique().scalars().all() + top_collections = self.session.execute( + select(CollectionModel) + .options(joinedload(CollectionModel.labels)) + .filter( + CollectionModel.conceptscheme_id == self.conceptscheme_id, + ~CollectionModel.broader_concepts.any(), + ~CollectionModel.member_of.any() + ) + ).unique().scalars().all() + res = top_concepts + top_collections lan = self._get_language(**kwargs) sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(res, sort, lan, sort_order=='desc')] + return [ + self._get_id_and_label(c, lan) + for c in self._sort(res, sort, lan, sort_order == 'desc') + ] - def get_children_display(self, id, **kwargs): + def get_children_display(self, thing_id, **kwargs): ''' Return a list of concepts or collections that should be displayed under this concept or collection. - :param id: A concept or collection id. + :param thing_id: A concept or collection id. :rtype: A list of concepts and collections. For each an id is present and a label. The label is determined by looking at the `**kwargs` parameter, the default language of the provider @@ -431,12 +473,13 @@ def get_children_display(self, id, **kwargs): exist, return `False`. ''' try: - thing = self.session\ - .query(Thing)\ - .filter( - Thing.concept_id == str(id), - Thing.conceptscheme_id == self.conceptscheme_id - ).one() + thing = self.session.execute( + select(Thing) + .filter( + Thing.concept_id == str(thing_id), + Thing.conceptscheme_id == self.conceptscheme_id + ) + ).scalar_one() except NoResultFound: return False lan = self._get_language(**kwargs) @@ -444,10 +487,13 @@ def get_children_display(self, id, **kwargs): if thing.type == 'concept': if len(thing.narrower_collections) > 0: res += thing.narrower_collections - elif len(thing.narrower_concepts)>0: + elif len(thing.narrower_concepts) > 0: res += thing.narrower_concepts if thing.type == 'collection' and hasattr(thing, 'members'): res += thing.members sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(res, sort, lan, sort_order=='desc')] + return [ + self._get_id_and_label(c, lan) + for c in self._sort(res, sort, lan, sort_order == 'desc') + ]