diff --git a/requirements-dev.txt b/requirements-dev.txt index f1db642..922b798 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,18 +2,15 @@ --requirement requirements.txt # Postgresql database access -psycopg2==2.9.2 +psycopg2==2.9.5 # Documentation -Sphinx==4.3.1 +Sphinx==6.0.0 # Testing -pytest==6.2.5 -pytest-cov==3.0.0 +pytest==7.2.0 +pytest-cov==4.0.0 coveralls==3.3.1 # Linting -flake8==4.0.1 -mccabe==0.6.1 -pep8==1.7.1 -pyflakes==2.4.0 +flake8==6.0.0 diff --git a/requirements.txt b/requirements.txt index 83ea0cc..599923d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -SQLAlchemy==1.4.28 -skosprovider==1.1.0 +SQLAlchemy==1.4.45 +skosprovider==1.2.0 \ No newline at end of file diff --git a/setup.py b/setup.py index c80754f..71236f3 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,5 @@ #!/usr/bin/env python -import os -import sys - try: from setuptools import setup except ImportError: @@ -13,7 +10,7 @@ ] requires = [ - 'skosprovider>=1.1.0', + 'skosprovider>=1.2.0', 'sqlalchemy', ] diff --git a/skosprovider_sqlalchemy/models.py b/skosprovider_sqlalchemy/models.py index 3d6d14a..f83e81a 100644 --- a/skosprovider_sqlalchemy/models.py +++ b/skosprovider_sqlalchemy/models.py @@ -10,13 +10,13 @@ from sqlalchemy import Text from sqlalchemy import UniqueConstraint from sqlalchemy import event -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import orm from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import backref from sqlalchemy.orm import relationship log = logging.getLogger(__name__) -Base = declarative_base() +Base = orm.declarative_base() concept_label = Table( 'concept_label', @@ -199,7 +199,9 @@ class Thing(Base): single_parent=True ) - conceptscheme = relationship('ConceptScheme', backref='concepts') + conceptscheme = relationship( + 'ConceptScheme', backref=orm.backref('concepts', cascade_backrefs=False) + ) conceptscheme_id = Column( Integer, ForeignKey('conceptscheme.id'), @@ -253,19 +255,21 @@ class Concept(Thing): narrower_concepts = relationship( 'Concept', secondary=concept_hierarchy_concept, - backref=backref('broader_concepts', collection_class=set), + backref=backref('broader_concepts', collection_class=set, cascade_backrefs=False), primaryjoin='Concept.id==concept_hierarchy_concept.c.concept_id_broader', secondaryjoin='Concept.id==concept_hierarchy_concept.c.concept_id_narrower', - collection_class=set + collection_class=set, + cascade_backrefs=False, ) narrower_collections = relationship( 'Collection', secondary=concept_hierarchy_collection, - backref=backref('broader_concepts', collection_class=set), + backref=backref('broader_concepts', collection_class=set, cascade_backrefs=False), primaryjoin='Concept.id==concept_hierarchy_collection.c.concept_id_broader', secondaryjoin='Concept.id==concept_hierarchy_collection.c.collection_id_narrower', - collection_class=set + collection_class=set, + cascade_backrefs=False, ) __mapper_args__ = { diff --git a/skosprovider_sqlalchemy/providers.py b/skosprovider_sqlalchemy/providers.py index 3b5c381..c625281 100644 --- a/skosprovider_sqlalchemy/providers.py +++ b/skosprovider_sqlalchemy/providers.py @@ -8,8 +8,9 @@ from skosprovider.skos import Note from skosprovider.skos import Source from skosprovider.uri import DefaultUrnGenerator +from sqlalchemy import select +from sqlalchemy.exc import NoResultFound from sqlalchemy.orm import joinedload -from sqlalchemy.orm.exc import NoResultFound from skosprovider_sqlalchemy.models import Collection as CollectionModel from skosprovider_sqlalchemy.models import Concept as ConceptModel @@ -56,7 +57,7 @@ def __init__(self, metadata, session, **kwargs): :param :class:`sqlachemy.orm.session.Session` session: The database session. This can also be a callable that returns a Session. ''' - if not 'subject' in metadata: + if 'subject' not in metadata: metadata['subject'] = [] self.metadata = metadata if 'uri_generator' in kwargs: @@ -65,7 +66,7 @@ def __init__(self, metadata, session, **kwargs): self.uri_generator = DefaultUrnGenerator(self.metadata.get('id')) try: self.session = session() - except TypeError as e: + except TypeError: self.session = session try: self.conceptscheme_id = int(metadata.get( @@ -94,28 +95,33 @@ def _get_concept_scheme(self): ''' Find a :class:`skosprovider.skos.ConceptScheme` for this provider. - :param id: Id of a conceptscheme. :rtype: :class:`skosprovider.skos.ConceptScheme` ''' - csm = self.session\ - .query(ConceptSchemeModel)\ - .options(joinedload('labels'))\ - .options(joinedload('notes'))\ - .options(joinedload('languages'))\ - .options(joinedload('sources'))\ - .get(self.conceptscheme_id) + csm = ( + self.session + .get( + ConceptSchemeModel, + self.conceptscheme_id, + options=[ + joinedload(ConceptSchemeModel.labels), + joinedload(ConceptSchemeModel.notes), + joinedload(ConceptSchemeModel.languages), + joinedload(ConceptSchemeModel.sources), + ] + ) + ) return ConceptScheme( uri=csm.uri, labels=[ - Label(l.label, l.labeltype_id, l.language_id) - for l in csm.labels + Label(label.label, label.labeltype_id, label.language_id) + for label in csm.labels ], notes=[ Note(n.note, n.notetype_id, n.language_id, n.markup) for n in csm.notes ], languages=[ - l.id for l in csm.languages + language.id for language in csm.languages ], sources=[ Source(s.citation, s.markup) for s in csm.sources @@ -130,40 +136,51 @@ def _from_thing(self, thing): to load. ''' if thing.type and thing.type == 'collection': + if thing.uri is not None: + uri = thing.uri + else: + uri = self.uri_generator.generate(type='collection', id=thing.concept_id) return Collection( id=thing.concept_id, - uri=thing.uri if thing.uri is not None else self.uri_generator.generate(type='collection', id=thing.concept_id), + uri=uri, concept_scheme=self.concept_scheme, labels=[ - Label(l.label, l.labeltype_id, l.language_id) - for l in thing.labels + Label(label.label, label.labeltype_id, label.language_id) + for label in thing.labels ], notes=[ Note(n.note, n.notetype_id, n.language_id, n.markup) for n in thing.notes ], sources=[ - Source(s.citation, s.markup ) for s in thing.sources + Source(s.citation, s.markup) for s in thing.sources ], - members=[member.concept_id for member in thing.members] if hasattr(thing, 'members') else [], + members=[member.concept_id for member in getattr(thing, 'members', [])], member_of=[member_of.concept_id for member_of in thing.member_of], - superordinates=[broader_concept.concept_id for broader_concept in thing.broader_concepts], + superordinates=[ + broader_concept.concept_id + for broader_concept in thing.broader_concepts + ], infer_concept_relations=thing.infer_concept_relations ) else: + if thing.uri is not None: + uri = thing.uri + else: + uri = self.uri_generator.generate(type='concept', id=thing.concept_id) matches = {} for m in thing.matches: key = m.matchtype.name[:m.matchtype.name.find('Match')] - if not key in matches: + if key not in matches: matches[key] = [] matches[key].append(m.uri) return Concept( id=thing.concept_id, - uri=thing.uri if thing.uri is not None else self.uri_generator.generate(type='concept', id=thing.concept_id), + uri=uri, concept_scheme=self.concept_scheme, labels=[ - Label(l.label, l.labeltype_id, l.language_id) - for l in thing.labels + Label(label.label, label.labeltype_id, label.language_id) + for label in thing.labels ], notes=[ Note(n.note, n.notetype_id, n.language_id, n.markup) @@ -176,21 +193,25 @@ def _from_thing(self, thing): narrower=[c.concept_id for c in thing.narrower_concepts], related=[c.concept_id for c in thing.related_concepts], member_of=[member_of.concept_id for member_of in thing.member_of], - subordinate_arrays=[narrower_collection.concept_id for narrower_collection in thing.narrower_collections], + subordinate_arrays=[ + narrower_collection.concept_id + for narrower_collection in thing.narrower_collections + ], matches=matches ) def get_by_id(self, concept_id): try: - thing = self.session\ - .query(Thing)\ - .options(joinedload('labels'))\ - .options(joinedload('notes'))\ - .options(joinedload('sources'))\ - .filter( - Thing.concept_id == str(concept_id), - Thing.conceptscheme_id == self.conceptscheme_id - ).one() + thing = self.session.execute( + select(Thing) + .options(joinedload(Thing.labels)) + .options(joinedload(Thing.notes)) + .options(joinedload(Thing.sources)) + .filter( + Thing.concept_id == str(concept_id), + Thing.conceptscheme_id == self.conceptscheme_id + ) + ).unique().scalar_one() except NoResultFound: return False return self._from_thing(thing) @@ -209,15 +230,16 @@ def get_by_uri(self, uri): collection is unknown to the provider. ''' try: - thing = self.session\ - .query(Thing)\ - .options(joinedload('labels'))\ - .options(joinedload('notes'))\ - .options(joinedload('sources'))\ - .filter( - Thing.uri == uri, - Thing.conceptscheme_id == self.conceptscheme_id - ).one() + thing = self.session.execute( + select(Thing) + .options(joinedload(Thing.labels)) + .options(joinedload(Thing.notes)) + .options(joinedload(Thing.sources)) + .filter( + Thing.uri == uri, + Thing.conceptscheme_id == self.conceptscheme_id + ) + ).unique().scalar_one() except NoResultFound: return False return self._from_thing(thing) @@ -227,12 +249,12 @@ def _get_id_and_label(self, c, lan): :param skosprovider_sqlalchemy.models.Thing c: A concept or collection. :param string lan: A language (eg. "en", "nl", "la", "fr") ''' - l = c.label(lan) + label = c.label(lan) return { 'id': c.concept_id, 'uri': c.uri, 'type': c.type, - 'label': l.label if l is not None else None + 'label': label.label if label is not None else None } def find(self, query, **kwargs): @@ -245,11 +267,12 @@ def find(self, query, **kwargs): 'Please provide a URI to match with.' ) model = ConceptModel - q = self.session\ - .query(model)\ - .options(joinedload('labels'))\ - .join(MatchModel)\ - .filter(model.conceptscheme_id == self.conceptscheme_id) + q = ( + select(model) + .options(joinedload(model.labels)) + .join(MatchModel) + .filter(model.conceptscheme_id == self.conceptscheme_id) + ) mtype = query['matches'].get('type') if mtype and mtype in Concept.matchtypes: mtype += 'Match' @@ -263,10 +286,11 @@ def find(self, query, **kwargs): else: q = q.filter(MatchModel.uri == match_uri) else: - q = self.session\ - .query(model)\ - .options(joinedload('labels'))\ - .filter(model.conceptscheme_id == self.conceptscheme_id) + q = ( + select(model) + .options(joinedload(model.labels)) + .filter(model.conceptscheme_id == self.conceptscheme_id) + ) if 'type' in query and query['type'] in ['concept', 'collection']: q = q.filter(model.type == query['type']) if 'label' in query: @@ -286,51 +310,67 @@ def find(self, query, **kwargs): else: members = coll.members q = q.filter(model.concept_id.in_(members)) - all = q.all() + things = self.session.execute(q).unique().scalars().all() sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(all, sort, lan, sort_order=='desc')] + return [ + self._get_id_and_label(c, lan) + for c in self._sort(things, sort, lan, sort_order == 'desc') + ] def get_all(self, **kwargs): - all = self.session\ - .query(Thing)\ - .options(joinedload('labels'))\ - .filter(Thing.conceptscheme_id == self.conceptscheme_id)\ - .all() + things = self.session.execute( + select(Thing) + .options(joinedload(Thing.labels)) + .filter(Thing.conceptscheme_id == self.conceptscheme_id) + ).unique().scalars().all() lan = self._get_language(**kwargs) sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(all, sort, lan, sort_order=='desc')] + return [ + self._get_id_and_label(c, lan) + for c in self._sort(things, sort, lan, sort_order == 'desc') + ] def get_top_concepts(self, **kwargs): # get the concepts that have no direct broader concept - top = self.session\ - .query(ConceptModel)\ - .options(joinedload('labels'))\ - .filter( - ConceptModel.conceptscheme_id == self.conceptscheme_id, - ConceptModel.broader_concepts == None - ).all() + top = self.session.execute( + select(ConceptModel) + .options(joinedload(ConceptModel.labels)) + .filter( + ConceptModel.conceptscheme_id == self.conceptscheme_id, + ~ConceptModel.broader_concepts.any() + ) + ).unique().scalars().all() + # check if they have an indirect broader concept def _has_higher_concept(c): for coll in c.member_of: - if coll.infer_concept_relations and (coll.broader_concepts or _has_higher_concept(coll)): + if ( + coll.infer_concept_relations + and coll.broader_concepts or _has_higher_concept(coll) + ): return True return False top = [c for c in top if not _has_higher_concept(c)] lan = self._get_language(**kwargs) sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(top, sort, lan, sort_order=='desc')] - def expand(self, id): + return [ + self._get_id_and_label(c, lan) + for c in self._sort(top, sort, lan, sort_order == 'desc') + ] + + def expand(self, concept_id): try: - thing = self.session\ - .query(Thing)\ - .filter( - Thing.concept_id == str(id), - Thing.conceptscheme_id == self.conceptscheme_id - ).one() + thing = self.session.execute( + select(Thing) + .filter( + Thing.concept_id == str(concept_id), + Thing.conceptscheme_id == self.conceptscheme_id + ) + ).scalar_one() except NoResultFound: return False @@ -355,27 +395,30 @@ def _expand_recurse(self, thing): def _expand_visit(self, thing): if thing.type == 'collection': - ret = [] + concept_ids = [] for m in thing.members: - ret += self._expand_visit(m) + concept_ids += self._expand_visit(m) else: try: - cov = self.session\ - .query(Visitation.lft, Visitation.rght)\ - .filter(Visitation.conceptscheme_id == self.conceptscheme_id)\ - .filter(Visitation.concept_id == thing.id)\ - .one() + visitation = self.session.execute( + select(Visitation.lft, Visitation.rght) + .filter( + Visitation.conceptscheme_id == self.conceptscheme_id, + Visitation.concept_id == thing.id + ) + ).one() except NoResultFound: return self._expand_recurse(thing) - ids = self.session\ - .query(Thing.concept_id)\ - .join(Visitation)\ - .filter(Thing.conceptscheme_id == self.conceptscheme_id)\ - .filter(Visitation.lft.between(cov[0], cov[1]))\ - .all() - ret = [id[0] for id in ids] - return list(set(ret)) + concept_ids = self.session.execute( + select(Thing.concept_id) + .join(Visitation) + .filter( + Thing.conceptscheme_id == self.conceptscheme_id, + Visitation.lft.between(visitation.lft, visitation.rght) + ) + ).scalars().all() + return list(set(concept_ids)) def get_top_display(self, **kwargs): ''' @@ -390,34 +433,39 @@ def get_top_display(self, **kwargs): the `**kwargs` parameter, the default language of the provider and falls back to `en` if nothing is present. ''' - tco = self.session\ - .query(ConceptModel)\ - .options(joinedload('labels'))\ - .filter( - ConceptModel.conceptscheme_id == self.conceptscheme_id, - ConceptModel.broader_concepts == None, - ConceptModel.member_of == None - ).all() - tcl = self.session\ - .query(CollectionModel)\ - .options(joinedload('labels'))\ - .filter( - CollectionModel.conceptscheme_id == self.conceptscheme_id, - CollectionModel.broader_concepts == None, - CollectionModel.member_of == None - ).all() - res = tco + tcl + top_concepts = self.session.execute( + select(ConceptModel) + .options(joinedload(ConceptModel.labels)) + .filter( + ConceptModel.conceptscheme_id == self.conceptscheme_id, + ~ConceptModel.broader_concepts.any(), + ~ConceptModel.member_of.any() + ) + ).unique().scalars().all() + top_collections = self.session.execute( + select(CollectionModel) + .options(joinedload(CollectionModel.labels)) + .filter( + CollectionModel.conceptscheme_id == self.conceptscheme_id, + ~CollectionModel.broader_concepts.any(), + ~CollectionModel.member_of.any() + ) + ).unique().scalars().all() + res = top_concepts + top_collections lan = self._get_language(**kwargs) sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(res, sort, lan, sort_order=='desc')] + return [ + self._get_id_and_label(c, lan) + for c in self._sort(res, sort, lan, sort_order == 'desc') + ] - def get_children_display(self, id, **kwargs): + def get_children_display(self, thing_id, **kwargs): ''' Return a list of concepts or collections that should be displayed under this concept or collection. - :param id: A concept or collection id. + :param thing_id: A concept or collection id. :rtype: A list of concepts and collections. For each an id is present and a label. The label is determined by looking at the `**kwargs` parameter, the default language of the provider @@ -425,12 +473,13 @@ def get_children_display(self, id, **kwargs): exist, return `False`. ''' try: - thing = self.session\ - .query(Thing)\ - .filter( - Thing.concept_id == str(id), - Thing.conceptscheme_id == self.conceptscheme_id - ).one() + thing = self.session.execute( + select(Thing) + .filter( + Thing.concept_id == str(thing_id), + Thing.conceptscheme_id == self.conceptscheme_id + ) + ).scalar_one() except NoResultFound: return False lan = self._get_language(**kwargs) @@ -438,10 +487,13 @@ def get_children_display(self, id, **kwargs): if thing.type == 'concept': if len(thing.narrower_collections) > 0: res += thing.narrower_collections - elif len(thing.narrower_concepts)>0: + elif len(thing.narrower_concepts) > 0: res += thing.narrower_concepts if thing.type == 'collection' and hasattr(thing, 'members'): res += thing.members sort = self._get_sort(**kwargs) sort_order = self._get_sort_order(**kwargs) - return [self._get_id_and_label(c, lan) for c in self._sort(res, sort, lan, sort_order=='desc')] + return [ + self._get_id_and_label(c, lan) + for c in self._sort(res, sort, lan, sort_order == 'desc') + ] diff --git a/skosprovider_sqlalchemy/scripts/calc_visitation.py b/skosprovider_sqlalchemy/scripts/calc_visitation.py index 0c7cc43..6ac0343 100644 --- a/skosprovider_sqlalchemy/scripts/calc_visitation.py +++ b/skosprovider_sqlalchemy/scripts/calc_visitation.py @@ -27,7 +27,7 @@ def main(argv=sys.argv): bind=engine, )() vc = VisitationCalculator(session) - cs = session.query(ConceptScheme).get(scheme_id) + cs = session.get(ConceptScheme, scheme_id) visit = vc.visit(cs) for v in visit: vrow = Visitation( diff --git a/skosprovider_sqlalchemy/utils.py b/skosprovider_sqlalchemy/utils.py index 62caae0..6e7f5fe 100644 --- a/skosprovider_sqlalchemy/utils.py +++ b/skosprovider_sqlalchemy/utils.py @@ -3,7 +3,8 @@ from language_tags import tags from skosprovider.skos import Collection from skosprovider.skos import Concept -from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy import select +from sqlalchemy.exc import NoResultFound from skosprovider_sqlalchemy.models import Collection as CollectionModel from skosprovider_sqlalchemy.models import Concept as ConceptModel @@ -56,6 +57,7 @@ def import_provider(provider, conceptscheme, session): uri=c.uri, conceptscheme=conceptscheme ) + session.add(cm) _add_labels(cm, c.labels, session) _add_notes(cm, c.notes, session) _add_sources(cm, c.sources, session) @@ -65,7 +67,6 @@ def import_provider(provider, conceptscheme, session): for m in c.matches[mt]: match = MatchModel(matchtype_id=matchtype, uri=m) cm.matches.append(match) - session.add(cm) session.flush() @@ -73,17 +74,23 @@ def import_provider(provider, conceptscheme, session): for stuff in provider.get_all(): c = provider.get_by_id(stuff['id']) if isinstance(c, Concept): - cm = session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == str(c.id)) \ - .one() + cm = session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme_id == conceptscheme.id, + ConceptModel.concept_id == str(c.id) + ) + ).scalar_one() if len(c.narrower) > 0: for nc in c.narrower: try: - nc = session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == str(nc)) \ - .one() + nc = session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme_id == conceptscheme.id, + ConceptModel.concept_id == str(nc) + ) + ).scalar_one() cm.narrower_concepts.add(nc) except NoResultFound: log.warning( @@ -92,10 +99,13 @@ def import_provider(provider, conceptscheme, session): if len(c.subordinate_arrays) > 0: for sa in c.subordinate_arrays: try: - sa = session.query(CollectionModel) \ - .filter(CollectionModel.conceptscheme_id == conceptscheme.id) \ - .filter(CollectionModel.concept_id == str(sa)) \ - .one() + sa = session.execute( + select(CollectionModel) + .filter( + CollectionModel.conceptscheme_id == conceptscheme.id, + CollectionModel.concept_id == str(sa) + ) + ).scalar_one() cm.narrower_collections.add(sa) except NoResultFound: log.warning( @@ -104,26 +114,35 @@ def import_provider(provider, conceptscheme, session): if len(c.related) > 0: for rc in c.related: try: - rc = session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == str(rc)) \ - .one() + rc = session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme_id == conceptscheme.id, + ConceptModel.concept_id == str(rc) + ) + ).scalar_one() cm.related_concepts.add(rc) except NoResultFound: log.warning( 'Tried to add a relation %s related %s, but target \ does not exist. Relation will be lost.' % (c.id, rc)) elif isinstance(c, Collection) and len(c.members) > 0: - cm = session.query(CollectionModel) \ - .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == str(c.id)) \ - .one() + cm = session.execute( + select(CollectionModel) + .filter( + ConceptModel.conceptscheme_id == conceptscheme.id, + ConceptModel.concept_id == str(c.id) + ) + ).scalar_one() for mc in c.members: try: - mc = session.query(ThingModel) \ - .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == str(mc)) \ - .one() + mc = session.execute( + select(ThingModel) + .filter( + ConceptModel.conceptscheme_id == conceptscheme.id, + ConceptModel.concept_id == str(mc) + ) + ).scalar_one() cm.members.add(mc) except NoResultFound: log.warning( @@ -139,9 +158,9 @@ def _check_language(language_tag, session): :param session: Database session to use :rtype: :class:`skosprovider_sqlalchemy.models.Language` ''' - if not language_tag: # pragma: no cover + if not language_tag: # pragma: no cover language_tag = 'und' - l = session.query(LanguageModel).get(language_tag) + l = session.get(LanguageModel, language_tag) if not l: if not tags.check(language_tag): raise ValueError('Unable to import provider. Invalid language tag: %s' % language_tag) @@ -200,6 +219,7 @@ def _add_sources(target, sources, session): )) return target + class VisitationCalculator: ''' Generates a nested set for a conceptscheme. @@ -211,6 +231,9 @@ def __init__(self, session): session. ''' self.session = session + self.count = 0 + self.depth = 0 + self.visitation = [] def visit(self, conceptscheme): ''' @@ -225,15 +248,21 @@ def visit(self, conceptscheme): self.depth = 0 self.visitation = [] # get all possible top concepts - topc = self.session \ - .query(ConceptModel) \ - .filter(ConceptModel.conceptscheme == conceptscheme) \ - .filter(ConceptModel.broader_concepts == None) \ - .all() + topc = self.session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme == conceptscheme, + ~ConceptModel.broader_concepts.any() + ) + ).scalars().all() + # check if they have an indirect broader concept def _has_higher_concept(c): for coll in c.member_of: - if coll.infer_concept_relations and (coll.broader_concepts or _has_higher_concept(coll)): + if ( + coll.infer_concept_relations + and coll.broader_concepts or _has_higher_concept(coll) + ): return True return False topc = [c for c in topc if not _has_higher_concept(c)] diff --git a/tests/conftest.py b/tests/conftest.py index 308d6e0..fa0a18f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import pytest +from sqlalchemy import select from sqlalchemy.orm import sessionmaker from sqlalchemy import create_engine @@ -16,7 +17,7 @@ def pytest_addoption(parser): def engine(request): engine = create_engine( request.config.getoption('--sqlalchemy_url'), - echo=True + echo=True, ) return engine @@ -41,8 +42,8 @@ def create_data(session): Match, Language ) - en = session.query(Language).get('en') - nl = session.query(Language).get('nl') + en = session.get(Language, 'en') + nl = session.get(Language, 'nl') cs = ConceptScheme( id=1, uri='urn:x-skosprovider:test', @@ -177,7 +178,7 @@ def create_visitation(session): ConceptScheme ) vc = VisitationCalculator(session) - conceptschemes = session.query(ConceptScheme).all() + conceptschemes = session.execute(select(ConceptScheme)).scalars().all() for cs in conceptschemes: visit = vc.visit(cs) for v in visit: diff --git a/tests/test_utils.py b/tests/test_utils.py index 52568f6..c840382 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ import csv import os +from sqlalchemy import select from sqlalchemy.orm import session from skosprovider_sqlalchemy.models import Base @@ -231,9 +232,8 @@ def _get_materials(): def _get_heritage_types(): import json - typology_data = json.load( - open(os.path.join(os.path.dirname(__file__), 'data', 'typologie.js')), - )['typologie'] + with open(os.path.join(os.path.dirname(__file__), 'data', 'typologie.js')) as f: + typology_data = json.load(f)['typologie'] from skosprovider.providers import DictionaryProvider from skosprovider.uri import UriPatternGenerator from skosprovider.skos import ConceptScheme @@ -268,9 +268,8 @@ def _get_heritage_types(): def _get_event_types(): import json - event_data = json.load( - open(os.path.join(os.path.dirname(__file__), 'data', 'gebeurtenis.js')), - )['gebeurtenis'] + with open(os.path.join(os.path.dirname(__file__), 'data', 'gebeurtenis.js')) as f: + event_data = json.load(f)['gebeurtenis'] from skosprovider.providers import DictionaryProvider from skosprovider.uri import UriPatternGenerator @@ -314,7 +313,7 @@ def test_empty_provider(self): cs = self._get_cs() self.session.add(cs) import_provider(p, cs, self.session) - scheme = self.session.query(ConceptSchemeModel).get(68) + scheme = self.session.get(ConceptSchemeModel, 68) assert scheme == cs def test_menu(self): @@ -326,10 +325,13 @@ def test_menu(self): cs = self._get_cs() self.session.add(cs) import_provider(csvprovider, cs, self.session) - lobster = self.session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == '11') \ - .one() + lobster = self.session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme == cs, + ConceptModel.concept_id == '11' + ) + ).scalar_one() assert 11 == lobster.concept_id assert 'urn:x-skosprovider:menu:11' == lobster.uri assert 'Lobster Thermidor' == str(lobster.label()) @@ -345,30 +347,39 @@ def test_geo(self): cs = self._get_cs() self.session.add(cs) import_provider(geoprovider, cs, self.session) - world = self.session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == '1') \ - .one() + world = self.session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme == cs, + ConceptModel.concept_id == '1' + ) + ).scalar_one() assert world.concept_id == 1 assert 'urn:x-skosprovider:geography:1' == world.uri assert 'World' == str(world.label('en')) assert 1 == len(world.labels) assert 2 == len(world.narrower_concepts) - dutch = self.session.query(CollectionModel) \ - .filter(CollectionModel.conceptscheme == cs) \ - .filter(CollectionModel.concept_id == '333') \ - .one() + dutch = self.session.execute( + select(CollectionModel) + .filter( + CollectionModel.conceptscheme == cs, + CollectionModel.concept_id == '333' + ) + ).scalar_one() assert 333 == dutch.concept_id assert 'urn:x-skosprovider:geography:333' == dutch.uri assert 'collection' == dutch.type assert 1 == len(dutch.labels) assert 4 == len(dutch.members) - netherlands = self.session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == '10') \ - .one() + netherlands = self.session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme == cs, + ConceptModel.concept_id == '10', + ) + ).scalar_one() assert 10 == netherlands.concept_id assert 'concept' == netherlands.type assert 1 == len(netherlands.labels) @@ -384,15 +395,21 @@ def test_buildings(self): cs = self._get_cs() self.session.add(cs) import_provider(buildingprovider, cs, self.session) - castle = self.session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == '2') \ - .one() + castle = self.session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme == cs, + ConceptModel.concept_id == '2' + ) + ).scalar_one() assert 2 == len(castle.broader_concepts) - hut = self.session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == '4') \ - .one() + hut = self.session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme == cs, + ConceptModel.concept_id == '4' + ) + ).scalar_one() assert 1 == len(hut.broader_concepts) assert 1 == len(hut.matches) assert 'exactMatch' == hut.matches[0].matchtype_id @@ -407,10 +424,13 @@ def test_heritage_types(self): cs = self._get_cs() self.session.add(cs) import_provider(heritagetypesprovider, cs, self.session) - bomen = self.session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == '72') \ - .one() + bomen = self.session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme == cs, + ConceptModel.concept_id == '72' + ) + ).scalar_one() assert 2 == len(bomen.narrower_collections) assert 2 == len(cs.labels) assert 'Erfgoedtypes' == cs.label('nl').label @@ -426,10 +446,13 @@ def test_event_types(self): cs = self._get_cs() self.session.add(cs) import_provider(eventtypesprovider, cs, self.session) - archeologische_opgravingen = self.session.query(ConceptModel) \ - .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == '38') \ - .one() + archeologische_opgravingen = self.session.execute( + select(ConceptModel) + .filter( + ConceptModel.conceptscheme == cs, + ConceptModel.concept_id == '38' + ) + ).scalar_one() assert 3 == len(archeologische_opgravingen.narrower_collections) def test_materials(self): @@ -441,9 +464,10 @@ def test_materials(self): cs = self._get_cs() self.session.add(cs) import_provider(materialsprovider, cs, self.session) - materials = self.session.query(ThingModel) \ - .filter(ThingModel.conceptscheme == cs) \ - .all() + materials = self.session.execute( + select(ThingModel) + .filter(ThingModel.conceptscheme == cs) + ).scalars().all() assert 2 == len(materials) @@ -538,7 +562,7 @@ def test_geo(self): visit = vc.visit(cs) assert 10 == len(visit) world = visit[0] - assert self.session.query(ConceptModel).get(world['id']).concept_id == 1 + assert self.session.get(ConceptModel, world['id']).concept_id == 1 assert 1 == world['lft'] assert 20 == world['rght'] assert 1 == world['depth'] @@ -563,7 +587,7 @@ def test_buildings(self): visit = vc.visit(cs) assert len(visit) == 5 # Check that castle is present twice - ids = [self.session.query(ConceptModel).get(v['id']).concept_id for v in visit] + ids = [self.session.get(ConceptModel, v['id']).concept_id for v in visit] assert ids.count(2) == 2 for v in visit: # Check that fortification has one child