From cbded38055ea0c08ed2839f7df770bf80407c6a8 Mon Sep 17 00:00:00 2001 From: Wim De Clercq Date: Mon, 2 Jan 2023 12:10:18 +0100 Subject: [PATCH] Change concept.concept_id from int to str. Issue #87 --- .travis.yml | 1 + README.rst | 11 +++ setup.py | 2 +- skosprovider_sqlalchemy/models.py | 2 +- skosprovider_sqlalchemy/providers.py | 8 +-- skosprovider_sqlalchemy/utils.py | 12 ++-- tests/conftest.py | 2 +- tests/test_providers.py | 104 +++++++++++++-------------- tests/test_utils.py | 16 ++--- 9 files changed, 85 insertions(+), 73 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a5d829..ad1e5c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: - DB="sqlite" SAURL="sqlite:///:memory:" - DB="postgres" SAURL="postgresql://postgres:postgres@localhost/skosprovider_sqlalchemy" install: + - pip install --upgrade pip setuptools - pip install -r requirements-dev.txt - python setup.py -q develop before_script: diff --git a/README.rst b/README.rst index 61dd986..9e3a6f5 100644 --- a/README.rst +++ b/README.rst @@ -14,6 +14,17 @@ A SQLAlchemy implementation of the skosprovider_ interface. :target: https://coveralls.io/r/koenedaele/skosprovider_sqlalchemy +Migrating to skosprovider_sqlalchemy 2.0.0 +------------------------------------------ +A change in the models has been made which requires a database upgrade. +The "concept" table's "concept_id" column has changed from being an int to a string. + +Existing databases will therefor require a small change to update table scheme. +Typically this will look like:: + + ALTER TABLE concept ALTER COLUMN concept_id TEXT NOT NULL; + + Building the docs ----------------- diff --git a/setup.py b/setup.py index 43970f7..e8b95ed 100755 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name='skosprovider_sqlalchemy', - version='1.0.0', + version='2.0.0', description='A sqlAlchemy implementation of skosprovider.', long_description=open('README.rst').read(), long_description_content_type='text/x-rst', diff --git a/skosprovider_sqlalchemy/models.py b/skosprovider_sqlalchemy/models.py index e11bcf3..3d6d14a 100644 --- a/skosprovider_sqlalchemy/models.py +++ b/skosprovider_sqlalchemy/models.py @@ -172,7 +172,7 @@ class Thing(Base): id = Column(Integer, primary_key=True) type = Column(String(30)) concept_id = Column( - Integer, + String, nullable=False, index=True ) diff --git a/skosprovider_sqlalchemy/providers.py b/skosprovider_sqlalchemy/providers.py index 3eab3bd..3b5c381 100644 --- a/skosprovider_sqlalchemy/providers.py +++ b/skosprovider_sqlalchemy/providers.py @@ -180,7 +180,7 @@ def _from_thing(self, thing): matches=matches ) - def get_by_id(self, id): + def get_by_id(self, concept_id): try: thing = self.session\ .query(Thing)\ @@ -188,7 +188,7 @@ def get_by_id(self, id): .options(joinedload('notes'))\ .options(joinedload('sources'))\ .filter( - Thing.concept_id == int(id), + Thing.concept_id == str(concept_id), Thing.conceptscheme_id == self.conceptscheme_id ).one() except NoResultFound: @@ -328,7 +328,7 @@ def expand(self, id): thing = self.session\ .query(Thing)\ .filter( - Thing.concept_id == id, + Thing.concept_id == str(id), Thing.conceptscheme_id == self.conceptscheme_id ).one() except NoResultFound: @@ -428,7 +428,7 @@ def get_children_display(self, id, **kwargs): thing = self.session\ .query(Thing)\ .filter( - Thing.concept_id == int(id), + Thing.concept_id == str(id), Thing.conceptscheme_id == self.conceptscheme_id ).one() except NoResultFound: diff --git a/skosprovider_sqlalchemy/utils.py b/skosprovider_sqlalchemy/utils.py index ff224e3..62caae0 100644 --- a/skosprovider_sqlalchemy/utils.py +++ b/skosprovider_sqlalchemy/utils.py @@ -75,14 +75,14 @@ def import_provider(provider, conceptscheme, session): if isinstance(c, Concept): cm = session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == int(c.id)) \ + .filter(ConceptModel.concept_id == str(c.id)) \ .one() if len(c.narrower) > 0: for nc in c.narrower: try: nc = session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == int(nc)) \ + .filter(ConceptModel.concept_id == str(nc)) \ .one() cm.narrower_concepts.add(nc) except NoResultFound: @@ -94,7 +94,7 @@ def import_provider(provider, conceptscheme, session): try: sa = session.query(CollectionModel) \ .filter(CollectionModel.conceptscheme_id == conceptscheme.id) \ - .filter(CollectionModel.concept_id == int(sa)) \ + .filter(CollectionModel.concept_id == str(sa)) \ .one() cm.narrower_collections.add(sa) except NoResultFound: @@ -106,7 +106,7 @@ def import_provider(provider, conceptscheme, session): try: rc = session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == int(rc)) \ + .filter(ConceptModel.concept_id == str(rc)) \ .one() cm.related_concepts.add(rc) except NoResultFound: @@ -116,13 +116,13 @@ def import_provider(provider, conceptscheme, session): elif isinstance(c, Collection) and len(c.members) > 0: cm = session.query(CollectionModel) \ .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == int(c.id)) \ + .filter(ConceptModel.concept_id == str(c.id)) \ .one() for mc in c.members: try: mc = session.query(ThingModel) \ .filter(ConceptModel.conceptscheme_id == conceptscheme.id) \ - .filter(ConceptModel.concept_id == int(mc)) \ + .filter(ConceptModel.concept_id == str(mc)) \ .one() cm.members.add(mc) except NoResultFound: diff --git a/tests/conftest.py b/tests/conftest.py index 308d6e0..c206162 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ def pytest_addoption(parser): parser.addoption( '--sqlalchemy_url', action='store', - default='sqlite:///:memory:', + default='postgresql://postgres:postgres@localhost/skosprovider_sqlalchemy', help='SQLAlchemy connection url to database under test.' ) diff --git a/tests/test_providers.py b/tests/test_providers.py index 58e49f6..d838e67 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -119,9 +119,9 @@ def test_get_concept_by_id(self): con = self.provider.get_by_id(1) assert isinstance(con, Concept) - assert 1 == con.id - assert [3] == con.related - assert [2, 8] == sorted(con.subordinate_arrays) + assert '1' == con.id + assert ['3'] == con.related + assert ['2', '8'] == sorted(con.subordinate_arrays) def test_concept_has_concept_scheme(self): from skosprovider.skos import ( @@ -136,9 +136,9 @@ def test_get_concept_by_id_string(self): con = self.provider.get_by_id('1') assert isinstance(con, Concept) - assert 1 == con.id - assert [3] == con.related - assert [2, 8] == sorted(con.subordinate_arrays) + assert '1' == con.id + assert ['3'] == con.related + assert ['2', '8'] == sorted(con.subordinate_arrays) def test_get_unexisting_by_id(self): con = self.provider.get_by_id(404) @@ -172,9 +172,9 @@ def test_get_collection_by_id(self): col = self.provider.get_by_id(2) assert isinstance(col, Collection) - assert 2 == col.id - assert [4, 6] == sorted(col.members) - assert [1] == col.superordinates + assert '2' == col.id + assert ['4', '6'] == sorted(col.members) + assert ['1'] == col.superordinates def test_collection_has_no_matches(self): col = self.provider.get_by_id(2) @@ -193,49 +193,49 @@ def test_get_all(self): all = self.provider.get_all() assert len(all) == 9 assert { - 'id': 1, + 'id': '1', 'uri': 'urn:x-skosprovider:test:1', 'type': 'concept', 'label': 'Churches' } in all assert { - 'id': 2, + 'id': '2', 'uri': 'urn:x-skosprovider:test:2', 'type': 'collection', 'label': 'Churches by function' } in all assert { - 'id': 3, + 'id': '3', 'uri': 'urn:x-skosprovider:test:3', 'type': 'concept', 'label': 'Chapels' } in all assert { - 'id': 4, + 'id': '4', 'uri': 'urn:x-skosprovider:test:4', 'type': 'concept', 'label': 'Cathedrals' } in all assert { - 'id': 5, + 'id': '5', 'uri': 'urn:x-skosprovider:test:5', 'type': 'concept', 'label': 'Boomkapellen' } in all assert { - 'id': 6, + 'id': '6', 'uri': 'urn:x-skosprovider:test:6', 'type': 'concept', 'label': 'Parochiekerken' } in all assert { - 'id': 7, + 'id': '7', 'uri': 'urn:x-skosprovider:test:7', 'type': 'concept', 'label': 'Hulpkerken' @@ -244,7 +244,7 @@ def test_get_all(self): def test_get_all_sorted_id_desc(self): all = self.provider.get_all(sort='id', sort_order='desc') assert len(all) == 9 - assert [9, 8, 7, 6, 5, 4, 3, 2, 1] == [c['id'] for c in all] + assert ['9', '8', '7', '6', '5', '4', '3', '2', '1'] == [c['id'] for c in all] def test_get_all_sorted_label(self): all = self.provider.get_all(sort='label') @@ -279,21 +279,21 @@ def test_get_top_concepts(self): assert len(all) == 3 assert { - 'id': 1, + 'id': '1', 'uri': 'urn:x-skosprovider:test:1', 'type': 'concept', 'label': 'Churches' } in all assert { - 'id': 3, + 'id': '3', 'uri': 'urn:x-skosprovider:test:3', 'type': 'concept', 'label': 'Chapels' } in all assert { - 'id': 9, + 'id': '9', 'uri': 'urn:x-skosprovider:test:9', 'type': 'concept', 'label': 'Churchtowers' @@ -313,14 +313,14 @@ def test_get_top_display(self): all = self.provider.get_top_display() assert len(all) == 2 assert { - 'id': 3, + 'id': '3', 'uri': 'urn:x-skosprovider:test:3', 'type': 'concept', 'label': 'Chapels' } in all assert { - 'id': 1, + 'id': '1', 'uri': 'urn:x-skosprovider:test:1', 'type': 'concept', 'label': 'Churches' @@ -343,7 +343,7 @@ def test_get_children_display_collection(self): children = self.provider.get_children_display(2) assert len(children) == 2 assert { - 'id': 4, + 'id': '4', 'uri': 'urn:x-skosprovider:test:4', 'type': 'concept', 'label': 'Cathedrals' @@ -353,7 +353,7 @@ def test_get_children_display_collection_sort_id(self): children = self.provider.get_children_display(2, sort='id') assert len(children) == 2 assert { - 'id': 4, + 'id': '4', 'uri': 'urn:x-skosprovider:test:4', 'type': 'concept', 'label': 'Cathedrals' @@ -363,7 +363,7 @@ def test_get_children_display_concept_with_narrower_collection(self): children = self.provider.get_children_display(1) assert len(children) == 2 assert { - 'id': 2, + 'id': '2', 'uri': 'urn:x-skosprovider:test:2', 'type': 'collection', 'label': 'Churches by function' @@ -373,7 +373,7 @@ def test_get_children_display_concept_with_narrower_concept(self): children = self.provider.get_children_display(3) assert len(children) == 1 assert { - 'id': 5, + 'id': '5', 'uri': 'urn:x-skosprovider:test:5', 'type': 'concept', 'label': 'Boomkapellen' @@ -395,7 +395,7 @@ def test_find_type_concept(self): all = self.provider.find({'type': 'concept'}) assert len(all) == 7 assert { - 'id': 2, + 'id': '2', 'uri': 'urn:x-skosprovider:test:2', 'type': 'collection', 'label': 'Churches by function' @@ -418,13 +418,13 @@ def test_find_type_collection(self): all = self.provider.find({'type': 'collection'}) assert len(all) == 2 assert { - 'id': 2, + 'id': '2', 'uri': 'urn:x-skosprovider:test:2', 'type': 'collection', 'label': 'Churches by function' } in all assert { - 'id': 8, + 'id': '8', 'uri': 'urn:x-skosprovider:test:8', 'type': 'collection', 'label': 'Parts of churches' @@ -434,19 +434,19 @@ def test_find_label_kerken(self): all = self.provider.find({'label': 'kerken'}) assert len(all) == 3 assert { - 'id': 1, + 'id': '1', 'uri': 'urn:x-skosprovider:test:1', 'type': 'concept', 'label': 'Churches' } in all assert { - 'id': 6, + 'id': '6', 'uri': 'urn:x-skosprovider:test:6', 'type': 'concept', 'label': 'Parochiekerken' } in all assert { - 'id': 7, + 'id': '7', 'uri': 'urn:x-skosprovider:test:7', 'type': 'concept', 'label': 'Hulpkerken' @@ -456,7 +456,7 @@ def test_find_label_churches_type_concept(self): all = self.provider.find({'label': 'churches', 'type': 'concept'}) assert len(all) == 1 assert { - 'id': 1, + 'id': '1', 'uri': 'urn:x-skosprovider:test:1', 'type': 'concept', 'label': 'Churches' @@ -470,7 +470,7 @@ def test_find_collection_2_depth_default_members(self): nodepth = self.provider.find({'collection': {'id': 2}}) depth = self.provider.find({ 'collection': { - 'id': 2, + 'id': '2', 'depth': 'members' } }) @@ -479,25 +479,25 @@ def test_find_collection_2_depth_default_members(self): def test_find_collection_2_depth_all(self): all = self.provider.find({ 'collection': { - 'id': 2, + 'id': '2', 'depth': 'all' } }) assert len(all) == 3 assert { - 'id': 4, + 'id': '4', 'uri': 'urn:x-skosprovider:test:4', 'type': 'concept', 'label': 'Cathedrals' } in all assert { - 'id': 6, + 'id': '6', 'uri': 'urn:x-skosprovider:test:6', 'type': 'concept', 'label': 'Parochiekerken' } in all assert { - 'id': 7, + 'id': '7', 'uri': 'urn:x-skosprovider:test:7', 'type': 'concept', 'label': 'Hulpkerken' @@ -506,19 +506,19 @@ def test_find_collection_2_depth_all(self): def test_find_collection_2_depth_members(self): all = self.provider.find({ 'collection': { - 'id': 2, + 'id': '2', 'depth': 'members' } }) assert len(all) == 2 assert { - 'id': 4, + 'id': '4', 'uri': 'urn:x-skosprovider:test:4', 'type': 'concept', 'label': 'Cathedrals' } in all assert { - 'id': 6, + 'id': '6', 'uri': 'urn:x-skosprovider:test:6', 'type': 'concept', 'label': 'Parochiekerken' @@ -540,7 +540,7 @@ def test_find_matches_one(self): }}) assert len(all) == 1 assert { - 'id': 4, + 'id': '4', 'uri': 'urn:x-skosprovider:test:4', 'type': 'concept', 'label': 'Cathedrals' @@ -553,7 +553,7 @@ def test_find_matches_one_close(self): }}) assert len(all) == 1 assert { - 'id': 4, + 'id': '4', 'uri': 'urn:x-skosprovider:test:4', 'type': 'concept', 'label': 'Cathedrals' @@ -566,7 +566,7 @@ def test_find_matches_one_close_inherits_exact(self): }}) assert len(all) == 1 assert { - 'id': 9, + 'id': '9', 'uri': 'urn:x-skosprovider:test:9', 'type': 'concept', 'label': 'Churchtowers' @@ -574,19 +574,19 @@ def test_find_matches_one_close_inherits_exact(self): def test_expand_concept(self): ids = self.provider.expand(1) - assert [1, 4, 6, 7] == ids + assert {'1', '4', '6', '7'} == set(ids) def test_expand_collection(self): ids = self.provider.expand(2) - assert [4, 6, 7] == ids + assert {'4', '6', '7'} == set(ids) def test_expand_collection_without_inference(self): ids = self.provider.expand(8) - assert [9] == ids + assert ['9'] == ids def test_expand_concept_without_narrower(self): ids = self.provider.expand(5) - assert [5] == ids + assert ['5'] == ids def test_expand_unexisting(self): ids = self.provider.expand(404) @@ -614,19 +614,19 @@ def tearDown(self): def test_expand_concept_visit(self): ids = self.visitationprovider.expand(1) - assert ids == [1, 4, 6, 7] + assert set(ids) == {'1', '4', '6', '7'} def test_expand_collection_visit(self): ids = self.visitationprovider.expand(2) - assert ids == [4, 6, 7] + assert set(ids) == {'4', '6', '7'} def test_expand_collection_without_inference_visit(self): ids = self.visitationprovider.expand(8) - assert [9] == ids + assert ['9'] == ids def test_expand_concept_without_narrower_visit(self): ids = self.visitationprovider.expand(4) - assert ids == [4] + assert ids == ['4'] def test_expand_unexisting_visit(self): ids = self.visitationprovider.expand(404) diff --git a/tests/test_utils.py b/tests/test_utils.py index e5eca85..52568f6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -328,7 +328,7 @@ def test_menu(self): import_provider(csvprovider, cs, self.session) lobster = self.session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == 11) \ + .filter(ConceptModel.concept_id == '11') \ .one() assert 11 == lobster.concept_id assert 'urn:x-skosprovider:menu:11' == lobster.uri @@ -347,7 +347,7 @@ def test_geo(self): import_provider(geoprovider, cs, self.session) world = self.session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == 1) \ + .filter(ConceptModel.concept_id == '1') \ .one() assert world.concept_id == 1 assert 'urn:x-skosprovider:geography:1' == world.uri @@ -357,7 +357,7 @@ def test_geo(self): dutch = self.session.query(CollectionModel) \ .filter(CollectionModel.conceptscheme == cs) \ - .filter(CollectionModel.concept_id == 333) \ + .filter(CollectionModel.concept_id == '333') \ .one() assert 333 == dutch.concept_id assert 'urn:x-skosprovider:geography:333' == dutch.uri @@ -367,7 +367,7 @@ def test_geo(self): netherlands = self.session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == 10) \ + .filter(ConceptModel.concept_id == '10') \ .one() assert 10 == netherlands.concept_id assert 'concept' == netherlands.type @@ -386,12 +386,12 @@ def test_buildings(self): import_provider(buildingprovider, cs, self.session) castle = self.session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == 2) \ + .filter(ConceptModel.concept_id == '2') \ .one() assert 2 == len(castle.broader_concepts) hut = self.session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == 4) \ + .filter(ConceptModel.concept_id == '4') \ .one() assert 1 == len(hut.broader_concepts) assert 1 == len(hut.matches) @@ -409,7 +409,7 @@ def test_heritage_types(self): import_provider(heritagetypesprovider, cs, self.session) bomen = self.session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == 72) \ + .filter(ConceptModel.concept_id == '72') \ .one() assert 2 == len(bomen.narrower_collections) assert 2 == len(cs.labels) @@ -428,7 +428,7 @@ def test_event_types(self): import_provider(eventtypesprovider, cs, self.session) archeologische_opgravingen = self.session.query(ConceptModel) \ .filter(ConceptModel.conceptscheme == cs) \ - .filter(ConceptModel.concept_id == 38) \ + .filter(ConceptModel.concept_id == '38') \ .one() assert 3 == len(archeologische_opgravingen.narrower_collections)