diff --git a/docs/modules/ROOT/pages/neo4jstore.adoc b/docs/modules/ROOT/pages/neo4jstore.adoc index 050d494..d877e0c 100644 --- a/docs/modules/ROOT/pages/neo4jstore.adoc +++ b/docs/modules/ROOT/pages/neo4jstore.adoc @@ -6,7 +6,8 @@ This class is an implementation of the rdflib link:https://rdflib.readthedocs.io == Constructor |=== | Name | Type | Required | Default | Description -|config|Neo4jStoreConfig|True||Neo4jStoreConfig object that contains all the useful informations to initialize the store. +|config|Neo4jStoreConfig|True||Neo4jStoreConfig object that contains all the useful information to initialize the store. +|driver|Neo4jStoreConfig|False|None|A pre-built Neo4j driver object to use to connect to the database. You cannot specify both a driver and credentials in the Neo4jStoreConfig. |=== == Functions diff --git a/docs/modules/ROOT/pages/neo4jstoreconfig.adoc b/docs/modules/ROOT/pages/neo4jstoreconfig.adoc index 2ef01f5..6659169 100644 --- a/docs/modules/ROOT/pages/neo4jstoreconfig.adoc +++ b/docs/modules/ROOT/pages/neo4jstoreconfig.adoc @@ -6,7 +6,7 @@ This object is used to configure the Neo4j Store to connect to your Neo4j Instan == Constructor |=== | Name | Type | Required | Values(Default) | Description -| auth_data | Dictionary | True | ("uri", "database", "user", "pwd") | A dictionary containing authentication data. The required keys are: ["uri", "database", "user", "pwd"]. +| auth_data | Dictionary | Yes, unless a driver object is passed in the store init | ("uri", "database", "user", "pwd") | A dictionary containing authentication data. The required keys are: ["uri", "database", "user", "pwd"]. | batching | Boolean | False | boolean (True) | A boolean indicating whether batching is enabled. | batch_size | Integer | False | (5000) | An integer representing the batch size (The batch size is intended as number of entities to store inside the database (nodes/relationships) and not triples. | custom_mappings | List[Tuple[Str,Str,Str]] | False | Empty list | A list of tuples containing custom mappings for prefixes in the form (prefix, object_to_replace, new_object). diff --git a/rdflib_neo4j/Neo4jStore.py b/rdflib_neo4j/Neo4jStore.py index 6cca1c9..596cead 100644 --- a/rdflib_neo4j/Neo4jStore.py +++ b/rdflib_neo4j/Neo4jStore.py @@ -1,13 +1,14 @@ from typing import Dict from rdflib.store import Store -from neo4j import GraphDatabase +from neo4j import GraphDatabase, Driver from neo4j import WRITE_ACCESS import logging from rdflib_neo4j.Neo4jTriple import Neo4jTriple from rdflib_neo4j.config.Neo4jStoreConfig import Neo4jStoreConfig from rdflib_neo4j.config.const import NEO4J_DRIVER_USER_AGENT_NAME +from rdflib_neo4j.config.utils import check_auth_data from rdflib_neo4j.query_composers.NodeQueryComposer import NodeQueryComposer from rdflib_neo4j.query_composers.RelationshipQueryComposer import RelationshipQueryComposer from rdflib_neo4j.utils import handle_neo4j_driver_exception @@ -17,11 +18,16 @@ class Neo4jStore(Store): context_aware = True - def __init__(self, config: Neo4jStoreConfig): + def __init__(self, config: Neo4jStoreConfig, neo4j_driver: Driver = None): self.__open = False - self.driver = None + self.driver = neo4j_driver self.session = None self.config = config + if not neo4j_driver: + check_auth_data(config.auth_data) + elif config.auth_data: + raise Exception("Either initialize the store with credentials or driver. You cannot do both.") + super(Neo4jStore, self).__init__(config.get_config_dict()) self.batching = config.batching @@ -62,7 +68,6 @@ def close(self, commit_pending_transaction=True): self.commit(commit_nodes=True) self.commit(commit_rels=True) self.session.close() - self.driver.close() self.__set_open(False) print(f"IMPORTED {self.total_triples} TRIPLES") self.total_triples=0 @@ -147,6 +152,16 @@ def __set_open(self, val: bool): self.__open = val print(f"The store is now: {'Open' if self.__open else 'Closed'}") + def __get_driver(self) -> Driver: + if not self.driver: + auth_data = self.config.auth_data + self.driver = GraphDatabase.driver( + auth_data['uri'], + auth=(auth_data['user'], auth_data['pwd']), + database=auth_data.get('database', 'neo4j'), + user_agent=NEO4J_DRIVER_USER_AGENT_NAME + ) + return self.driver def __create_session(self): """ @@ -156,13 +171,7 @@ def __create_session(self): """ auth_data = self.config.auth_data - self.driver = GraphDatabase.driver( - auth_data['uri'], - auth=(auth_data['user'], auth_data['pwd']), - user_agent=NEO4J_DRIVER_USER_AGENT_NAME - ) - self.session = self.driver.session( - database=auth_data.get('database', 'neo4j'), + self.session = self.__get_driver().session( default_access_mode=WRITE_ACCESS ) diff --git a/rdflib_neo4j/config/Neo4jStoreConfig.py b/rdflib_neo4j/config/Neo4jStoreConfig.py index 2b4686b..b15f928 100644 --- a/rdflib_neo4j/config/Neo4jStoreConfig.py +++ b/rdflib_neo4j/config/Neo4jStoreConfig.py @@ -191,10 +191,7 @@ def set_auth_data(self, auth): Parameters: - auth: A dictionary containing authentication data. - Raises: - - WrongAuthenticationException: If any of the required authentication fields is missing. """ - check_auth_data(auth=auth) self.auth_data = auth def set_batching(self, val: bool): @@ -225,5 +222,4 @@ def get_config_dict(self): Raises: - WrongAuthenticationException: If any of the required authentication fields is missing. """ - check_auth_data(auth=self.auth_data) return vars(self) diff --git a/test/integration/containers_test.py b/test/integration/containers_test.py index 14261d1..cf06d9a 100644 --- a/test/integration/containers_test.py +++ b/test/integration/containers_test.py @@ -4,7 +4,7 @@ from test.integration.utils import records_equal, read_file_n10s_and_rdflib import pytest from test.integration.fixtures import neo4j_container, neo4j_driver, graph_store, graph_store_batched, \ - cleanup_databases + cleanup_databases, neo4j_connection_parameters def test_import_person(neo4j_driver, graph_store): diff --git a/test/integration/custom_mappings_test.py b/test/integration/custom_mappings_test.py index 045b22b..505680a 100644 --- a/test/integration/custom_mappings_test.py +++ b/test/integration/custom_mappings_test.py @@ -3,21 +3,21 @@ from rdflib_neo4j.Neo4jStore import Neo4jStore from rdflib_neo4j.config.Neo4jStoreConfig import Neo4jStoreConfig from test.integration.constants import LOCAL -from test.integration.utils import records_equal, read_file_n10s_and_rdflib, get_credentials +from test.integration.utils import records_equal, read_file_n10s_and_rdflib from rdflib_neo4j.config.const import HANDLE_VOCAB_URI_STRATEGY import os from dotenv import load_dotenv from test.integration.fixtures import neo4j_container, neo4j_driver, graph_store, graph_store_batched, \ - cleanup_databases + cleanup_databases, neo4j_connection_parameters -def test_custom_mapping_match(neo4j_container, neo4j_driver): +def test_custom_mapping_match(neo4j_driver, neo4j_connection_parameters): """ If we define a custom mapping and the strategy is HANDLE_VOCAB_URI_STRATEGY.MAP, it should match it and use the mapping if the predicate satisfies the mapping. """ - auth_data = get_credentials(LOCAL, neo4j_container) + auth_data = neo4j_connection_parameters # Define your prefixes prefixes = { 'neo4voc': Namespace('http://neo4j.org/vocab/sw#') @@ -56,7 +56,7 @@ def test_custom_mapping_match(neo4j_container, neo4j_driver): assert records_equal(rels[i], rels_from_rdflib[i], rels=True) -def test_custom_mapping_no_match(neo4j_container, neo4j_driver): +def test_custom_mapping_no_match(neo4j_driver, neo4j_connection_parameters): """ If we define a custom mapping and the strategy is HANDLE_VOCAB_URI_STRATEGY.MAP, it shouldn't apply the mapping if the predicate doesn't satisfy the mapping and use IGNORE as a strategy. @@ -66,7 +66,7 @@ def test_custom_mapping_no_match(neo4j_container, neo4j_driver): if the predicate satisfies the mapping. """ - auth_data = get_credentials(LOCAL, neo4j_container) + auth_data = neo4j_connection_parameters # Define your prefixes prefixes = { @@ -106,12 +106,12 @@ def test_custom_mapping_no_match(neo4j_container, neo4j_driver): assert records_equal(rels[i], rels_from_rdflib[i], rels=True) -def test_custom_mapping_map_strategy_zero_custom_mappings(neo4j_container, neo4j_driver): +def test_custom_mapping_map_strategy_zero_custom_mappings(neo4j_driver, neo4j_connection_parameters): """ If we don't define custom mapping and the strategy is HANDLE_VOCAB_URI_STRATEGY.MAP, it shouldn't apply the mapping on anything and just use IGNORE mode. """ - auth_data = get_credentials(LOCAL, neo4j_container) + auth_data = neo4j_connection_parameters # Define your prefixes prefixes = { diff --git a/test/integration/fixtures.py b/test/integration/fixtures.py index 0f18ecd..fbad787 100644 --- a/test/integration/fixtures.py +++ b/test/integration/fixtures.py @@ -1,9 +1,10 @@ import pytest from neo4j import GraphDatabase +from rdflib import Graph from testcontainers.neo4j import Neo4jContainer +from rdflib_neo4j import HANDLE_VOCAB_URI_STRATEGY, Neo4jStoreConfig, Neo4jStore from test.integration.constants import LOCAL, N10S_CONSTRAINT_QUERY, RDFLIB_DB -from test.integration.utils import create_graph_store import os @@ -48,13 +49,43 @@ def neo4j_driver(neo4j_container): @pytest.fixture -def graph_store(neo4j_container, neo4j_driver): - return create_graph_store(neo4j_container) +def graph_store(neo4j_connection_parameters): + return config_graph_store(neo4j_connection_parameters) @pytest.fixture -def graph_store_batched(neo4j_container, neo4j_driver): - return create_graph_store(neo4j_container, batching=True) +def graph_store_batched(neo4j_connection_parameters): + return config_graph_store(neo4j_connection_parameters, True) + + +def config_graph_store(auth_data, batching=False): + + config = Neo4jStoreConfig(auth_data=auth_data, + custom_prefixes={}, + custom_mappings=[], + multival_props_names=[], + handle_vocab_uri_strategy=HANDLE_VOCAB_URI_STRATEGY.IGNORE, + batching=batching) + + g = Graph(store=Neo4jStore(config=config)) + return g + + +@pytest.fixture +def neo4j_connection_parameters(neo4j_container): + if LOCAL: + auth_data = { + 'uri': os.getenv("NEO4J_URI_LOCAL"), + 'database': RDFLIB_DB, + 'user': os.getenv("NEO4J_USER_LOCAL"), + 'pwd': os.getenv("NEO4J_PWD_LOCAL") + } + else: + auth_data = {'uri': neo4j_container.get_connection_url(), + 'database': RDFLIB_DB, + 'user': "neo4j", + 'pwd': Neo4jContainer.NEO4J_ADMIN_PASSWORD} + return auth_data @pytest.fixture(autouse=True) diff --git a/test/integration/handle_vocab_uri_test.py b/test/integration/handle_vocab_uri_test.py index cb7a2c8..581fa94 100644 --- a/test/integration/handle_vocab_uri_test.py +++ b/test/integration/handle_vocab_uri_test.py @@ -6,18 +6,18 @@ from rdflib_neo4j.config.Neo4jStoreConfig import Neo4jStoreConfig from rdflib_neo4j.config.const import ShortenStrictException, HANDLE_VOCAB_URI_STRATEGY from test.integration.constants import LOCAL -from test.integration.utils import records_equal, read_file_n10s_and_rdflib, get_credentials +from test.integration.utils import records_equal, read_file_n10s_and_rdflib import pytest -from test.integration.fixtures import neo4j_container, neo4j_driver, graph_store, graph_store_batched, \ +from test.integration.fixtures import neo4j_container, neo4j_connection_parameters, neo4j_driver, graph_store, graph_store_batched, \ cleanup_databases -def test_shorten_all_prefixes_defined(neo4j_container, neo4j_driver): +def test_shorten_all_prefixes_defined(neo4j_driver, neo4j_connection_parameters): """ If we use the strategy HANDLE_VOCAB_URI_STRATEGY.SHORTEN and we provide all the required namespaces, it should load all the data without raising an error for a missing prefix """ - auth_data = get_credentials(LOCAL, neo4j_container) + auth_data = neo4j_connection_parameters # Define your prefixes prefixes = { @@ -61,8 +61,8 @@ def test_shorten_all_prefixes_defined(neo4j_container, neo4j_driver): assert records_equal(rels[i], rels_from_rdflib[i], rels=True) -def test_shorten_missing_prefix(neo4j_container, neo4j_driver): - auth_data = get_credentials(LOCAL, neo4j_container) +def test_shorten_missing_prefix(neo4j_driver, neo4j_connection_parameters): + auth_data = neo4j_connection_parameters # Define your prefixes prefixes = { @@ -90,8 +90,8 @@ def test_shorten_missing_prefix(neo4j_container, neo4j_driver): assert True -def test_keep_strategy(neo4j_container, neo4j_driver): - auth_data = get_credentials(LOCAL, neo4j_container) +def test_keep_strategy(neo4j_driver, neo4j_connection_parameters): + auth_data = neo4j_connection_parameters config = Neo4jStoreConfig(auth_data=auth_data, handle_vocab_uri_strategy=HANDLE_VOCAB_URI_STRATEGY.KEEP, @@ -111,8 +111,8 @@ def test_keep_strategy(neo4j_container, neo4j_driver): assert records_equal(rels[i], rels_from_rdflib[i], rels=True) -def test_ignore_strategy(neo4j_container, neo4j_driver): - auth_data = get_credentials(LOCAL, neo4j_container) +def test_ignore_strategy(neo4j_driver, neo4j_connection_parameters): + auth_data = neo4j_connection_parameters config = Neo4jStoreConfig(auth_data=auth_data, handle_vocab_uri_strategy=HANDLE_VOCAB_URI_STRATEGY.IGNORE, @@ -132,8 +132,8 @@ def test_ignore_strategy(neo4j_container, neo4j_driver): assert records_equal(rels[i], rels_from_rdflib[i], rels=True) -def test_ignore_strategy_on_json_ld_file(neo4j_container, neo4j_driver): - auth_data = get_credentials(LOCAL, neo4j_container) +def test_ignore_strategy_on_json_ld_file(neo4j_driver, neo4j_connection_parameters): + auth_data = neo4j_connection_parameters # Define your prefixes prefixes = { diff --git a/test/integration/multival_test.py b/test/integration/multival_test.py index db5d49e..05ca98a 100644 --- a/test/integration/multival_test.py +++ b/test/integration/multival_test.py @@ -2,17 +2,17 @@ from rdflib import Graph, Namespace from rdflib_neo4j.Neo4jStore import Neo4jStore from rdflib_neo4j.config.Neo4jStoreConfig import Neo4jStoreConfig -from test.integration.utils import records_equal, read_file_n10s_and_rdflib, create_graph_store, get_credentials +from test.integration.utils import records_equal, read_file_n10s_and_rdflib from rdflib_neo4j.config.const import HANDLE_VOCAB_URI_STRATEGY, HANDLE_MULTIVAL_STRATEGY import pytest from test.integration.fixtures import neo4j_container, neo4j_driver, graph_store, graph_store_batched, \ - cleanup_databases + cleanup_databases, neo4j_connection_parameters -def test_read_file_multival_with_strategy_no_predicates(neo4j_container, neo4j_driver): +def test_read_file_multival_with_strategy_no_predicates(neo4j_driver, neo4j_connection_parameters): """Compare data imported with n10s procs and n10s + rdflib in single add mode for multivalues""" - auth_data = get_credentials(LOCAL, neo4j_container) + auth_data = neo4j_connection_parameters # Define your prefixes prefixes = {} @@ -40,9 +40,9 @@ def test_read_file_multival_with_strategy_no_predicates(neo4j_container, neo4j_d assert records_equal(records[i], records_from_rdf_lib[i]) -def test_read_file_multival_with_strategy_and_predicates(neo4j_container, neo4j_driver): +def test_read_file_multival_with_strategy_and_predicates(neo4j_driver, neo4j_connection_parameters): """Compare data imported with n10s procs and n10s + rdflib in single add mode for multivalues""" - auth_data = get_credentials(LOCAL, neo4j_container) + auth_data = neo4j_connection_parameters # Define your prefixes prefixes = { @@ -72,9 +72,9 @@ def test_read_file_multival_with_strategy_and_predicates(neo4j_container, neo4j_ assert records_equal(records[i], records_from_rdf_lib[i]) -def test_read_file_multival_with_no_strategy_and_predicates(neo4j_container, neo4j_driver): +def test_read_file_multival_with_no_strategy_and_predicates(neo4j_driver, neo4j_connection_parameters): """Compare data imported with n10s procs and n10s + rdflib in single add mode for multivalues""" - auth_data = get_credentials(LOCAL, neo4j_container) + auth_data = neo4j_connection_parameters # Define your prefixes prefixes = { @@ -101,9 +101,9 @@ def test_read_file_multival_with_no_strategy_and_predicates(neo4j_container, neo for i in range(len(records)): assert records_equal(records[i], records_from_rdf_lib[i]) -def test_read_file_multival_array_as_set_behavior(neo4j_container, neo4j_driver): +def test_read_file_multival_array_as_set_behavior(neo4j_driver, neo4j_connection_parameters): """When importing the data, if a triple will add the same value to a multivalued property it won't be added""" - auth_data = get_credentials(LOCAL, neo4j_container) + auth_data = neo4j_connection_parameters prefixes = {'music': Namespace('neo4j://graph.schema#')} diff --git a/test/integration/single_triple_test.py b/test/integration/single_triple_test.py index 14a2cdb..dc89a43 100644 --- a/test/integration/single_triple_test.py +++ b/test/integration/single_triple_test.py @@ -3,7 +3,7 @@ from test.integration.constants import GET_DATA_QUERY, RDFLIB_DB import pytest from test.integration.fixtures import neo4j_container, neo4j_driver, graph_store, graph_store_batched, \ - cleanup_databases + cleanup_databases, neo4j_connection_parameters def test_import_type_as_label(neo4j_driver, graph_store): diff --git a/test/integration/store_initialization_test.py b/test/integration/store_initialization_test.py new file mode 100644 index 0000000..b97dbef --- /dev/null +++ b/test/integration/store_initialization_test.py @@ -0,0 +1,56 @@ +from rdflib import Literal, RDF, URIRef, Graph +from rdflib.namespace import FOAF + +from rdflib_neo4j import HANDLE_VOCAB_URI_STRATEGY, Neo4jStoreConfig, Neo4jStore +from test.integration.constants import GET_DATA_QUERY, RDFLIB_DB +import pytest +from test.integration.fixtures import neo4j_connection_parameters, neo4j_driver, neo4j_container + + +def test_initialize_store_with_credentials(neo4j_connection_parameters, neo4j_driver): + + auth_data = neo4j_connection_parameters + + config = Neo4jStoreConfig(auth_data=auth_data, + custom_prefixes={}, + custom_mappings=[], + multival_props_names=[], + handle_vocab_uri_strategy=HANDLE_VOCAB_URI_STRATEGY.MAP, + batching=False) + + graph_store = Graph(store=Neo4jStore(config=config)) + donna = URIRef("https://example.org/donna") + graph_store.add((donna, FOAF.name, Literal("Donna Fales"))) + graph_store.commit() + records, summary, keys = neo4j_driver.execute_query(GET_DATA_QUERY, database_=RDFLIB_DB) + assert len(records) == 1 + + +def test_initialize_store_with_driver(neo4j_driver): + + config = Neo4jStoreConfig(auth_data=None, + custom_prefixes={}, + custom_mappings=[], + multival_props_names=[], + handle_vocab_uri_strategy=HANDLE_VOCAB_URI_STRATEGY.MAP, + batching=False) + + graph_store = Graph(store=Neo4jStore(config=config, neo4j_driver=neo4j_driver)) + donna = URIRef("https://example.org/donna") + graph_store.add((donna, FOAF.name, Literal("Donna Fales"))) + graph_store.commit() + records, summary, keys = neo4j_driver.execute_query(GET_DATA_QUERY, database_=RDFLIB_DB) + assert len(records) == 1 + + +def test_initialize_with_both_credentials_and_driver_should_fail(neo4j_connection_parameters, neo4j_driver): + + config = Neo4jStoreConfig(auth_data=neo4j_connection_parameters, + custom_prefixes={}, + custom_mappings=[], + multival_props_names=[], + handle_vocab_uri_strategy=HANDLE_VOCAB_URI_STRATEGY.MAP, + batching=False) + + with pytest.raises(Exception): + Graph(store=Neo4jStore(config=config, neo4j_driver=neo4j_driver)) diff --git a/test/integration/utils.py b/test/integration/utils.py index d659dc3..e24624a 100644 --- a/test/integration/utils.py +++ b/test/integration/utils.py @@ -1,7 +1,5 @@ from neo4j import Record from rdflib import Graph -from testcontainers.neo4j import Neo4jContainer - from rdflib_neo4j.Neo4jStore import Neo4jStore from rdflib_neo4j.config.Neo4jStoreConfig import Neo4jStoreConfig from rdflib_neo4j.config.const import HANDLE_VOCAB_URI_STRATEGY @@ -70,57 +68,3 @@ def read_file_n10s_and_rdflib(neo4j_driver, graph_store, batching=False, n10s_pa n10s_rels, summary, keys = neo4j_driver.execute_query(GET_RELS_QUERY) rdflib_rels, summary, keys = neo4j_driver.execute_query(GET_RELS_QUERY, database_=RDFLIB_DB) return records_from_rdf_lib, records, rdflib_rels, n10s_rels - - -def create_graph_store(neo4j_container, batching=False): - if neo4j_container: - auth_data = {'uri': neo4j_container.get_connection_url(), - 'database': RDFLIB_DB, - 'user': "neo4j", - 'pwd': Neo4jContainer.NEO4J_ADMIN_PASSWORD} - return config_graph_store(auth_data, batching) - else: - auth_data = { - 'uri': os.getenv("NEO4J_URI_LOCAL"), - 'database': RDFLIB_DB, - 'user': os.getenv("NEO4J_USER_LOCAL"), - 'pwd': os.getenv("NEO4J_PWD_LOCAL") - } - - return config_graph_store(auth_data, batching) - - -def config_graph_store(auth_data, batching=False): - # Define your prefixes - prefixes = {} - - # Define your custom mappings - custom_mappings = [] - - multival_props_names = [] - - config = Neo4jStoreConfig(auth_data=auth_data, - custom_prefixes=prefixes, - custom_mappings=custom_mappings, - multival_props_names=multival_props_names, - handle_vocab_uri_strategy=HANDLE_VOCAB_URI_STRATEGY.IGNORE, - batching=batching) - - g = Graph(store=Neo4jStore(config=config)) - return g - - -def get_credentials(local, neo4j_container): - if local: - auth_data = { - 'uri': os.getenv("NEO4J_URI_LOCAL"), - 'database': RDFLIB_DB, - 'user': os.getenv("NEO4J_USER_LOCAL"), - 'pwd': os.getenv("NEO4J_PWD_LOCAL") - } - else: - auth_data = {'uri': neo4j_container.get_connection_url(), - 'database': RDFLIB_DB, - 'user': "neo4j", - 'pwd': Neo4jContainer.NEO4J_ADMIN_PASSWORD} - return auth_data