add file to provider functions

atramhasis/scripts/import_file.py

@@ -7,6 +7,12 @@
 from rdflib import Graph
 from rdflib.util import guess_format
 
+from skosprovider.providers import SimpleCsvProvider
+import csv
+
+from skosprovider.providers import DictionaryProvider
+import json
+
 from skosprovider_sqlalchemy.utils import import_provider
 from skosprovider_sqlalchemy.models import (
     ConceptScheme,
@@ -30,6 +36,32 @@ def file_to_rdf_provider(input_file):
         graph
     )
 
+
+def file_to_csv_provider(input_file):
+    '''
+    Create CSV provider from the input file
+    '''
+    input_name, input_ext = os.path.splitext(os.path.basename(input_file))
+    with open(input_file, "rb") as ifile:
+        reader = csv.reader(ifile)
+    return SimpleCsvProvider(
+        {'id': input_name.upper()},
+        reader,
+    )
+
+
+def file_to_json_provider(input_file):
+    '''
+    Create Dictionary provider from the input file
+    '''
+    input_name, input_ext = os.path.splitext(os.path.basename(input_file))
+    with open(input_file, 'rb') as data_file:
+        dictionary = json.load(data_file)
+    return DictionaryProvider(
+        {'id': input_name.upper()},
+        dictionary,
+    )
+
 supported_types = {
     'RDF': {
         'extensions': ['.html', '.hturtle', '.mdata', '.microdata', '.n3', '.nquads', '.nt', '.rdfa', '.rdfa1.0',
@@ -38,11 +70,11 @@ def file_to_rdf_provider(input_file):
     },
     'CSV': {
         'extensions': ['.csv'],
-        'file_to_provider': ''
+        'file_to_provider': file_to_csv_provider
     },
     'JSON': {
         'extensions': ['.json'],
-        'file_to_provider': ''
+        'file_to_provider': file_to_json_provider
     }
 }
 
@@ -165,6 +197,6 @@ def main(argv=sys.argv):
         print('Importer is not yet implemented')
         sys.exit(1)
     else:
-        provider = file_to_provider_function(args.input_file, input_name, input_ext)
+        provider = file_to_provider_function(args.input_file)
     cs = create_conceptscheme(input_name.capitalize())
     provider_to_db(provider, cs, session)

tests/data/trees.json

@@ -0,0 +1,60 @@
+[{"broader": [],
+  "id": 1,
+  "labels": [{"label": "The Larch",
+               "language": "en",
+               "type": "prefLabel"},
+              {"label": "De Lariks",
+               "language": "nl",
+               "type": "prefLabel"}],
+  "matches": {"broad": [],
+               "close": [],
+               "exact": [],
+               "narrow": [],
+               "related": []},
+  "member_of": [3],
+  "narrower": [],
+  "notes": [{"language": "en",
+              "note": "A type of tree.",
+              "type": "definition"}],
+  "related": [],
+  "subordinate_arrays": [],
+  "type": "concept",
+  "uri": "http://id.trees.org/1"},
+ {"broader": [],
+  "id": 2,
+  "labels": [{"label": "The Chestnut",
+               "language": "en",
+               "type": "prefLabel"},
+              {"label": "De Paardekastanje",
+               "language": "nl",
+               "type": "altLabel"},
+              {"label": "la châtaigne",
+               "language": "fr",
+               "type": "altLabel"}],
+  "matches": {"broad": [],
+               "close": [],
+               "exact": [],
+               "narrow": [],
+               "related": []},
+  "member_of": [3],
+  "narrower": [],
+  "notes": [{"language": "en",
+              "note": "A different type of tree.",
+              "type": "definition"}],
+  "related": [],
+  "subordinate_arrays": [],
+  "type": "concept",
+  "uri": "http://id.trees.org/2"},
+ {"id": 3,
+  "labels": [{"label": "Bomen per soort",
+               "language": "nl",
+               "type": "prefLabel"},
+              {"label": "Trees by species",
+               "language": "en",
+               "type": "prefLabel"}],
+  "member_of": [],
+  "members": [1, 2],
+  "notes": [],
+  "superordinates": [],
+  "type": "collection",
+  "uri": "http://id.trees.org/3"}]

tests/test_import_scripts.py

@@ -16,7 +16,8 @@
 
 here = os.path.dirname(__file__)
 settings = get_appsettings(os.path.join(here, '../', 'tests/conf_test.ini'))
-test_data = os.path.join(here, '../', 'tests/data/trees.xml')
+test_data_rdf = os.path.join(here, '../', 'tests/data/trees.xml')
+test_data_json = os.path.join(here, '../', 'tests/data/trees.json')
 
 
 class ImportTests(unittest.TestCase):
@@ -39,10 +40,7 @@ def setUp(self):
     def tearDown(self):
         Base.metadata.drop_all(self.engine)
 
-    def test_import_rdf(self):
-        sys.argv = ['import_file', '--from', test_data, '--to', settings['sqlalchemy.url']]
-        import_file.main(sys.argv)
-
+    def _check_trees(self):
         sql_prov = SQLAlchemyProvider({'id': 'TREES', 'conceptscheme_id': 1}, self.session_maker)
         dump = dict_dumper(sql_prov)
 
@@ -61,6 +59,14 @@ def test_import_rdf(self):
         self.assertDictEqual(obj_1['notes'][0],
                              {'language': 'en', 'note': 'A different type of tree.', 'type': 'definition'})
 
+    def test_import_rdf(self):
+        sys.argv = ['import_file', '--from', test_data_rdf, '--to', settings['sqlalchemy.url']]
+        import_file.main(sys.argv)
+
+    def test_import_json(self):
+        sys.argv = ['import_file', '--from', test_data_json, '--to', settings['sqlalchemy.url']]
+        import_file.main(sys.argv)
+