Add common data structures (#134)

* add hdmf-common-schema as submodule

* add API classes for common

* add some useful stuff to Container and Data

* commiting current state so I dont lose anything

* always set data const arg for Data contaienrs

* fix tests and satisfy flake8

* fix tests and satisfy flake8

* add DynamicTable tests

* deepcopy when autogenerating a constructor docval

* fix imports

* fix formatting

* add merge method

* populate global type map as registers get called

* commit current state so I dont lose it

* fix import error

* fix minor error

* fix cached spec loading

* generalize get_ancestor

* add clearer name

* comment __fields__

* add tests for common

* add IO subpackage for common

* remove delayed registration

* fix package data

* checkout submodules on Circle

* add scipy as requirement

* Use line-specific flake8 exception

* remove debugging leftovers

* add comments and remove references to NWB

* Fix #145

* Fix tests to test for "is" instead of ==

* add some more comments/documentation

.circleci/config.yml

@@ -20,6 +20,8 @@ references:
   ci-steps: &ci-steps
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
       - run:
           <<: *initialize-venv
       - run:
@@ -65,6 +67,8 @@ references:
   conda-steps: &conda-steps
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
       - run:
           name: Configure conda
           command: |
@@ -89,6 +93,8 @@ references:
   gallery-steps: &gallery-steps
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
       - restore_cache:
           keys:
             - ophys-data-cache

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/hdmf/common/hdmf-common-schema"]
+	path = src/hdmf/common/hdmf-common-schema
+	url = https://github.com/hdmf-dev/hdmf-common-schema.git

Makefile

@@ -42,7 +42,7 @@ flake:
 	$(FLAKE) tests/
 
 checkpdb:
-	find {src,tests} -name "[a-z]*.py" -exec grep -Hn -e pdb -e print -e breakpoint {} \;
+	find {src,tests} -name "[a-z]*.py" -exec grep -Hn -e pdb -e print\( -e breakpoint {} \;
 
 devtest:
 	$(PYTHON) test.py

requirements.txt

@@ -1,6 +1,7 @@
 chardet==3.0.4
 h5py==2.9.0
 numpy==1.17.0
+scipy==1.3.1
 pandas==0.25.0
 python-dateutil==2.8.0
 ruamel.yaml==0.16.0

setup.py

@@ -10,7 +10,7 @@
 pkgs = find_packages('src', exclude=['data'])
 print('found these packages:', pkgs)
 
-schema_dir = 'data'
+schema_dir = 'common/hdmf-common-schema/common'
 
 setup_args = {
     'name': 'hdmf',
@@ -35,6 +35,7 @@
     ],
     'packages': pkgs,
     'package_dir': {'': 'src'},
+    'package_data': {'hdmf': ["%s/*.yaml" % schema_dir, "%s/*.json" % schema_dir]},
     'classifiers': [
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.5",

src/hdmf/__init__.py

@@ -1,8 +1,8 @@
+from . import query  # noqa: F401
 from .container import Container, Data, DataRegion
 from .utils import docval, getargs
 from .data_utils import ListSlicer
 from .backends.hdf5.h5_utils import H5RegionSlicer, H5Dataset
-from . import query
 
 
 @docval({'name': 'dataset', 'type': None, 'doc': 'the HDF5 dataset to slice'},

src/hdmf/backends/hdf5/h5_utils.py

@@ -160,6 +160,7 @@ def __init__(self, **kwargs):
         self.__group = getargs('group', kwargs)
         super_kwargs = {'source': "%s:%s" % (os.path.abspath(self.__group.file.name), self.__group.name)}
         call_docval_func(super(H5SpecReader, self).__init__, super_kwargs)
+        self.__cache = None
 
     def __read(self, path):
         s = self.__group[path][()]
@@ -177,8 +178,9 @@ def read_spec(self, spec_path):
         return self.__read(spec_path)
 
     def read_namespace(self, ns_path):
-        ret = self.__read(ns_path)
-        ret = ret['namespaces']
+        if self.__cache is None:
+            self.__cache = self.__read(ns_path)
+        ret = self.__cache['namespaces']
         return ret
 
 

src/hdmf/backends/hdf5/h5tools.py

@@ -104,15 +104,59 @@ def load_namespaces(cls, namespace_catalog, path, namespaces=None):
             if namespaces is None:
                 namespaces = list(spec_group.keys())
 
+            readers = dict()
+            deps = dict()
             for ns in namespaces:
                 ns_group = spec_group[ns]
                 latest_version = list(ns_group.keys())[-1]
                 ns_group = ns_group[latest_version]
                 reader = H5SpecReader(ns_group)
+                readers[ns] = reader
+                for spec_ns in reader.read_namespace('namespace'):
+                    deps[ns] = list()
+                    for s in spec_ns['schema']:
+                        dep = s.get('namespace')
+                        if dep is not None:
+                            deps[ns].append(dep)
+
+            order = cls._order_deps(deps)
+            for ns in order:
+                reader = readers[ns]
                 d.update(namespace_catalog.load_namespaces('namespace', reader=reader))
 
             return d
 
+    @classmethod
+    def _order_deps(cls, deps):
+        """
+        Order namespaces according to dependency for loading into a NamespaceCatalog
+
+        Args:
+            deps (dict): a dictionary that maps a namespace name to a list of name of
+                         the namespaces on which the the namespace is directly dependent
+                         Example: {'a': ['b', 'c'], 'b': ['d'], c: ['d'], 'd': []}
+                         Expected output: ['d', 'b', 'c', 'a']
+        """
+        order = list()
+        keys = list(deps.keys())
+        deps = dict(deps)
+        for k in keys:
+            if k in deps:
+                cls.__order_deps_aux(order, deps, k)
+        return order
+
+    @classmethod
+    def __order_deps_aux(cls, order, deps, key):
+        """
+        A recursive helper function for _order_deps
+        """
+        if key not in deps:
+            return
+        subdeps = deps.pop(key)
+        for subk in subdeps:
+            cls.__order_deps_aux(order, deps, subk)
+        order.append(key)
+
     @classmethod
     def __convert_namespace(cls, ns_catalog, namespace):
         ns = ns_catalog.get_namespace(namespace)