create all_specs as dict in loader to avoid unnecessary de-/serializa…

…tion
poldracklab · Sep 20, 2021 · 54fe948 · 54fe948
1 parent b60de7c
commit 54fe948
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 37 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -54,7 +54,7 @@ jobs:
           paths:
             - src/fitlins
       - save_cache:
-          key: docker-v0-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
+          key: docker-v2-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
           paths:
             - /tmp/cache/docker.tar.gz
 
@@ -67,9 +67,9 @@ jobs:
           path: /tmp/src
       - restore_cache:
           keys:
-            - ds003-v1-{{ .Branch }}-
-            - ds003-v1-master-
-            - ds003-v1-
+            - ds003-v2-{{ .Branch }}-
+            - ds003-v2-master-
+            - ds003-v2-
       - run:
           name: Set git ID
           command: |
@@ -86,7 +86,7 @@ jobs:
                                     fitlins_tests/ds003/fmriprep/sub-*/*/*.json
             datalad get -r fitlins_tests/ds003/nistats_smooth/ fitlins_tests/ds003/afni_smooth/ fitlins_tests/ds003/afni_blurto/ fitlins_tests/ds003/nistats_blurto/
       - save_cache:
-         key: ds003-v1-{{ .Branch }}-{{ .BuildNum }}
+         key: ds003-v2-{{ .Branch }}-{{ .BuildNum }}
          paths:
             - /tmp/data
 
@@ -126,7 +126,7 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - docker-v0-{{ .Branch }}-{{ .Revision }}
+            - docker-v2-{{ .Branch }}-{{ .Revision }}
       - run:
           name: Load Docker image layer cache
           no_output_timeout: 30m
@@ -172,12 +172,12 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - docker-v0-{{ .Branch }}-{{ .Revision }}
+            - docker-v2-{{ .Branch }}-{{ .Revision }}
       - restore_cache:
           keys:
-            - ds003-v1-{{ .Branch }}-
-            - ds003-v1-master-
-            - ds003-v1-
+            - ds003-v2-{{ .Branch }}-
+            - ds003-v2-master-
+            - ds003-v2-
       - run:
           name: Load Docker image layer cache
           no_output_timeout: 30m
@@ -235,12 +235,12 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - docker-v0-{{ .Branch }}-{{ .Revision }}
+            - docker-v2-{{ .Branch }}-{{ .Revision }}
       - restore_cache:
           keys:
-            - ds003-v1-{{ .Branch }}-
-            - ds003-v1-master-
-            - ds003-v1-
+            - ds003-v2-{{ .Branch }}-
+            - ds003-v2-master-
+            - ds003-v2-
       - run:
           name: Load Docker image layer cache
           no_output_timeout: 30m
@@ -395,7 +395,7 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - docker-v0-{{ .Branch }}-{{ .Revision }}
+            - docker-v2-{{ .Branch }}-{{ .Revision }}
       - run:
           name: Load Docker image layer cache
           no_output_timeout: 30m
@@ -423,7 +423,7 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - docker-v0-{{ .Branch }}-{{ .Revision }}
+            - docker-v2-{{ .Branch }}-{{ .Revision }}
       - run:
           name: Load Docker image layer cache
           no_output_timeout: 30m

diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py
@@ -212,7 +212,23 @@ def _load_graph(self, runtime, graph, node=None, inputs=None, **filters):
         if node.level == 'run':
             self._load_run_level(runtime, graph, specs)
 
-        all_specs = {node.name: specs}
+        level_spec = []
+        for coll in specs:
+            spec = {}
+            spec['contrasts'] = coll.contrasts
+            spec['entities'] = coll.entities.copy()
+            spec['level'] = coll.node.level
+            spec['X'] = coll.X.copy()
+            spec['model'] = coll.node.model.copy()
+
+            # slight optimization since metadata is only
+            # used in higher level models
+            if node.level != 'run':
+                spec['metadata'] = coll.metadata.copy()
+
+            level_spec.append(spec)
+
+        all_specs = {node.name: level_spec}
 
         for child in node.children:
             all_specs.update(

diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py
@@ -223,37 +223,20 @@ def _deindex(tsv):
     model = l1_model
 
     def select_stats_spec(all_specs, name):
-        level_specs = []
-
-        for coll in all_specs[name]:
-            spec = {}
-            spec['contrasts'] = coll.contrasts
-            spec['entities'] = coll.entities.copy()
-            spec['level'] = coll.node.level
-            spec['X'] = coll.X.copy()
-            spec['model'] = coll.node.model.copy()
-
-            # slight optimization since metadata is only
-            # used in higher level models
-            if coll.node.level != 'run':
-                spec['metadata'] = coll.metadata.copy()
-
-            level_specs.append(spec)
-
-        return level_specs
+        return all_specs[name]
 
     def select_ents(all_specs, name):
         entities = []
         spec = all_specs[name]
         for coll in spec:
-            entities.append(coll.entities.copy())
+            entities.append(coll['entities'].copy())
         return entities
 
     def select_conts(all_specs, name):
         contrasts = []
         spec = all_specs[name]
         for coll in spec:
-            contrasts.append(coll.contrasts)
+            contrasts.append(coll['contrasts'])
         return contrasts
 
     for node, nobj in graph.nodes.items():