initial import

.editorconfig

@@ -0,0 +1,3 @@
+[*]
+indent_style = space
+indent_size = 4

.gitignore

@@ -19,3 +19,6 @@ yarn-error.log*
 *.njsproj
 *.sln
 *.sw?
+
+testdata
+

.stignore

@@ -0,0 +1,2 @@
+testdata
+testdata/*

api/bin/expand.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+#for 7z sudo apt-get install p7zip-full
+
+set -e
+set -x
+
+root=$1
+cd $root
+
+echo "expanding zip/gz/tar in $root"
+
+#expand various things that we can expand
+function expand {
+    for tar in $(find -name "*.tar*"); do
+        echo $tar
+        #tar is too verbose
+        tar -xf $tar -C $(dirname $tar)
+        rm $tar
+    done
+
+    for tar in $(find -name "*.tgz"); do
+        echo $tar
+        #tar is too verbose
+        tar -xf $tar -C $(dirname $tar)
+        rm $tar
+    done
+
+    for gz in $(find -name "*.gz"); do
+        echo $tar
+        gunzip $gz
+    done
+
+    for zip in $(find -name "*.7z"); do
+        echo $tar
+        7z x $zip
+        rm $zip
+    done
+
+    for zip in $(find -name "*.bz2"); do
+        echo $tar
+        bunzip2 $zip
+    done
+
+    for zip in $(find -name "*.zip"); do
+        echo $tar
+        unzip -o $zip -d $(dirname $zip)
+        rm $zip
+    done
+
+    #TODO - 7z, .xz .bz2
+}
+
+#keep expanding until there is nothing else to expand
+while true; do
+    expand | tee expand.log
+    if [ ! -s expand.log ]; then
+        echo "expand log empty ..done"
+        break
+    else
+        echo "keep going"
+    fi
+done
+

api/bin/find_dicomdir.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+def find_leaf(dir):
+    leaf=True
+
+    #is it leaf?
+    for x in os.listdir(dir):
+        fulldir=os.path.join(dir, x)
+        if os.path.isdir(fulldir):
+            #print(fulldir+" is directory")
+            leaf=False
+            continue
+
+    #if it contains DICOMDIR, treat it as leaf
+    if not leaf:
+        for x in os.listdir(dir):
+            if x == "DICOMDIR":
+                #print("dicomdir detected")
+                leaf=True
+
+    if leaf:
+        #print(dir+" is leaf - no sub directory")
+        print(dir)
+    else:
+        #recurse to all child dirs
+        for x in os.listdir(dir):
+            fulldir=os.path.join(dir, x)
+            if os.path.isdir(fulldir):
+                find_leaf(fulldir)
+
+os.chdir(sys.argv[1])
+
+find_leaf('.')
+

api/bin/preprocess.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+module load pigz
+
+set -e
+set -x
+
+if [ -z $1 ]; then
+    echo "please specify root dir"
+    exit 1
+fi
+root=$1
+
+./expand.sh $root
+ls $root
+
+echo processing $root
+(cd $root && find . -print > raw.list)
+
+#find leaf directories
+for dir in $(python find_dicomdir.py $root);
+do
+        (cd $root && dcm2niix -z yes -f 'time-%t-sn-%s' -v 1 $dir)
+done
+
+(cd $root && find . -type f \( -name "*.json" -o -name "*.nii.gz" \) > list)
+

api/bin/preprocess.ts

@@ -0,0 +1,69 @@
+
+const { spawn } = require('child_process');
+import fs = require('fs');
+
+import config = require('../config');
+import models = require('../models');
+
+console.log("starting preprocess");
+models.connect(err=>{
+    if(err) throw err;
+    run();
+});
+
+function run() {
+    console.log("finding sessions to preprocess");
+    models.Session.find({
+        upload_finish_date: {$exists: true}, 
+        pre_begin_date: {$exists: false},
+    }).then(async sessions=>{
+        for(let session of sessions) {
+            await handle_session(session);
+        }
+        console.log("done.. taking a break")
+        setTimeout(run, 1000*5);
+    });
+}
+
+function handle_session(session) {
+    console.log("handling session "+session._id);
+    return new Promise((resolve, reject)=>{
+        session.pre_begin_date = new Date();
+        session.status = "preprocessing"; //not working?
+        session.save().then(()=>{
+
+            let workdir = config.workdir+"/"+session._id;
+            const p = spawn('./preprocess.sh', [workdir], {cwd: __dirname});
+            const logout = fs.openSync(workdir+"/preprocess.log", "w");
+            const errout = fs.openSync(workdir+"/preprocess.err", "w");
+            p.stdout.on('data', data=>{
+                console.log(data.toString("utf8"));
+                fs.writeSync(logout, data);
+            });
+            p.stderr.on('data', data=>{
+                console.log(data.toString("utf8"));
+                fs.writeSync(errout, data);
+            })
+            p.on('close', code=>{
+
+                //check status
+                console.debug("preprocess.sh finished: "+code);
+                if(code != 0) {
+                    session.status = "failed";
+                    session.status_msg = "failed to run preprocess.sh";
+                } else {
+                    session.status = "validating";
+                    session.status_msg = "successfully run preprocess.sh";
+                    session.pre_finish_date = new Date();
+                }
+
+                //update session and done.
+                session.save().then(()=>{
+                    resolve();
+                }).catch(err=>{
+                    reject();
+                });
+            })
+        });
+    });
+}

api/config.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.mongodb = "mongodb://localhost/autobids";
+exports.mongoose_debug = true;
+
+//multer incoming upload directory
+exports.multer = {
+    dest: "/mnt/datalad/easybids/upload",
+}
+//directory to copy uploaded files (it needs to be on the same filesystem as multer incoming dir)
+exports.workdir = "/mnt/datalad/easybids/workdir",
+
+exports.express = {
+    port: 12507,
+}