From 67c3c7aa743acfb33c9c52de7d3bfe37995a773e Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Fri, 8 Mar 2024 18:00:43 +0100 Subject: [PATCH 01/10] compare _scans.tsv with .bidsignore --- bids-validator/utils/files/readDir.js | 2 ++ bids-validator/validators/tsv/tsv.js | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/bids-validator/utils/files/readDir.js b/bids-validator/utils/files/readDir.js index 902114204..0b931e8bf 100644 --- a/bids-validator/utils/files/readDir.js +++ b/bids-validator/utils/files/readDir.js @@ -330,6 +330,8 @@ async function getBIDSIgnore(dir) { if (bidsIgnoreFileObj) { const content = await readFile(bidsIgnoreFileObj) ig.add(content) + // Store the .bidsignore content in session storage + sessionStorage.setItem('bidsignoreContent', JSON.stringify(content)); } return ig } diff --git a/bids-validator/validators/tsv/tsv.js b/bids-validator/validators/tsv/tsv.js index fa48b83bc..6c8e59d2b 100644 --- a/bids-validator/validators/tsv/tsv.js +++ b/bids-validator/validators/tsv/tsv.js @@ -6,6 +6,7 @@ import checkStatusCol from './checkStatusCol' import checkTypecol from './checkTypeCol' import parseTSV from './tsvParser' import checkMotionComponent from './checkMotionComponent' +import ignore from 'ignore' var path = require('path') /** @@ -612,6 +613,10 @@ const TSV = (file, contents, fileList, callback) => { }), ) } else { + // Retrieve the .bidsignore content (if any) from session storage + const content = sessionStorage.getItem('bidsignoreContent'); + const ig = content ? ignore().add(JSON.parse(content)) : null; + // check scans filenames match pathList const filenameColumn = headers.indexOf('filename') for (let l = 1; l < rows.length; l++) { @@ -619,6 +624,11 @@ const TSV = (file, contents, fileList, callback) => { const scanRelativePath = row[filenameColumn] const scanFullPath = scanDirPath + '/' + scanRelativePath + // check if file should be ignored based on .bidsignore content + if (ig && ig.ignores((path.relative('/', scanRelativePath)))) { + continue + } + // check if scan matches full dataset path list if (!pathList.includes(scanFullPath)) { issues.push( From 7195fd240e053caba53c5699ed895f2bacfb674b Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Fri, 8 Mar 2024 18:01:38 +0100 Subject: [PATCH 02/10] update tests with mock session storage --- bids-validator/tests/bids.spec.js | 7 +++++++ bids-validator/tests/tsv.spec.js | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/bids-validator/tests/bids.spec.js b/bids-validator/tests/bids.spec.js index 40a6e88b3..c2b351e6a 100644 --- a/bids-validator/tests/bids.spec.js +++ b/bids-validator/tests/bids.spec.js @@ -9,6 +9,13 @@ import path from 'path' import { createFileList } from './env/FileList.js' import isNode from '../utils/isNode.js' +// Mock sessionStorage +global.sessionStorage = { + getItem: jest.fn(), + setItem: jest.fn(), + clear: jest.fn() +}; + function getDirectories(srcpath) { return fs.readdirSync(srcpath).filter(function (file) { return ( diff --git a/bids-validator/tests/tsv.spec.js b/bids-validator/tests/tsv.spec.js index 2d43b809d..12cb5b725 100644 --- a/bids-validator/tests/tsv.spec.js +++ b/bids-validator/tests/tsv.spec.js @@ -1,6 +1,13 @@ import assert from 'assert' import validate from '../index' +// Mock sessionStorage +global.sessionStorage = { + getItem: jest.fn(), + setItem: jest.fn(), + clear: jest.fn() +}; + describe('TSV', function () { // general tsv checks ------------------------------------------------------------------ From ad2dd28a3ae8d9f10e4d0b2e0ca291e8dfa0aff9 Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Sat, 9 Mar 2024 16:06:40 +0100 Subject: [PATCH 03/10] sessionstorage fix for testing --- bids-validator/tests/bids.spec.js | 7 ------- bids-validator/tests/setupTests.js | 6 ++++++ bids-validator/tests/tsv.spec.js | 9 ++------- package.json | 1 + 4 files changed, 9 insertions(+), 14 deletions(-) create mode 100644 bids-validator/tests/setupTests.js diff --git a/bids-validator/tests/bids.spec.js b/bids-validator/tests/bids.spec.js index c2b351e6a..40a6e88b3 100644 --- a/bids-validator/tests/bids.spec.js +++ b/bids-validator/tests/bids.spec.js @@ -9,13 +9,6 @@ import path from 'path' import { createFileList } from './env/FileList.js' import isNode from '../utils/isNode.js' -// Mock sessionStorage -global.sessionStorage = { - getItem: jest.fn(), - setItem: jest.fn(), - clear: jest.fn() -}; - function getDirectories(srcpath) { return fs.readdirSync(srcpath).filter(function (file) { return ( diff --git a/bids-validator/tests/setupTests.js b/bids-validator/tests/setupTests.js new file mode 100644 index 000000000..ad50e082f --- /dev/null +++ b/bids-validator/tests/setupTests.js @@ -0,0 +1,6 @@ +// Mock sessionStorage +global.sessionStorage = { + getItem: jest.fn(), + setItem: jest.fn(), + clear: jest.fn() +}; \ No newline at end of file diff --git a/bids-validator/tests/tsv.spec.js b/bids-validator/tests/tsv.spec.js index 12cb5b725..c81c0cfe5 100644 --- a/bids-validator/tests/tsv.spec.js +++ b/bids-validator/tests/tsv.spec.js @@ -1,13 +1,6 @@ import assert from 'assert' import validate from '../index' -// Mock sessionStorage -global.sessionStorage = { - getItem: jest.fn(), - setItem: jest.fn(), - clear: jest.fn() -}; - describe('TSV', function () { // general tsv checks ------------------------------------------------------------------ @@ -355,10 +348,12 @@ describe('TSV', function () { it('should not allow mismatched filename entries', function () { const fileList = [eegFile] + console.log(fileList) const tsv = 'filename\tacq_time\n' + 'func/sub-08_ses-test_task-linebisection_run-01_bold.nii.gz\t2017-05-03T06:45:45' validate.TSV.TSV(scansFile, tsv, fileList, function (issues) { + console.log(issues) assert(issues.length === 1 && issues[0].code === 129) }) }) diff --git a/package.json b/package.json index 80ffe421e..680c658de 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ }, "jest": { "testEnvironment": "node", + "setupFilesAfterEnv": ["/bids-validator/tests/setupTests.js"], "moduleNameMapper": { "^uuid$": "uuid" }, From 65da18fedf04fb9c9f57d0e53638b07ca49d7434 Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Sat, 9 Mar 2024 17:16:30 +0100 Subject: [PATCH 04/10] update mock session storage for tests --- bids-validator/tests/setupTests.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bids-validator/tests/setupTests.js b/bids-validator/tests/setupTests.js index ad50e082f..96472c69a 100644 --- a/bids-validator/tests/setupTests.js +++ b/bids-validator/tests/setupTests.js @@ -1,6 +1,7 @@ // Mock sessionStorage global.sessionStorage = { - getItem: jest.fn(), - setItem: jest.fn(), - clear: jest.fn() + getItem: key => sessionStorage[key], + setItem: (key, value) => { sessionStorage[key] = value; }, + removeItem: key => { delete sessionStorage[key]; }, + clear: () => { sessionStorage = {}; }, }; \ No newline at end of file From 5388f27abc639c5d798b5580dba7e96229367d2a Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Sat, 9 Mar 2024 17:21:17 +0100 Subject: [PATCH 05/10] add test for .bidsignore and scans.tsv check --- bids-validator/tests/tsv.spec.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bids-validator/tests/tsv.spec.js b/bids-validator/tests/tsv.spec.js index c81c0cfe5..858081f2e 100644 --- a/bids-validator/tests/tsv.spec.js +++ b/bids-validator/tests/tsv.spec.js @@ -410,6 +410,21 @@ describe('TSV', function () { ) }) + it('should ignore files in scans.tsv that correspond to entries in .bidsignore', function () { + sessionStorage.setItem('bidsignoreContent', JSON.stringify('sodium/')) + const fileList = [niftiFile, eegFile, ieegFile] + const tsv = + 'filename\tacq_time\n' + + 'func/sub-08_ses-test_task-linebisection_run-01_bold.nii.gz\t2017-05-03T06:45:45\n' + + 'eeg/sub-08_ses-test_task-linebisection_run-01_eeg.fif\t2017-05-03T06:45:45\n' + + 'ieeg/sub-08_ses-test_task-linebisection_run-01_ieeg.edf\t2017-05-03T06:45:45\n' + + 'sodium/sub-08_acq-23Na_echo-01.nii.gz\t2018-04-26T21:30:00' + validate.TSV.TSV(scansFile, tsv, fileList, function (issues) { + assert.deepEqual(issues, []) + }) + }) + + // channels checks ----------------------------------------------------------------- var channelsFileMEG = { From 9c28a60d93a9f1d6875430a374bd00de2c57a5cd Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Sat, 9 Mar 2024 17:33:54 +0100 Subject: [PATCH 06/10] add test for missing files not in .bidsignore; clean up --- bids-validator/tests/tsv.spec.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/bids-validator/tests/tsv.spec.js b/bids-validator/tests/tsv.spec.js index 858081f2e..3f701855c 100644 --- a/bids-validator/tests/tsv.spec.js +++ b/bids-validator/tests/tsv.spec.js @@ -348,12 +348,10 @@ describe('TSV', function () { it('should not allow mismatched filename entries', function () { const fileList = [eegFile] - console.log(fileList) const tsv = 'filename\tacq_time\n' + 'func/sub-08_ses-test_task-linebisection_run-01_bold.nii.gz\t2017-05-03T06:45:45' validate.TSV.TSV(scansFile, tsv, fileList, function (issues) { - console.log(issues) assert(issues.length === 1 && issues[0].code === 129) }) }) @@ -422,9 +420,25 @@ describe('TSV', function () { validate.TSV.TSV(scansFile, tsv, fileList, function (issues) { assert.deepEqual(issues, []) }) + sessionStorage.removeItem('bidsignoreContent') }) + it('should not allow missing files listed in scans.tsv and not accounted for by .bidsignore', function () { + sessionStorage.setItem('bidsignoreContent', JSON.stringify('sodium/')) + const fileList = [niftiFile, eegFile] + const tsv = + 'filename\tacq_time\n' + + 'func/sub-08_ses-test_task-linebisection_run-01_bold.nii.gz\t2017-05-03T06:45:45\n' + + 'eeg/sub-08_ses-test_task-linebisection_run-01_eeg.fif\t2017-05-03T06:45:45\n' + + 'ieeg/sub-08_ses-test_task-linebisection_run-01_ieeg.edf\t2017-05-03T06:45:45\n' + + 'sodium/sub-08_acq-23Na_echo-01.nii.gz\t2018-04-26T21:30:00' + validate.TSV.TSV(scansFile, tsv, fileList, function (issues) { + assert(issues.length === 1 && issues[0].code === 129) + }) + sessionStorage.removeItem('bidsignoreContent') + }) + // channels checks ----------------------------------------------------------------- var channelsFileMEG = { From e2030b5a855c74fc243188eedc6950587ed38ecb Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Sat, 9 Mar 2024 18:32:21 +0100 Subject: [PATCH 07/10] fix lint related issues --- bids-validator/tests/setupTests.js | 16 +++++++++++----- bids-validator/tests/tsv.spec.js | 1 - bids-validator/utils/files/readDir.js | 2 +- bids-validator/validators/tsv/tsv.js | 6 +++--- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/bids-validator/tests/setupTests.js b/bids-validator/tests/setupTests.js index 96472c69a..ce031b31e 100644 --- a/bids-validator/tests/setupTests.js +++ b/bids-validator/tests/setupTests.js @@ -1,7 +1,13 @@ // Mock sessionStorage global.sessionStorage = { - getItem: key => sessionStorage[key], - setItem: (key, value) => { sessionStorage[key] = value; }, - removeItem: key => { delete sessionStorage[key]; }, - clear: () => { sessionStorage = {}; }, -}; \ No newline at end of file + getItem: (key) => sessionStorage[key], + setItem: (key, value) => { + sessionStorage[key] = value + }, + removeItem: (key) => { + delete sessionStorage[key] + }, + clear: () => { + Object.keys(sessionStorage).forEach((key) => sessionStorage.removeItem(key)) + }, +} diff --git a/bids-validator/tests/tsv.spec.js b/bids-validator/tests/tsv.spec.js index 3f701855c..8c81fa354 100644 --- a/bids-validator/tests/tsv.spec.js +++ b/bids-validator/tests/tsv.spec.js @@ -423,7 +423,6 @@ describe('TSV', function () { sessionStorage.removeItem('bidsignoreContent') }) - it('should not allow missing files listed in scans.tsv and not accounted for by .bidsignore', function () { sessionStorage.setItem('bidsignoreContent', JSON.stringify('sodium/')) const fileList = [niftiFile, eegFile] diff --git a/bids-validator/utils/files/readDir.js b/bids-validator/utils/files/readDir.js index 0b931e8bf..97eff639f 100644 --- a/bids-validator/utils/files/readDir.js +++ b/bids-validator/utils/files/readDir.js @@ -331,7 +331,7 @@ async function getBIDSIgnore(dir) { const content = await readFile(bidsIgnoreFileObj) ig.add(content) // Store the .bidsignore content in session storage - sessionStorage.setItem('bidsignoreContent', JSON.stringify(content)); + sessionStorage.setItem('bidsignoreContent', JSON.stringify(content)) } return ig } diff --git a/bids-validator/validators/tsv/tsv.js b/bids-validator/validators/tsv/tsv.js index 6c8e59d2b..95d2f27a4 100644 --- a/bids-validator/validators/tsv/tsv.js +++ b/bids-validator/validators/tsv/tsv.js @@ -614,8 +614,8 @@ const TSV = (file, contents, fileList, callback) => { ) } else { // Retrieve the .bidsignore content (if any) from session storage - const content = sessionStorage.getItem('bidsignoreContent'); - const ig = content ? ignore().add(JSON.parse(content)) : null; + const content = sessionStorage.getItem('bidsignoreContent') + const ig = content ? ignore().add(JSON.parse(content)) : null // check scans filenames match pathList const filenameColumn = headers.indexOf('filename') @@ -625,7 +625,7 @@ const TSV = (file, contents, fileList, callback) => { const scanFullPath = scanDirPath + '/' + scanRelativePath // check if file should be ignored based on .bidsignore content - if (ig && ig.ignores((path.relative('/', scanRelativePath)))) { + if (ig && ig.ignores(path.relative('/', scanRelativePath))) { continue } From a7ca0523e66472bf8875eb910b0aeae655793145 Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Thu, 21 Mar 2024 16:32:18 +0100 Subject: [PATCH 08/10] add browser check when reading directory --- bids-validator/utils/files/readDir.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bids-validator/utils/files/readDir.js b/bids-validator/utils/files/readDir.js index 97eff639f..5560483d9 100644 --- a/bids-validator/utils/files/readDir.js +++ b/bids-validator/utils/files/readDir.js @@ -5,6 +5,29 @@ import fs from 'fs' import * as child_proccess from 'child_process' import isNode from '../isNode' +function createSessionStorageMock() { + const sessionStorage = {} + + return { + getItem: (key) => sessionStorage[key], + setItem: (key, value) => { + sessionStorage[key] = value + }, + removeItem: (key) => { + delete sessionStorage[key] + }, + clear: () => { + Object.keys(sessionStorage).forEach((key) => + sessionStorage.removeItem(key), + ) + }, + } +} + +const sessionStorage = isNode + ? createSessionStorageMock() + : window.sessionStorage + /** * Read Directory * From aa04411d76d60babdfd39baa834e6b63d5205ed0 Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Fri, 22 Mar 2024 13:04:19 +0100 Subject: [PATCH 09/10] rework use of session storage --- bids-validator/tests/setupTests.js | 14 ++-------- bids-validator/utils/files/readDir.js | 24 ++---------------- bids-validator/utils/getSessionStorage.js | 31 +++++++++++++++++++++++ bids-validator/validators/tsv/tsv.js | 3 +++ 4 files changed, 38 insertions(+), 34 deletions(-) create mode 100644 bids-validator/utils/getSessionStorage.js diff --git a/bids-validator/tests/setupTests.js b/bids-validator/tests/setupTests.js index ce031b31e..0d9b97d89 100644 --- a/bids-validator/tests/setupTests.js +++ b/bids-validator/tests/setupTests.js @@ -1,13 +1,3 @@ // Mock sessionStorage -global.sessionStorage = { - getItem: (key) => sessionStorage[key], - setItem: (key, value) => { - sessionStorage[key] = value - }, - removeItem: (key) => { - delete sessionStorage[key] - }, - clear: () => { - Object.keys(sessionStorage).forEach((key) => sessionStorage.removeItem(key)) - }, -} +import getSessionStorage from '../utils/getSessionStorage' +global.sessionStorage = getSessionStorage() diff --git a/bids-validator/utils/files/readDir.js b/bids-validator/utils/files/readDir.js index 5560483d9..95b579b08 100644 --- a/bids-validator/utils/files/readDir.js +++ b/bids-validator/utils/files/readDir.js @@ -4,29 +4,9 @@ import path from 'path' import fs from 'fs' import * as child_proccess from 'child_process' import isNode from '../isNode' +import getSessionStorage from '../getSessionStorage' -function createSessionStorageMock() { - const sessionStorage = {} - - return { - getItem: (key) => sessionStorage[key], - setItem: (key, value) => { - sessionStorage[key] = value - }, - removeItem: (key) => { - delete sessionStorage[key] - }, - clear: () => { - Object.keys(sessionStorage).forEach((key) => - sessionStorage.removeItem(key), - ) - }, - } -} - -const sessionStorage = isNode - ? createSessionStorageMock() - : window.sessionStorage +const sessionStorage = isNode ? getSessionStorage() : window.sessionStorage /** * Read Directory diff --git a/bids-validator/utils/getSessionStorage.js b/bids-validator/utils/getSessionStorage.js new file mode 100644 index 000000000..374045da6 --- /dev/null +++ b/bids-validator/utils/getSessionStorage.js @@ -0,0 +1,31 @@ +// return sessionStorage based on environment +// uses mock object for use in tests and GitHub workflows +import isNode from './isNode' + +function getSessionStorage() { + if ('sessionStorage' in global) { + // created in setupTests.js; enables data sharing using same object + return global.sessionStorage + } else if (!isNode) { + return window.sessionStorage + } else { + const sessionStorage = {} + + return { + getItem: (key) => sessionStorage[key], + setItem: (key, value) => { + sessionStorage[key] = value + }, + removeItem: (key) => { + delete sessionStorage[key] + }, + clear: () => { + Object.keys(sessionStorage).forEach((key) => + sessionStorage.removeItem(key), + ) + }, + } + } +} + +export default getSessionStorage diff --git a/bids-validator/validators/tsv/tsv.js b/bids-validator/validators/tsv/tsv.js index 95d2f27a4..0b76e6cd6 100644 --- a/bids-validator/validators/tsv/tsv.js +++ b/bids-validator/validators/tsv/tsv.js @@ -6,9 +6,12 @@ import checkStatusCol from './checkStatusCol' import checkTypecol from './checkTypeCol' import parseTSV from './tsvParser' import checkMotionComponent from './checkMotionComponent' +import getSessionStorage from '../../utils/getSessionStorage' import ignore from 'ignore' var path = require('path') +const sessionStorage = getSessionStorage() + /** * Format TSV headers for evidence string * @param {Array[string]} headers From 6dacdea404b66b8dc533b8c239f2d017dcb3cd0e Mon Sep 17 00:00:00 2001 From: appukuttan-shailesh Date: Mon, 25 Mar 2024 11:22:55 +0100 Subject: [PATCH 10/10] increase coverage --- bids-validator/tests/tsv.spec.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bids-validator/tests/tsv.spec.js b/bids-validator/tests/tsv.spec.js index 8c81fa354..9055b10f4 100644 --- a/bids-validator/tests/tsv.spec.js +++ b/bids-validator/tests/tsv.spec.js @@ -409,6 +409,7 @@ describe('TSV', function () { }) it('should ignore files in scans.tsv that correspond to entries in .bidsignore', function () { + sessionStorage.clear() sessionStorage.setItem('bidsignoreContent', JSON.stringify('sodium/')) const fileList = [niftiFile, eegFile, ieegFile] const tsv = @@ -424,6 +425,7 @@ describe('TSV', function () { }) it('should not allow missing files listed in scans.tsv and not accounted for by .bidsignore', function () { + sessionStorage.clear() sessionStorage.setItem('bidsignoreContent', JSON.stringify('sodium/')) const fileList = [niftiFile, eegFile] const tsv =