From 174c0c5bd759e19914db0d190e5a82f78f9ea8ba Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 4 Jun 2024 17:44:40 -0700 Subject: [PATCH 1/6] Allow validating timezone with Inspector calls --- .../pages/guided-mode/data/GuidedMetadata.js | 2 +- .../pages/guided-mode/setup/GuidedSubjects.js | 2 +- .../frontend/core/validation/index.js | 9 +++++- .../manageNeuroconv/manage_neuroconv.py | 28 +++++++++++++++---- src/pyflask/namespaces/neuroconv.py | 3 +- 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/electron/frontend/core/components/pages/guided-mode/data/GuidedMetadata.js b/src/electron/frontend/core/components/pages/guided-mode/data/GuidedMetadata.js index 724327498b..f1deca9a48 100644 --- a/src/electron/frontend/core/components/pages/guided-mode/data/GuidedMetadata.js +++ b/src/electron/frontend/core/components/pages/guided-mode/data/GuidedMetadata.js @@ -333,7 +333,7 @@ export class GuidedMetadataPage extends ManagedPage { onUpdate: () => (this.unsavedUpdates = "conversions"), - validateOnChange, + validateOnChange: (...args) => validateOnChange.call(this, ...args), onlyRequired: false, onStatusChange: (state) => this.manager.updateState(`sub-${subject}/ses-${session}`, state), diff --git a/src/electron/frontend/core/components/pages/guided-mode/setup/GuidedSubjects.js b/src/electron/frontend/core/components/pages/guided-mode/setup/GuidedSubjects.js index 7d1d516773..e385b843f1 100644 --- a/src/electron/frontend/core/components/pages/guided-mode/setup/GuidedSubjects.js +++ b/src/electron/frontend/core/components/pages/guided-mode/setup/GuidedSubjects.js @@ -111,7 +111,7 @@ export class GuidedSubjectsPage extends Page { schema, formProps: { validateOnChange: (localPath, parent, path) => { - return validateOnChange(localPath, parent, ["Subject", ...path]); + return validateOnChange.call(this, localPath, parent, ["Subject", ...path]); }, }, })); diff --git a/src/electron/frontend/core/validation/index.js b/src/electron/frontend/core/validation/index.js index d21e1bb24e..dc8c648dd6 100644 --- a/src/electron/frontend/core/validation/index.js +++ b/src/electron/frontend/core/validation/index.js @@ -9,7 +9,12 @@ export function getMessageType(item) { return item.type ?? (isErrorImportance.includes(item.importance) ? "error" : "warning"); } -export async function validateOnChange(name, parent, path, value) { +export async function validateOnChange( + name, + parent, + path, + value +) { let functions = []; const fullPath = [...path, name]; @@ -63,10 +68,12 @@ export async function validateOnChange(name, parent, path, value) { return func.call(this, name, copy, path, value); // Can specify alternative client-side validation } else { const resolvedFunctionName = func.replace(`{*}`, `${name}`); + return fetch(`${baseUrl}/neuroconv/validate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ + timezone: this.workflow?.timezone?.value, parent: copy, function_name: resolvedFunctionName, }), diff --git a/src/pyflask/manageNeuroconv/manage_neuroconv.py b/src/pyflask/manageNeuroconv/manage_neuroconv.py index cd23882eb9..3b3b06dece 100644 --- a/src/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/src/pyflask/manageNeuroconv/manage_neuroconv.py @@ -641,7 +641,9 @@ def run_check_function(check_function: callable, arg: dict) -> dict: def validate_subject_metadata( - subject_metadata: dict, check_function_name: str + subject_metadata: dict, + check_function_name: str, + timezone: Optional[str] = None ): # -> Union[None, InspectorMessage, List[InspectorMessage]]: """Function used to validate subject metadata.""" from pynwb.file import Subject @@ -650,25 +652,40 @@ def validate_subject_metadata( if isinstance(subject_metadata.get("date_of_birth"), str): subject_metadata["date_of_birth"] = datetime.fromisoformat(subject_metadata["date_of_birth"]) + if timezone is not None: + subject_metadata["date_of_birth"] = subject_metadata["date_of_birth"].astimezone(pytz.timezone(timezone)) + + return run_check_function(check_function, Subject(**subject_metadata)) def validate_nwbfile_metadata( - nwbfile_metadata: dict, check_function_name: str + nwbfile_metadata: dict, + check_function_name: str, + timezone: Optional[str] = None ): # -> Union[None, InspectorMessage, List[InspectorMessage]]: """Function used to validate NWBFile metadata.""" from pynwb.testing.mock.file import mock_NWBFile + import pytz check_function = get_check_function(check_function_name) if isinstance(nwbfile_metadata.get("session_start_time"), str): nwbfile_metadata["session_start_time"] = datetime.fromisoformat(nwbfile_metadata["session_start_time"]) + if timezone is not None: + nwbfile_metadata["session_start_time"] = nwbfile_metadata["session_start_time"].replace(tzinfo=pytz.timezone(timezone)) + + # raise ValueError(f"{original}, {nwbfile_metadata['session_start_time']} ({timezone})") return run_check_function(check_function, mock_NWBFile(**nwbfile_metadata)) -def validate_metadata(metadata: dict, check_function_name: str) -> dict: +def validate_metadata( + metadata: dict, + check_function_name: str, + timezone: Optional[str] = None, + ) -> dict: """Function used to validate data using an arbitrary NWB Inspector function.""" from nwbinspector.nwbinspector import InspectorOutputJSONEncoder from pynwb.file import NWBFile, Subject @@ -676,9 +693,9 @@ def validate_metadata(metadata: dict, check_function_name: str) -> dict: check_function = get_check_function(check_function_name) if issubclass(check_function.neurodata_type, Subject): - result = validate_subject_metadata(metadata, check_function_name) + result = validate_subject_metadata(metadata, check_function_name, timezone) elif issubclass(check_function.neurodata_type, NWBFile): - result = validate_nwbfile_metadata(metadata, check_function_name) + result = validate_nwbfile_metadata(metadata, check_function_name, timezone) else: raise ValueError( f"Function {check_function_name} with neurodata_type {check_function.neurodata_type} " @@ -996,6 +1013,7 @@ def update_conversion_progress(message): resolved_metadata["NWBFile"]["session_start_time"] = datetime.fromisoformat( resolved_metadata["NWBFile"]["session_start_time"] ).replace(tzinfo=zoneinfo.ZoneInfo(info["timezone"])) + if "date_of_birth" in resolved_metadata["Subject"]: resolved_metadata["Subject"]["date_of_birth"] = datetime.fromisoformat( resolved_metadata["Subject"]["date_of_birth"] diff --git a/src/pyflask/namespaces/neuroconv.py b/src/pyflask/namespaces/neuroconv.py index 24b16f3077..3d37dcc156 100644 --- a/src/pyflask/namespaces/neuroconv.py +++ b/src/pyflask/namespaces/neuroconv.py @@ -96,6 +96,7 @@ def post(self): validate_parser = neuroconv_namespace.parser() validate_parser.add_argument("parent", type=dict, required=True) validate_parser.add_argument("function_name", type=str, required=True) +validate_parser.add_argument("timezone", type=str, required=False) @neuroconv_namespace.route("/validate") @@ -104,7 +105,7 @@ class Validate(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): args = validate_parser.parse_args() - return validate_metadata(args.get("parent"), args.get("function_name")) + return validate_metadata(args.get("parent"), args.get("function_name"), args.get("timezone")) @neuroconv_namespace.route("/upload/project") From 9e25b7df94c53be10ba982fb51da103cad7073ed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 00:46:29 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../frontend/core/validation/index.js | 7 +----- .../manageNeuroconv/manage_neuroconv.py | 24 ++++++++----------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/electron/frontend/core/validation/index.js b/src/electron/frontend/core/validation/index.js index dc8c648dd6..f4fd1b4db8 100644 --- a/src/electron/frontend/core/validation/index.js +++ b/src/electron/frontend/core/validation/index.js @@ -9,12 +9,7 @@ export function getMessageType(item) { return item.type ?? (isErrorImportance.includes(item.importance) ? "error" : "warning"); } -export async function validateOnChange( - name, - parent, - path, - value -) { +export async function validateOnChange(name, parent, path, value) { let functions = []; const fullPath = [...path, name]; diff --git a/src/pyflask/manageNeuroconv/manage_neuroconv.py b/src/pyflask/manageNeuroconv/manage_neuroconv.py index 3b3b06dece..bf6f7e6aa4 100644 --- a/src/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/src/pyflask/manageNeuroconv/manage_neuroconv.py @@ -641,9 +641,7 @@ def run_check_function(check_function: callable, arg: dict) -> dict: def validate_subject_metadata( - subject_metadata: dict, - check_function_name: str, - timezone: Optional[str] = None + subject_metadata: dict, check_function_name: str, timezone: Optional[str] = None ): # -> Union[None, InspectorMessage, List[InspectorMessage]]: """Function used to validate subject metadata.""" from pynwb.file import Subject @@ -655,26 +653,24 @@ def validate_subject_metadata( if timezone is not None: subject_metadata["date_of_birth"] = subject_metadata["date_of_birth"].astimezone(pytz.timezone(timezone)) - - return run_check_function(check_function, Subject(**subject_metadata)) def validate_nwbfile_metadata( - nwbfile_metadata: dict, - check_function_name: str, - timezone: Optional[str] = None + nwbfile_metadata: dict, check_function_name: str, timezone: Optional[str] = None ): # -> Union[None, InspectorMessage, List[InspectorMessage]]: """Function used to validate NWBFile metadata.""" + import pytz from pynwb.testing.mock.file import mock_NWBFile - import pytz check_function = get_check_function(check_function_name) if isinstance(nwbfile_metadata.get("session_start_time"), str): nwbfile_metadata["session_start_time"] = datetime.fromisoformat(nwbfile_metadata["session_start_time"]) if timezone is not None: - nwbfile_metadata["session_start_time"] = nwbfile_metadata["session_start_time"].replace(tzinfo=pytz.timezone(timezone)) + nwbfile_metadata["session_start_time"] = nwbfile_metadata["session_start_time"].replace( + tzinfo=pytz.timezone(timezone) + ) # raise ValueError(f"{original}, {nwbfile_metadata['session_start_time']} ({timezone})") @@ -682,10 +678,10 @@ def validate_nwbfile_metadata( def validate_metadata( - metadata: dict, - check_function_name: str, - timezone: Optional[str] = None, - ) -> dict: + metadata: dict, + check_function_name: str, + timezone: Optional[str] = None, +) -> dict: """Function used to validate data using an arbitrary NWB Inspector function.""" from nwbinspector.nwbinspector import InspectorOutputJSONEncoder from pynwb.file import NWBFile, Subject From a7315a30b3609fe92b287c73356f900634dd577c Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 4 Jun 2024 17:48:25 -0700 Subject: [PATCH 3/6] Fix DOB call --- src/pyflask/manageNeuroconv/manage_neuroconv.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pyflask/manageNeuroconv/manage_neuroconv.py b/src/pyflask/manageNeuroconv/manage_neuroconv.py index 3b3b06dece..7fad64633c 100644 --- a/src/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/src/pyflask/manageNeuroconv/manage_neuroconv.py @@ -647,15 +647,14 @@ def validate_subject_metadata( ): # -> Union[None, InspectorMessage, List[InspectorMessage]]: """Function used to validate subject metadata.""" from pynwb.file import Subject + import pytz check_function = get_check_function(check_function_name) if isinstance(subject_metadata.get("date_of_birth"), str): subject_metadata["date_of_birth"] = datetime.fromisoformat(subject_metadata["date_of_birth"]) if timezone is not None: - subject_metadata["date_of_birth"] = subject_metadata["date_of_birth"].astimezone(pytz.timezone(timezone)) - - + subject_metadata["date_of_birth"] = subject_metadata["date_of_birth"].replace(tzinfo=pytz.timezone(timezone)) return run_check_function(check_function, Subject(**subject_metadata)) @@ -676,8 +675,6 @@ def validate_nwbfile_metadata( if timezone is not None: nwbfile_metadata["session_start_time"] = nwbfile_metadata["session_start_time"].replace(tzinfo=pytz.timezone(timezone)) - # raise ValueError(f"{original}, {nwbfile_metadata['session_start_time']} ({timezone})") - return run_check_function(check_function, mock_NWBFile(**nwbfile_metadata)) From ae798234add612097229e7637604c7082628408d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 00:48:56 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pyflask/manageNeuroconv/manage_neuroconv.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pyflask/manageNeuroconv/manage_neuroconv.py b/src/pyflask/manageNeuroconv/manage_neuroconv.py index bda1a4fac2..f875874459 100644 --- a/src/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/src/pyflask/manageNeuroconv/manage_neuroconv.py @@ -644,15 +644,17 @@ def validate_subject_metadata( subject_metadata: dict, check_function_name: str, timezone: Optional[str] = None ): # -> Union[None, InspectorMessage, List[InspectorMessage]]: """Function used to validate subject metadata.""" + import pytz from pynwb.file import Subject - import pytz check_function = get_check_function(check_function_name) if isinstance(subject_metadata.get("date_of_birth"), str): subject_metadata["date_of_birth"] = datetime.fromisoformat(subject_metadata["date_of_birth"]) if timezone is not None: - subject_metadata["date_of_birth"] = subject_metadata["date_of_birth"].replace(tzinfo=pytz.timezone(timezone)) + subject_metadata["date_of_birth"] = subject_metadata["date_of_birth"].replace( + tzinfo=pytz.timezone(timezone) + ) return run_check_function(check_function, Subject(**subject_metadata)) From d9c6a922cefd0af20f23e35b86bcd6bef8144769 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 4 Jun 2024 17:50:39 -0700 Subject: [PATCH 5/6] Loosen the max date --- src/schemas/base-metadata.schema.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/schemas/base-metadata.schema.ts b/src/schemas/base-metadata.schema.ts index dc17ce9820..466993ebe3 100644 --- a/src/schemas/base-metadata.schema.ts +++ b/src/schemas/base-metadata.schema.ts @@ -96,7 +96,11 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa copy.order = [ "NWBFile", "Subject" ] const minDate = "1900-01-01T00:00" - const maxDate = getISODateInTimezone().slice(0, -2) // Restrict date to current date with timezone awareness + + // Set the maximum at tomorrow + const nextDay = new Date() + nextDay.setDate(nextDay.getDate() + 1) + const maxDate = getISODateInTimezone(nextDay).slice(0, -2) // Restrict date to tomorrow (with timezone awareness) // Add unit to weight From 3793a71dde5df511e87230bbab92854c88e679ea Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 4 Jun 2024 17:55:48 -0700 Subject: [PATCH 6/6] Add back a filtered timezone if used (e.g. in Actions) --- src/schemas/timezone.schema.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/schemas/timezone.schema.ts b/src/schemas/timezone.schema.ts index 445a66e4cf..16ed1c6569 100644 --- a/src/schemas/timezone.schema.ts +++ b/src/schemas/timezone.schema.ts @@ -99,6 +99,9 @@ ready.timezones.then((timezones) => { && !tz.toLowerCase().includes('etc/') }); + if (!filteredTimezones.includes(timezone)) filteredTimezones.push(timezone) // Add the local timezone if it's not in the list + + timezoneSchema.enumLabels = filteredTimezones.reduce((acc, tz) => { const [ _, ...other ] = tz.split('/') acc[tz] = other.map(part => header(part)).join(' — ')