Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

run conversion: Use context manager only in append mode #1180

Merged
merged 21 commits into from
Jan 24, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
improve variable naming
h-mayorquin committed Jan 21, 2025
commit 52080dc99bc81621042e455d08368397735ad751
32 changes: 18 additions & 14 deletions src/neuroconv/basedatainterface.py
Original file line number Diff line number Diff line change
@@ -166,11 +166,12 @@ def run_conversion(
Parameters
----------
nwbfile_path : FilePathType
Path for where the data will be written or appended.
Path where the data will be written or appended.
nwbfile : NWBFile, optional
An in-memory NWBFile object to write to the location.
metadata : dict, optional
Metadata dictionary with information used to create the NWBFile when one does not exist or overwrite=True.
Metadata dictionary with information used to create the NWBFile
when one does not exist or overwrite=True.
overwrite : bool, default: False
Whether to overwrite the NWBFile if one exists at the nwbfile_path.
The default is False (append mode).
@@ -180,28 +181,29 @@ def run_conversion(
If a `backend_configuration` is specified, then the type will be auto-detected.
backend_configuration : HDF5BackendConfiguration or ZarrBackendConfiguration, optional
The configuration model to use when configuring the datasets for this backend.
To customize, call the `.get_default_backend_configuration(...)` method, modify the returned
BackendConfiguration object, and pass that instead.
To customize, call the `.get_default_backend_configuration(...)` method,
modify the returned BackendConfiguration object, and pass that instead.
Otherwise, all datasets will use default configuration settings.
"""

backend = _resolve_backend(backend, backend_configuration)
no_nwbfile_provided = nwbfile is None # Otherwise, variable reference may mutate later on inside the context
# Check if we're appending to an in-memory NWBFile
appending_to_in_memory_nwbfile = nwbfile is not None

if metadata is None:
metadata = self.get_metadata()

# Determine whether we're appending to a file on disk
file_initially_exists = Path(nwbfile_path).exists() if nwbfile_path is not None else False
append_mode = file_initially_exists and not overwrite
appending_to_disk_nwbfile = file_initially_exists and not overwrite

self.validate_metadata(metadata=metadata, append_mode=append_mode)
self.validate_metadata(metadata=metadata, append_mode=appending_to_disk_nwbfile)

if not append_mode:

if no_nwbfile_provided:
nwbfile = self.create_nwbfile(metadata=metadata, **conversion_options)
else:
# If we're NOT appending to an existing file on disk (i.e., new file or overwrite=True)
if not appending_to_disk_nwbfile:
if appending_to_in_memory_nwbfile:
self.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, **conversion_options)
else:
nwbfile = self.create_nwbfile(metadata=metadata, **conversion_options)

configure_and_write_nwbfile(
nwbfile=nwbfile,
@@ -211,6 +213,8 @@ def run_conversion(
)

else: # We are only using the context in append mode, see #1143
backend = _resolve_backend(backend, backend_configuration)

with make_or_load_nwbfile(
nwbfile_path=nwbfile_path,
nwbfile=nwbfile,
@@ -219,7 +223,7 @@ def run_conversion(
backend=backend,
verbose=getattr(self, "verbose", False),
) as nwbfile_out:
if no_nwbfile_provided:
if not appending_to_in_memory_nwbfile:
self.add_to_nwbfile(nwbfile=nwbfile_out, metadata=metadata, **conversion_options)

if backend_configuration is None:
25 changes: 13 additions & 12 deletions src/neuroconv/nwbconverter.py
Original file line number Diff line number Diff line change
@@ -244,26 +244,24 @@ def run_conversion(
" use Converter.add_to_nwbfile."
)

backend = _resolve_backend(backend, backend_configuration)
no_nwbfile_provided = nwbfile is None # Otherwise, variable reference may mutate later on inside the context

file_initially_exists = Path(nwbfile_path).exists() if nwbfile_path is not None else False
append_mode = file_initially_exists and not overwrite
appending_to_in_memory_nwbfile = nwbfile is not None

if metadata is None:
metadata = self.get_metadata()

self.validate_metadata(metadata=metadata, append_mode=append_mode)
file_initially_exists = Path(nwbfile_path).exists() if nwbfile_path is not None else False
appending_to_disk_nwbfile = file_initially_exists and not overwrite
self.validate_metadata(metadata=metadata, append_mode=appending_to_disk_nwbfile)
self.validate_conversion_options(conversion_options=conversion_options)

self.temporally_align_data_interfaces(metadata=metadata, conversion_options=conversion_options)

if not append_mode:
if not appending_to_disk_nwbfile:

if no_nwbfile_provided:
nwbfile = self.create_nwbfile(metadata=metadata, conversion_options=conversion_options)
else:
if appending_to_in_memory_nwbfile:
self.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, conversion_options=conversion_options)
else:
nwbfile = self.create_nwbfile(metadata=metadata, conversion_options=conversion_options)

configure_and_write_nwbfile(
nwbfile=nwbfile,
@@ -272,7 +270,10 @@ def run_conversion(
backend_configuration=backend_configuration,
)

else: # We are only using the context in append mode, see #1143
else: # We are only using the context in append mode, see issue #1143

backend = _resolve_backend(backend, backend_configuration)

with make_or_load_nwbfile(
nwbfile_path=nwbfile_path,
nwbfile=nwbfile,
@@ -281,7 +282,7 @@ def run_conversion(
backend=backend,
verbose=getattr(self, "verbose", False),
) as nwbfile_out:
if no_nwbfile_provided:
if not appending_to_in_memory_nwbfile:
self.add_to_nwbfile(nwbfile=nwbfile_out, metadata=metadata, conversion_options=conversion_options)

if backend_configuration is None: