From 6926282609c7f6915bdeac62d1f07b2119feb420 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 15 May 2024 22:54:54 +0100 Subject: [PATCH] Override tqdm's default locks --- bio2zarr/core.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bio2zarr/core.py b/bio2zarr/core.py index 59b45fe..00f42e3 100644 --- a/bio2zarr/core.py +++ b/bio2zarr/core.py @@ -20,6 +20,15 @@ numcodecs.blosc.use_threads = False +# By default Tqdm creates a multiprocessing Lock to synchronise across processes, +# which seems to cause some problems with leaked semaphores on certain combinations +# of Mac and Python versions. We only access tqdm from the main process though, +# so we don't need it and can override with a simpler threading Lock. +# NOTE: this gets set multiple times to different locks as subprocesses are +# spawned, but it doesn't matter because the only tqdm instance that is +# used is the one in the main process. +tqdm.tqdm.set_lock(threading.RLock()) + def display_number(x): ret = "n/a" @@ -248,7 +257,6 @@ def __init__(self, worker_processes=1, progress_config=None): def _update_progress(self): current = get_progress() inc = current - self.progress_bar.n - # print("UPDATE PROGRESS: current = ", current, self.progress_config.total, inc) self.progress_bar.update(inc) def _update_progress_worker(self):