Migrate to pydantic version 2

pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
 dependencies = [
     "backports.entry_points_selectable",
     "defusedxml", # For safely parsing XML files
-    "pydantic<2", # Locked to <2 by cygwin terminal
+    "pydantic>=2",
     "requests",
     "rich",
     "werkzeug",
@@ -47,7 +47,7 @@ client = [
     "websocket-client",
 ]
 developer = [
-    "bump-my-version<0.11.0", # Version control
+    "bump-my-version", # Version control
     "ipykernel", # Enable interactive coding with VS Code and Jupyter Notebook
     "pre-commit", # Formatting, linting, type checking, etc.
     "pytest", # Test code functionality
@@ -61,7 +61,7 @@ server = [
     "aiohttp",
     "cryptography",
     "fastapi[standard]",
-    "ispyb", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python; v10.0.0: sqlalchemy <2, mysql-connector-python >=8.0.32
+    "ispyb>=10.2.4", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python;
     "jinja2",
     "mrcfile",
     "numpy",
@@ -74,7 +74,7 @@ server = [
     "sqlmodel",
     "stomp-py<=8.1.0", # 8.1.1 (released 2024-04-06) doesn't work with our project
     "uvicorn[standard]",
-    "zocalo",
+    "zocalo>=1",
 ]
 [project.urls]
 Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues"

src/murfey/client/instance_environment.py

@@ -8,7 +8,7 @@
 from typing import Dict, List, NamedTuple, Optional
 from urllib.parse import ParseResult
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from murfey.client.watchdir import DirWatcher
 
@@ -53,9 +53,7 @@ class MurfeyInstanceEnvironment(BaseModel):
     murfey_session: Optional[int] = None
     samples: Dict[Path, SampleInfo] = {}
 
-    class Config:
-        validate_assignment: bool = True
-        arbitrary_types_allowed: bool = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
     def clear(self):
         self.sources = []

src/murfey/client/tui/screens.py

@@ -198,7 +198,7 @@
     try:
         convert = lambda x: None if x == "None" else x
         validated = model(**{k: convert(v) for k, v in form.items()})
-        log.info(validated.dict())
+        log.info(validated.model_dump())
         return True
     except (AttributeError, ValidationError) as e:
         log.warning(f"Form validation failed: {str(e)}")

src/murfey/instrument_server/api.py

@@ -153,7 +153,7 @@ def start_multigrid_watcher(
         demo=True,
         do_transfer=True,
         processing_enabled=not watcher_spec.skip_existing_processing,
-        _machine_config=watcher_spec.configuration.dict(),
+        _machine_config=watcher_spec.configuration.model_dump(),
         token=tokens.get(session_id, "token"),
         data_collection_parameters=data_collection_parameters.get(label, {}),
         rsync_restarts=watcher_spec.rsync_restarts,
@@ -167,7 +167,7 @@ def start_multigrid_watcher(
         (watcher_spec.source / d).mkdir(exist_ok=True)
     watchers[session_id] = MultigridDirWatcher(
         watcher_spec.source,
-        watcher_spec.configuration.dict(),
+        watcher_spec.configuration.model_dump(),
         skip_existing_processing=watcher_spec.skip_existing_processing,
     )
     watchers[session_id].subscribe(
@@ -237,7 +237,7 @@ def register_processing_parameters(
     session_id: MurfeySessionID, proc_param_block: ProcessingParameterBlock
 ):
     data_collection_parameters[proc_param_block.label] = {}
-    for k, v in proc_param_block.params.dict().items():
+    for k, v in proc_param_block.params.model_dump().items():
         data_collection_parameters[proc_param_block.label][k] = v
     return {"success": True}
 

src/murfey/server/api/clem.py

@@ -10,7 +10,7 @@
 
 from backports.entry_points_selectable import entry_points
 from fastapi import APIRouter
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 from sqlalchemy.exc import NoResultFound
 from sqlmodel import Session, select
 
@@ -707,10 +707,8 @@ class AlignAndMergeParams(BaseModel):
     flatten: Literal["mean", "min", "max", ""] = ""
     align_across: Literal["enabled", ""] = ""
 
-    @validator(
-        "images",
-        pre=True,
-    )
+    @field_validator("images", mode="before")
+    @classmethod
     def parse_stringified_list(cls, value):
         if isinstance(value, str):
             try:

src/murfey/server/api/spa.py

@@ -20,12 +20,14 @@ def _cryolo_model_path(visit: str, instrument_name: str) -> Path:
     machine_config = get_machine_config(instrument_name=instrument_name)[
         instrument_name
     ]
-    if machine_config.model_search_directory:
+    if machine_config.picking_model_search_directory:
         visit_directory = (
             machine_config.rsync_basepath / str(datetime.now().year) / visit
         )
         possible_models = list(
-            (visit_directory / machine_config.model_search_directory).glob("*.h5")
+            (visit_directory / machine_config.picking_model_search_directory).glob(
+                "*.h5"
+            )
         )
         if possible_models:
             return sorted(possible_models, key=lambda x: x.stat().st_ctime)[-1]

src/murfey/server/demo_api.py

@@ -15,7 +15,8 @@
 from fastapi.responses import FileResponse, HTMLResponse
 from ispyb.sqlalchemy import BLSession
 from PIL import Image
-from pydantic import BaseModel, BaseSettings
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings
 from sqlalchemy import func
 from sqlmodel import col, select
 from werkzeug.utils import secure_filename

src/murfey/server/main.py

@@ -8,7 +8,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from prometheus_client import make_asgi_app
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings
 
 import murfey.server
 import murfey.server.api.auth

src/murfey/util/config.py

@@ -8,7 +8,8 @@
 
 import yaml
 from backports.entry_points_selectable import entry_points
-from pydantic import BaseModel, BaseSettings, Extra, validator
+from pydantic import BaseModel, Extra, field_validator
+from pydantic_settings import BaseSettings
 
 
 class MachineConfig(BaseModel, extra=Extra.allow):  # type: ignore
@@ -58,7 +59,7 @@ class MachineConfig(BaseModel, extra=Extra.allow):  # type: ignore
     upstream_data_download_directory: Optional[Path] = None  # Set by microscope config
     upstream_data_tiff_locations: List[str] = ["processed"]  # Location of CLEM TIFFs
 
-    model_search_directory: str = "processing"
+    picking_model_search_directory: str = "processing"
     initial_model_search_directory: str = "processing/initial_model"
 
     failure_queue: str = ""
@@ -97,7 +98,7 @@ class Security(BaseModel):
     graylog_host: str = ""
     graylog_port: Optional[int] = None
 
-    @validator("graylog_port")
+    @field_validator("graylog_port")
     def check_port_present_if_host_is(
         cls, v: Optional[int], values: dict, **kwargs
     ) -> Optional[int]:

src/murfey/util/models.py

@@ -97,7 +97,7 @@ class ConnectionFileParameters(BaseModel):
 
 
 class SessionInfo(BaseModel):
-    session_id: Optional[int]
+    session_id: Optional[int] = None
     session_name: str = ""
     rescale: bool = True
 
@@ -160,7 +160,7 @@ class Sample(BaseModel):
     sample_group_id: int
     sample_id: int
     subsample_id: int
-    image_path: Optional[Path]
+    image_path: Optional[Path] = None
 
 
 class BLSampleImageParameters(BaseModel):
@@ -194,13 +194,13 @@ class SPAProcessFile(BaseModel):
     tag: str
     path: str
     description: str
-    processing_job: Optional[int]
-    data_collection_id: Optional[int]
+    processing_job: Optional[int] = None
+    data_collection_id: Optional[int] = None
     image_number: int
-    autoproc_program_id: Optional[int]
-    foil_hole_id: Optional[int]
-    pixel_size: Optional[float]
-    dose_per_frame: Optional[float]
+    autoproc_program_id: Optional[int] = None
+    foil_hole_id: Optional[int] = None
+    pixel_size: Optional[float] = None
+    dose_per_frame: Optional[float] = None
     mc_binning: Optional[int] = 1
     gain_ref: Optional[str] = None
     extract_downscale: bool = True
@@ -211,7 +211,7 @@ class SPAProcessFile(BaseModel):
 class ProcessingParametersSPA(BaseModel):
     tag: str
     dose_per_frame: float
-    gain_ref: Optional[str]
+    gain_ref: Optional[str] = None
     experiment_type: str
     voltage: float
     image_size_x: int
@@ -222,12 +222,12 @@ class ProcessingParametersSPA(BaseModel):
     acquisition_software: str
     use_cryolo: bool
     symmetry: str
-    mask_diameter: Optional[int]
-    boxsize: Optional[int]
+    mask_diameter: Optional[int] = None
+    boxsize: Optional[int] = None
     downscale: bool
-    small_boxsize: Optional[int]
+    small_boxsize: Optional[int] = None
     eer_fractionation: int
-    particle_diameter: Optional[float]
+    particle_diameter: Optional[float] = None
     magnification: Optional[int] = None
     total_exposed_dose: Optional[float] = None
     c2aperture: Optional[float] = None
@@ -236,14 +236,14 @@ class ProcessingParametersSPA(BaseModel):
     phase_plate: bool = False
 
     class Base(BaseModel):
-        dose_per_frame: Optional[float]
-        gain_ref: Optional[str]
+        dose_per_frame: Optional[float] = None
+        gain_ref: Optional[str] = None
         use_cryolo: bool
         symmetry: str
-        mask_diameter: Optional[int]
-        boxsize: Optional[int]
+        mask_diameter: Optional[int] = None
+        boxsize: Optional[int] = None
         downscale: bool
-        small_boxsize: Optional[int]
+        small_boxsize: Optional[int] = None
         eer_fractionation: int
 
 
@@ -350,7 +350,7 @@ class CompletedTiltSeries(BaseModel):
 class PreprocessingParametersTomo(BaseModel):
     dose_per_frame: float
     frame_count: int
-    gain_ref: Optional[str]
+    gain_ref: Optional[str] = None
     experiment_type: str
     voltage: float
     image_size_x: int
@@ -361,12 +361,12 @@ class PreprocessingParametersTomo(BaseModel):
     file_extension: str
     tag: str
     tilt_series_tag: str
-    eer_fractionation_file: Optional[str]
+    eer_fractionation_file: Optional[str] = None
     eer_fractionation: int
 
     class Base(BaseModel):
         dose_per_frame: float
-        gain_ref: Optional[str]
+        gain_ref: Optional[str] = None
         manual_tilt_offset: float
         eer_fractionation: int
 

src/murfey/util/rsync.py

@@ -108,6 +108,7 @@ def _single_rsync(
         else:
             cmd.append(str(self._finaldir / sub_struct) + "/")
         self._transferring = True
+
         runner = subprocess.run(
             cmd,
             capture_output=True,

src/murfey/workflows/clem/register_align_and_merge_results.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 from typing import Optional
 
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 from sqlmodel import Session
 
 from murfey.util.db import CLEMImageSeries
@@ -24,10 +24,8 @@
     align_across: Optional[str] = None
     composite_image: Path
 
-    @validator(
-        "image_stacks",
-        pre=True,
-    )
+    @field_validator("image_stacks", mode="before")
+    @classmethod
     def parse_stringified_list(cls, value):
         if isinstance(value, str):
             try:

src/murfey/workflows/clem/register_preprocessing_results.py

@@ -13,7 +13,7 @@
 from ast import literal_eval
 from pathlib import Path
 
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 from sqlmodel import Session, select
 
 from murfey.server import _transport_object
@@ -221,10 +221,8 @@
     number_of_members: int
     parent_tiffs: list[Path]
 
-    @validator(
-        "parent_tiffs",
-        pre=True,
-    )
+    @field_validator("parent_tiffs", mode="before")
+    @classmethod
     def parse_stringified_list(cls, value):
         if isinstance(value, str):
             try:

tests/cli/test_decrypt_password.py

@@ -13,7 +13,7 @@ def test_decrypt_password(capsys, tmp_path):
     crypto_key = Fernet.generate_key()
     security_config.crypto_key = crypto_key.decode("ascii")
     with open(tmp_path / "config.yaml", "w") as cfg:
-        yaml.dump(security_config.dict(), cfg)
+        yaml.dump(security_config.model_dump(), cfg)
     os.environ["MURFEY_SECURITY_CONFIGURATION"] = str(tmp_path / "config.yaml")
     password = "abcd"
     f = Fernet(crypto_key)

tests/cli/test_generate_password.py

@@ -12,7 +12,7 @@ def test_generate_password(capsys, tmp_path):
     crypto_key = Fernet.generate_key()
     security_config.crypto_key = crypto_key.decode("ascii")
     with open(tmp_path / "config.yaml", "w") as cfg:
-        yaml.dump(security_config.dict(), cfg)
+        yaml.dump(security_config.model_dump(), cfg)
     os.environ["MURFEY_SECURITY_CONFIGURATION"] = str(tmp_path / "config.yaml")
     run()
     captured = capsys.readouterr()