diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index 0158777..d6df28f 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -7,36 +7,6 @@ stages: pool: vmImage: ubuntu-20.04 steps: - # Run syntax validation using oldest and latest Python - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.8 - - - bash: python .azure-pipelines/syntax-validation.py - displayName: Syntax validation (3.7) - - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.11 - - - bash: python .azure-pipelines/syntax-validation.py - displayName: Syntax validation (3.10) - - - bash: | - set -eux - pip install --disable-pip-version-check ruff - python .azure-pipelines/ruff-validation.py - displayName: Ruff validation - - - bash: | - set -eux - # install versions matching the ones in the corresponding pre-commit hook - pip install --disable-pip-version-check mypy==1.9.0 types-PyYAML==6.0.12 types-requests==2.31.0 - mypy --no-strict-optional dials_data/ - displayName: Type checking - # Set up constants for further build steps - bash: | echo "##vso[task.setvariable variable=BUILD_REPOSITORY_NAME;isOutput=true]${BUILD_REPOSITORY_NAME}" @@ -157,14 +127,15 @@ stages: vmImage: ubuntu-20.04 strategy: matrix: - python38: - PYTHON_VERSION: 3.8 python39: PYTHON_VERSION: 3.9 python310: PYTHON_VERSION: 3.10 python311: PYTHON_VERSION: 3.11 + python312: + PYTHON_VERSION: 3.12 + steps: - template: ci.yml - job: macOS @@ -172,14 +143,15 @@ stages: vmImage: macOS-latest strategy: matrix: - python38: - PYTHON_VERSION: 3.8 python39: PYTHON_VERSION: 3.9 python310: PYTHON_VERSION: 3.10 python311: PYTHON_VERSION: 3.11 + python312: + PYTHON_VERSION: 3.12 + steps: - template: ci.yml - job: windows @@ -187,14 +159,15 @@ stages: vmImage: windows-latest strategy: matrix: - python38: - PYTHON_VERSION: 3.8 python39: PYTHON_VERSION: 3.9 python310: PYTHON_VERSION: 3.10 python311: PYTHON_VERSION: 3.11 + python312: + PYTHON_VERSION: 3.12 + steps: - template: ci-windows.yml @@ -205,7 +178,7 @@ stages: - tests condition: and(succeeded(), eq(dependencies.static.outputs['checks.constants.BUILD_REPOSITORY_NAME'], 'dials/data'), - eq(dependencies.static.outputs['checks.constants.BUILD_SOURCEBRANCH'], 'refs/heads/master')) + eq(dependencies.static.outputs['checks.constants.BUILD_SOURCEBRANCH'], 'refs/heads/main')) jobs: - job: pypi displayName: Publish pypi release @@ -247,7 +220,7 @@ stages: - build - tests condition: and(succeeded(), - eq(dependencies.static.outputs['checks.constants.BUILD_SOURCEBRANCH'], 'refs/heads/master'), + eq(dependencies.static.outputs['checks.constants.BUILD_SOURCEBRANCH'], 'refs/heads/main'), eq(dependencies.static.outputs['checks.constants.BUILD_REPOSITORY_NAME'], 'dials/data')) # only run this job in the main branch of the main repository jobs: diff --git a/.azure-pipelines/create-hashinfo-pull-requests b/.azure-pipelines/create-hashinfo-pull-requests index ce81773..8ea5650 100755 --- a/.azure-pipelines/create-hashinfo-pull-requests +++ b/.azure-pipelines/create-hashinfo-pull-requests @@ -17,7 +17,7 @@ echo echo "##[section]Open pull requests:" declare -A UPDATEBRANCHES -for BRANCH in $(hub pr list -b master -f '%H%n'); do +for BRANCH in $(hub pr list -b main -f '%H%n'); do echo ${BRANCH} UPDATEBRANCHES[${BRANCH}]=1 done diff --git a/.azure-pipelines/ruff-validation.py b/.azure-pipelines/ruff-validation.py deleted file mode 100644 index 907217b..0000000 --- a/.azure-pipelines/ruff-validation.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -import subprocess -import json - -failures = 0 -try: - output = subprocess.run( - ["ruff", "check", "--exit-zero", "--output-format", "json", "."], - capture_output=True, - check=True, - timeout=300, - ) -except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - print( - "##vso[task.logissue type=error;]ruff validation failed with", - str(e.__class__.__name__), - ) - print(e.stdout) - print(e.stderr) - print("##vso[task.complete result=Failed;]ruff validation failed") - exit() - -results = json.loads(output.stdout) - -for violation in results: - # filename, lineno, column, error = line.split(":", maxsplit=3) - # errcode, error = error.strip().split(" ", maxsplit=1) - # filename = os.path.normpath(filename) - failures += 1 - print( - f"##vso[task.logissue type=error;sourcepath={violation['filename']};" - f"linenumber={violation['location']['row']};columnnumber={violation['location']['column']};code={violation['code']};]" - + violation["message"] - ) - -if failures: - print(f"##vso[task.logissue type=warning]Found {failures} ruff violation(s)") - print(f"##vso[task.complete result=Failed;]Found {failures} ruff violation(s)") diff --git a/.azure-pipelines/syntax-validation.py b/.azure-pipelines/syntax-validation.py deleted file mode 100644 index 2d74948..0000000 --- a/.azure-pipelines/syntax-validation.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import ast -import os -import sys - -print("Python", sys.version, "\n") - -failures = 0 - -for base, _, files in os.walk("."): - for f in files: - if not f.endswith(".py"): - continue - filename = os.path.normpath(os.path.join(base, f)) - try: - with open(filename) as fh: - ast.parse(fh.read()) - except SyntaxError as se: - failures += 1 - print( - f"##vso[task.logissue type=error;sourcepath={filename};" - f"linenumber={se.lineno};columnnumber={se.offset};]" - f"SyntaxError: {se.msg}" - ) - print(" " + se.text + " " * se.offset + "^") - print(f"SyntaxError: {se.msg} in {filename} line {se.lineno}") - print() - -if failures: - print(f"##vso[task.logissue type=warning]Found {failures} syntax error(s)") - print(f"##vso[task.complete result=Failed;]Found {failures} syntax error(s)") diff --git a/.codecov.yml b/.codecov.yml index 7c8e50e..2097f9d 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -12,5 +12,5 @@ coverage: comment: layout: "diff, flags" branches: - - master + - main after_n_builds: 3 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 641322d..2151c42 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,9 +2,9 @@ name: "CodeQL" on: push: - branches: [ "master" ] + branches: ["main"] pull_request: - branches: [ "master" ] + branches: ["main"] schedule: - cron: "22 2 * * 3" @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - language: [ javascript, python ] + language: [javascript, python] steps: - name: Checkout diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a150b05..9cef1d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,11 @@ +ci: + autoupdate_schedule: quarterly + skip: [pip-compile] + repos: # Syntax validation and some basic sanity checks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: check-merge-conflict - id: check-ast @@ -12,8 +16,8 @@ repos: - id: check-yaml # Linting, sorting and formatting -- repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.3.3 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.2 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -23,8 +27,21 @@ repos: # Remember to change versions in .azure-pipelines/azure-pipelines.yml to match # the versions here. - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.13.0 hooks: - id: mypy files: 'dials_data/.*\.py$' additional_dependencies: ['types-PyYAML==6.0.12', 'types-requests==2.31.0'] + +- repo: https://github.com/astral-sh/uv-pre-commit + rev: 0.4.30 + hooks: + - id: pip-compile + args: [pyproject.toml, -q, -o, requirements.txt] + files: ^(pyproject.toml|requirements(_dev|_doc)?.txt)$ + - id: pip-compile + args: [pyproject.toml, -q, --extra, test, -o, requirements_dev.txt] + files: ^(pyproject.toml|requirements(_dev|_doc)?.txt)$ + - id: pip-compile + args: [pyproject.toml, -q, --extra, doc, -o, requirements_doc.txt] + files: ^(pyproject.toml|requirements(_dev|_doc)?.txt)$ diff --git a/dials_data/cli.py b/dials_data/cli.py index eb75a69..6dd5f96 100644 --- a/dials_data/cli.py +++ b/dials_data/cli.py @@ -119,7 +119,7 @@ def main(): version = dials_data.__version__ + "-dev" parser = argparse.ArgumentParser( usage="dials.data []", - description="""DIALS regression data manager v{version} + description=f"""DIALS regression data manager v{version} The most commonly used commands are: list List available datasets @@ -128,7 +128,7 @@ def main(): Each command has its own set of parameters, and you can get more information by running dials.data --help -""".format(version=version), +""", formatter_class=argparse.RawTextHelpFormatter, ) parser.add_argument("subcommand", help=argparse.SUPPRESS) diff --git a/dials_data/datasets.py b/dials_data/datasets.py index 2fe193a..42549ab 100644 --- a/dials_data/datasets.py +++ b/dials_data/datasets.py @@ -134,11 +134,7 @@ def list_known_definitions(ds_list, quiet=False) -> None: size_information = "unverified dataset" else: size_information = _human_readable(get_resident_size(shortname)) - print( - "{shortname}: {dataset[name]} ({size_information})".format( - shortname=shortname, dataset=dataset, size_information=size_information - ) - ) + print(f"{shortname}: {dataset['name']} ({size_information})") print( "{indent}{author} ({license})".format( author=dataset.get("author", "unknown author"), @@ -147,7 +143,7 @@ def list_known_definitions(ds_list, quiet=False) -> None: ) ) if dataset.get("url"): - print("{indent}{dataset[url]}".format(indent=indent, dataset=dataset)) + print(f"{indent}{dataset['url']}") print( "\n{}\n".format( textwrap.fill( diff --git a/dials_data/download.py b/dials_data/download.py index dcb26c9..15798e6 100644 --- a/dials_data/download.py +++ b/dials_data/download.py @@ -10,7 +10,7 @@ import warnings import zipfile from pathlib import Path -from typing import Any, Optional, Union +from typing import Any from urllib.parse import urlparse import py.path @@ -104,7 +104,7 @@ def fetch_dataset( read_only: bool = False, verbose: bool = False, pre_scan: bool = True, -) -> Union[bool, dict[str, Any]]: +) -> bool | dict[str, Any]: """Check for the presence or integrity of the local copy of the specified test dataset. If the dataset is not available or out of date then attempt to download/update it transparently. @@ -280,7 +280,7 @@ class DataFetcher: """ def __init__(self, read_only: bool = False, verify: bool = True): - self._cache: dict[str, Optional[Path]] = {} + self._cache: dict[str, Path | None] = {} self._target_dir: Path = dials_data.datasets.repository_location() self._read_only: bool = read_only and os.access(self._target_dir, os.W_OK) self._verify: bool = verify @@ -332,7 +332,7 @@ def __call__(self, test_data: str, pathlib=None, **kwargs): return self.result_filter(result=py.path.local(self._cache[test_data])) return self.result_filter(result=self._cache[test_data]) - def _attempt_fetch(self, test_data: str) -> Optional[Path]: + def _attempt_fetch(self, test_data: str) -> Path | None: if self._read_only: hashinfo = fetch_dataset(test_data, pre_scan=True, read_only=True) else: diff --git a/pyproject.toml b/pyproject.toml index 570febd..dec8455 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,13 +23,12 @@ classifiers = [ ] keywords = ["dials", "dials_data"] dynamic = ["readme"] -requires-python = ">=3.8" -dependencies = [ - "importlib_resources>=1.1,!=6.2,!=6.3.0,!=6.3.1", - "pytest", - "pyyaml", - "requests", -] +requires-python = ">=3.9" +dependencies = ["pyyaml", "requests", "py", "importlib_resources~=6.3"] + +[project.optional-dependencies] +test = ["coverage", "pytest"] +doc = ["Sphinx~=6.2", "sphinx_rtd_theme~=1.3"] [project.urls] Homepage = "https://github.com/dials/data" @@ -38,64 +37,18 @@ Documentation = "https://dials-data.readthedocs.io/" "Source Code" = "https://github.com/dials/data" [project.entry-points] -"libtbx.dispatcher.script" = { "dials.data" = "dials.data" } "libtbx.precommit" = { dials_data = "dials_data" } pytest11 = { dials_data = "dials_data.pytest11" } [project.scripts] "dials.data" = "dials_data.cli:main" -[tool.setuptools] -include-package-data = true -zip-safe = false -license-files = ["LICENSE"] - -[tool.setuptools.packages] -find = { namespaces = false } - -[tool.setuptools.package-data] -dials_data = ["py.typed"] - [tool.setuptools.dynamic] readme = { file = ["README.rst", "HISTORY.rst"], content-type = "text/x-rst" } [tool.ruff.lint] -# Black disagrees with flake8 on a few points. Ignore those. -# E203 whitespace before ':' -# E266 too many leading '#' for block comment -# E501 line too long -ignore = ["E203", "E266", "E501"] - -select = [ - "E401", - "E711", - "E712", - "E713", - "E714", - "E721", - "E722", - "F401", - "F402", - "F403", - "F405", - "F541", - "F631", - "F632", - "F633", - "F811", - "F821", - "F822", - "F841", - "F901", - "W191", - "W291", - "W292", - "W293", - "W605", - "C4", -] -# For converting to pyproject/ruff, only duplicate isort. Can add more later. -fixable = ["I"] +ignore = ["E501"] +select = ["E", "F", "I", "C4"] [tool.ruff.lint.isort] required-imports = ["from __future__ import annotations"] diff --git a/requirements.txt b/requirements.txt index b196687..b5f06fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,18 @@ -importlib_resources==6.0.1 +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml -o requirements.txt +certifi==2024.8.30 + # via requests +charset-normalizer==3.4.0 + # via requests +idna==3.10 + # via requests +importlib-resources==6.4.5 + # via dials-data (pyproject.toml) py==1.11.0 -pytest==7.4.2 + # via dials-data (pyproject.toml) pyyaml==6.0.1 + # via dials-data (pyproject.toml) requests==2.31.0 + # via dials-data (pyproject.toml) +urllib3==2.2.3 + # via requests diff --git a/requirements_dev.txt b/requirements_dev.txt index e947538..ff4db0a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,7 +1,28 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml --extra test -o requirements_dev.txt +certifi==2024.8.30 + # via requests +charset-normalizer==3.4.0 + # via requests coverage==7.3.1 -importlib_resources==6.3.2 + # via dials-data (pyproject.toml) +idna==3.10 + # via requests +importlib-resources==6.4.5 + # via dials-data (pyproject.toml) +iniconfig==2.0.0 + # via pytest +packaging==24.1 + # via pytest +pluggy==1.5.0 + # via pytest py==1.11.0 + # via dials-data (pyproject.toml) pytest==7.4.2 + # via dials-data (pyproject.toml) pyyaml==6.0.1 + # via dials-data (pyproject.toml) requests==2.31.0 -wheel==0.41.2 + # via dials-data (pyproject.toml) +urllib3==2.2.3 + # via requests diff --git a/requirements_doc.txt b/requirements_doc.txt index bcdb491..6908463 100644 --- a/requirements_doc.txt +++ b/requirements_doc.txt @@ -1,5 +1,61 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml --extra doc -o requirements_doc.txt +alabaster==0.7.16 + # via sphinx +babel==2.16.0 + # via sphinx +certifi==2024.8.30 + # via requests +charset-normalizer==3.4.0 + # via requests +docutils==0.18.1 + # via + # sphinx + # sphinx-rtd-theme +idna==3.10 + # via requests +imagesize==1.4.1 + # via sphinx +importlib-resources==6.4.5 + # via dials-data (pyproject.toml) +jinja2==3.1.4 + # via sphinx +markupsafe==3.0.2 + # via jinja2 +packaging==24.1 + # via sphinx py==1.11.0 -pytest==7.4.2 + # via dials-data (pyproject.toml) +pygments==2.18.0 + # via sphinx pyyaml==6.0.1 -Sphinx==6.2.1 -sphinx_rtd_theme==1.3.0 + # via dials-data (pyproject.toml) +requests==2.32.3 + # via + # dials-data (pyproject.toml) + # sphinx +snowballstemmer==2.2.0 + # via sphinx +sphinx==6.2.1 + # via + # dials-data (pyproject.toml) + # sphinx-rtd-theme + # sphinxcontrib-jquery +sphinx-rtd-theme==1.3.0 + # via dials-data (pyproject.toml) +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +urllib3==2.2.3 + # via requests