Consolidate config to pyproject only, and update packaging (#453)

- Now uses ruff for isort, formatting and linting
- Everything merged down to pyproject.toml
- Updated CI to use new tooling

.azure-pipelines/azure-pipelines.yml

@@ -26,14 +26,14 @@ stages:
 
       - bash: |
           set -eux
-          pip install --disable-pip-version-check flake8 flake8-comprehensions
-          python .azure-pipelines/flake8-validation.py
-        displayName: Flake8 validation
+          pip install --disable-pip-version-check ruff
+          python .azure-pipelines/ruff-validation.py
+        displayName: Ruff validation
 
       - bash: |
           set -eux
           # install versions matching the ones in the corresponding pre-commit hook
-          pip install --disable-pip-version-check mypy==0.931 types-PyYAML==6.0.4 types-requests==2.27.11
+          pip install --disable-pip-version-check mypy==1.9.0 types-PyYAML==6.0.12 types-requests==2.31.0
           mypy --no-strict-optional dials_data/
         displayName: Type checking
 
@@ -85,15 +85,15 @@ stages:
           versionSpec: 3.9
 
       - bash: |
-          pip install --disable-pip-version-check -r requirements_dev.txt
+          pip install --disable-pip-version-check -r requirements_dev.txt build
         displayName: Install dependencies
 
       - bash: .azure-pipelines/update-package-version
         displayName: Set package version
 
       - bash: |
           set -ex
-          python setup.py sdist bdist_wheel
+          python -m build
           mkdir -p dist/pypi
           shopt -s extglob
           mv -v dist/!(pypi) dist/pypi
@@ -107,11 +107,6 @@ stages:
           pathToPublish: dist/
           artifactName: package
 
-      - bash: |
-          twine check dist/pypi/*
-          python setup.py checkdocs
-        displayName: Check package description
-
 - stage: update
   displayName: Update hashes
   dependsOn:

.azure-pipelines/ruff-validation.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import subprocess
+import json
+
+failures = 0
+try:
+    output = subprocess.run(
+        ["ruff", "check", "--exit-zero", "--output-format", "json", "."],
+        capture_output=True,
+        check=True,
+        timeout=300,
+    )
+except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+    print(
+        "##vso[task.logissue type=error;]ruff validation failed with",
+        str(e.__class__.__name__),
+    )
+    print(e.stdout)
+    print(e.stderr)
+    print("##vso[task.complete result=Failed;]ruff validation failed")
+    exit()
+
+results = json.loads(output.stdout)
+
+for violation in results:
+    # filename, lineno, column, error = line.split(":", maxsplit=3)
+    # errcode, error = error.strip().split(" ", maxsplit=1)
+    # filename = os.path.normpath(filename)
+    failures += 1
+    print(
+        f"##vso[task.logissue type=error;sourcepath={violation['filename']};"
+        f"linenumber={violation['location']['row']};columnnumber={violation['location']['column']};code={violation['code']};]"
+        + violation["message"]
+    )
+
+if failures:
+    print(f"##vso[task.logissue type=warning]Found {failures} ruff violation(s)")
+    print(f"##vso[task.complete result=Failed;]Found {failures} ruff violation(s)")

.azure-pipelines/update-package-version

@@ -19,7 +19,7 @@ DEPTH=$(git rev-list ${BASETAG}..HEAD --count --first-parent)
 REAL_VERSION=${VERSION%.0}.${DEPTH}
 echo Setting package version from ${VERSION} to ${REAL_VERSION}
 sed -i "s/^__version__ =.*/__version__ = \"${REAL_VERSION}\"/" dials_data/__init__.py
-sed -i "s/^version = .*$/version = ${REAL_VERSION}/" setup.cfg
+sed -i "s/^version = .*$/version = \"${REAL_VERSION}\"/" pyproject.toml
 
 COMMIT=$(git rev-parse --verify HEAD)
 echo Setting package commit to ${COMMIT}

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
 # Syntax validation and some basic sanity checks
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: check-merge-conflict
   - id: check-ast
@@ -11,43 +11,20 @@ repos:
     args: ['--maxkb=200']
   - id: check-yaml
 
-# Automatically sort imports
-- repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
+# Linting, sorting and formatting
+- repo: https://github.com/charliermarsh/ruff-pre-commit
+  rev: v0.3.3
   hooks:
-  - id: isort
-    args: [
-           '-a', 'from __future__ import annotations',        # 3.7-3.11
-           '--rm', 'from __future__ import absolute_import',  # -3.0
-           '--rm', 'from __future__ import division',         # -3.0
-           '--rm', 'from __future__ import generator_stop',   # -3.7
-           '--rm', 'from __future__ import generators',       # -2.3
-           '--rm', 'from __future__ import nested_scopes',    # -2.2
-           '--rm', 'from __future__ import print_function',   # -3.0
-           '--rm', 'from __future__ import unicode_literals', # -3.0
-           '--rm', 'from __future__ import with_statement',   # -2.6
-          ]
-
-# Automatic source code formatting
-- repo: https://github.com/psf/black
-  rev: 23.7.0
-  hooks:
-  - id: black
-    args: [--safe, --quiet]
-
-# Linting
-- repo: https://github.com/PyCQA/flake8
-  rev: 6.1.0
-  hooks:
-  - id: flake8
-    additional_dependencies: ['flake8-comprehensions==3.8.0']
+    - id: ruff
+      args: [--fix, --exit-non-zero-on-fix]
+    - id: ruff-format
 
 # Type checking
 # Remember to change versions in .azure-pipelines/azure-pipelines.yml to match
 # the versions here.
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.5.0
+  rev: v1.9.0
   hooks:
   - id: mypy
     files: 'dials_data/.*\.py$'
-    additional_dependencies: ['types-PyYAML==6.0.4', 'types-requests==2.27.11']
+    additional_dependencies: ['types-PyYAML==6.0.12', 'types-requests==2.31.0']

dials_data/cli.py

@@ -128,9 +128,7 @@ def main():
 Each command has its own set of parameters, and you can get more information
 by running dials.data <command> --help
 
-""".format(
-            version=version
-        ),
+""".format(version=version),
         formatter_class=argparse.RawTextHelpFormatter,
     )
     parser.add_argument("subcommand", help=argparse.SUPPRESS)

dials_data/pytest11.py

@@ -2,7 +2,6 @@
 pytest plugin functions
 """
 
-
 from __future__ import annotations
 
 import pytest

pyproject.toml

@@ -1,7 +1,100 @@
 [build-system]
-requires = ["setuptools >= 40.6.0", "wheel"]
+requires = ["setuptools>=61.2"]
 build-backend = "setuptools.build_meta"
 
+[project]
+name = "dials_data"
+version = "2.4.0"
+description = "DIALS Regression Data Manager"
+authors = [
+    { name = "DIALS development team", email = "[email protected]" },
+]
+license = { text = "BSD 3-Clause License" }
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: BSD License",
+    "Natural Language :: English",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+keywords = ["dials", "dials_data"]
+dynamic = ["readme"]
+requires-python = ">=3.8"
+dependencies = ["importlib_resources>=1.1,<6.2", "pytest", "pyyaml", "requests"]
+
+[project.urls]
+Homepage = "https://github.com/dials/data"
+"Bug Tracker" = "https://github.com/dials/data/issues"
+Documentation = "https://dials-data.readthedocs.io/"
+"Source Code" = "https://github.com/dials/data"
+
+[project.entry-points]
+"libtbx.dispatcher.script" = { "dials.data" = "dials.data" }
+"libtbx.precommit" = { dials_data = "dials_data" }
+pytest11 = { dials_data = "dials_data.pytest11" }
+
+[project.scripts]
+"dials.data" = "dials_data.cli:main"
+
+[tool.setuptools]
+include-package-data = true
+zip-safe = false
+license-files = ["LICENSE"]
+
+[tool.setuptools.packages]
+find = { namespaces = false }
+
+[tool.setuptools.package-data]
+dials_data = ["py.typed"]
+
+[tool.setuptools.dynamic]
+readme = { file = ["README.rst", "HISTORY.rst"], content-type = "text/x-rst" }
+
+[tool.ruff.lint]
+# Black disagrees with flake8 on a few points. Ignore those.
+# E203 whitespace before ':'
+# E266 too many leading '#' for block comment
+# E501 line too long
+ignore = ["E203", "E266", "E501"]
+
+select = [
+    "E401",
+    "E711",
+    "E712",
+    "E713",
+    "E714",
+    "E721",
+    "E722",
+    "F401",
+    "F402",
+    "F403",
+    "F405",
+    "F541",
+    "F631",
+    "F632",
+    "F633",
+    "F811",
+    "F821",
+    "F822",
+    "F841",
+    "F901",
+    "W191",
+    "W291",
+    "W292",
+    "W293",
+    "W605",
+    "C4",
+]
+# For converting to pyproject/ruff, only duplicate isort. Can add more later.
+fixable = ["I"]
+
+[tool.ruff.lint.isort]
+required-imports = ["from __future__ import annotations"]
+
 [[tool.mypy.overrides]]
-module = [ "py", "py.path", "pytest", "importlib_resources" ]
+module = ["py", "py.path", "pytest", "importlib_resources"]
 ignore_missing_imports = true

requirements_dev.txt

@@ -1,4 +1,3 @@
-collective.checkdocs==0.2
 coverage==7.3.1
 importlib_resources==6.0.1
 py==1.11.0