diff --git a/.cruft.json b/.cruft.json index d17bbb3..fdbc99c 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/iterative/py-template", - "commit": "c4e24f909659b6ce9c34a1da631290f0c70ff2f2", + "commit": "e4ec95f4cfd03d4af0a8604d462ee11d07d63b42", "checkout": null, "context": { "cookiecutter": { @@ -14,7 +14,7 @@ "copyright_year": "2022", "license": "Apache-2.0", "docs": "False", - "short_description": "More FSSpec filesystems", + "short_description": "A collection of self-contained fsspec-based filesystems", "development_status": "Development Status :: 4 - Beta", "_template": "https://github.com/iterative/py-template" } diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 56f6f98..393a944 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,12 +7,6 @@ updates: interval: "weekly" labels: - "maintenance" - # Update via cruft - ignore: - - dependency-name: "mkdocs*" - - dependency-name: "pytest*" - - dependency-name: "pylint" - - dependency-name: "mypy" - directory: "/" package-ecosystem: "github-actions" @@ -20,10 +14,3 @@ updates: interval: "weekly" labels: - "maintenance" - # Update via cruft - ignore: - - dependency-name: "actions/checkout" - - dependency-name: "actions/setup-python" - - dependency-name: "pypa/gh-action-pypi-publish" - - dependency-name: "codecov/codecov-action" - - dependency-name: "peter-evans/create-pull-request" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 199e7fe..c0e3d29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,25 +16,25 @@ jobs: id-token: write runs-on: ubuntu-latest steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - name: Check out the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' - - name: Upgrade pip and nox - run: | - pip install --upgrade pip nox - pip --version - nox --version + - name: Upgrade pip and nox + run: | + pip install --upgrade pip nox + pip --version + nox --version - - name: Build package - run: nox -s build + - name: Build package + run: nox -s build - - name: Upload package - if: github.event_name == 'release' - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Upload package + if: github.event_name == 'release' + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5ab3450..83cb153 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,35 +21,35 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, windows-latest, macos-latest] - pyv: ['3.8', '3.9', '3.10', '3.11'] + pyv: ['3.8', '3.9', '3.10', '3.11', '3.12'] include: - - {os: ubuntu-latest, pyv: 'pypy3.8'} + - {os: ubuntu-latest, pyv: 'pypy3.8'} steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - name: Check out the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Set up Python ${{ matrix.pyv }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.pyv }} + - name: Set up Python ${{ matrix.pyv }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.pyv }} - - name: Upgrade pip and nox - run: | - python -m pip install --upgrade pip nox - pip --version - nox --version + - name: Upgrade pip and nox + run: | + python -m pip install --upgrade pip nox + pip --version + nox --version - - name: Lint code and check dependencies - run: nox -s lint safety + - name: Lint code and check dependencies + run: nox -s lint safety - - name: Run tests - run: nox -s tests-${{ matrix.nox_pyv || matrix.pyv }} -- --cov-report=xml + - name: Run tests + run: nox -s tests-${{ matrix.nox_pyv || matrix.pyv }} -- --cov-report=xml - - name: Upload coverage report - uses: codecov/codecov-action@v3.1.1 + - name: Upload coverage report + uses: codecov/codecov-action@v3 - - name: Build package - run: nox -s build + - name: Build package + run: nox -s build diff --git a/.github/workflows/update-template.yaml b/.github/workflows/update-template.yaml index 2fe700d..0b91f76 100644 --- a/.github/workflows/update-template.yaml +++ b/.github/workflows/update-template.yaml @@ -9,8 +9,8 @@ jobs: update: runs-on: ubuntu-latest steps: - - name: Check out the repository - uses: actions/checkout@v3 + - name: Check out the repository + uses: actions/checkout@v4 - - name: Update template - uses: iterative/py-template@main + - name: Update template + uses: iterative/py-template@main diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b226cc4..445c7f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,6 @@ default_language_version: python: python3 repos: - - repo: https://github.com/psf/black - rev: 24.3.0 - hooks: - - id: black - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: @@ -17,38 +13,26 @@ repos: args: ['--assume-in-merge'] - id: check-toml - id: check-yaml - - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending args: ['--fix=lf'] - id: sort-simple-yaml - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.3.4' + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/codespell-project/codespell rev: v2.2.6 hooks: - id: codespell additional_dependencies: ["tomli"] - - repo: https://github.com/asottile/pyupgrade - rev: v3.15.1 - hooks: - - id: pyupgrade - args: [--py38-plus] - - repo: https://github.com/PyCQA/isort - rev: 5.13.2 - hooks: - - id: isort - - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 - hooks: - - id: flake8 - additional_dependencies: - - flake8-bugbear==23.1.20 - - flake8-comprehensions==3.10.1 - - flake8-debugger==4.1.2 - - flake8-string-format==0.3.0 - - repo: https://github.com/pycqa/bandit - rev: 1.7.8 + - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.12.0 hooks: - - id: bandit - args: [-c, pyproject.toml] - additional_dependencies: ["toml"] + - id: pretty-format-toml + args: [--autofix, --no-sort] + - id: pretty-format-yaml + args: [--autofix, --indent, '2', '--offset', '2', --preserve-quotes] diff --git a/noxfile.py b/noxfile.py index d4c4b58..1fa86b2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -10,7 +10,9 @@ locations = "src", "tests" -@nox.session(python=["3.8", "3.9", "3.10", "3.11", "pypy3.8", "pypy3.9"]) +@nox.session( + python=["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.8", "pypy3.9", "pypy3.10"] +) def tests(session: nox.Session) -> None: session.install(".[tests]") session.run( @@ -30,7 +32,6 @@ def lint(session: nox.Session) -> None: args = *(session.posargs or ("--show-diff-on-failure",)), "--all-files" session.run("pre-commit", "run", *args) session.run("python", "-m", "mypy") - session.run("python", "-m", "pylint", "src") @nox.session diff --git a/pyproject.toml b/pyproject.toml index 079f1fb..fbcd53e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,30 +2,70 @@ requires = ["setuptools>=48", "setuptools_scm[toml]>=6.3.1"] build-backend = "setuptools.build_meta" -[tool.setuptools_scm] +[project] +name = "morefs" +description = "A collection of self-contained fsspec-based filesystems" +readme = "README.rst" +license = {text = "Apache-2.0"} +authors = [{name = "Iterative", email = "saugat@iterative.ai"}] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Development Status :: 4 - Beta" +] +requires-python = ">=3.8" +dynamic = ["version"] +dependencies = [ + "fsspec" +] + +[project.urls] +Issues = "https://github.com/iterative/morefs/issues" +Source = "https://github.com/iterative/morefs" + +[project.optional-dependencies] +memfs = [ + "pygtrie>=2.3.2" +] +asynclocalfs = [ + "fsspec>=2022.10.0", + "typing_extensions>=3.10.0; python_version < '3.10'", + "aiofile>=3.8.5,<4" +] +asynclocal = [ + "morefs[asynclocalfs]" +] +all = [ + "morefs[memfs,asynclocalfs]" +] +tests = [ + "morefs[all]", + "pytest>=7,<9", + "pytest-asyncio>=0.21.0,<1", + "pytest-cov>=4.1.0", + "pytest-mock", + "pytest-sugar" +] +dev = [ + "morefs[tests,all]", + "mypy==1.9.0" +] -[tool.black] -line-length = 88 -include = '\.pyi?$' -exclude = ''' -/( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist -)/ -''' - -[tool.isort] -profile = "black" -known_first_party = ["morefs"] -line_length = 88 +[project.entry-points."fsspec.specs"] +memfs = "morefs.memory:MemFS" +dictfs = "morefs.dict:DictFS" +overlayfs = "morefs.overlay:OverlayFileSystem" +asynclocal = "morefs.asyn_local:AsyncLocalFileSystem" + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = false + +[tool.setuptools_scm] [tool.pytest.ini_options] addopts = "-ra" @@ -40,13 +80,13 @@ source = ["src", "*/site-packages"] [tool.coverage.report] show_missing = true exclude_lines = [ - "pragma: no cover", - "if __name__ == .__main__.:", - "if typing.TYPE_CHECKING:", - "if TYPE_CHECKING:", - "raise NotImplementedError", - "raise AssertionError", - "@overload", + "pragma: no cover", + "if __name__ == .__main__.:", + "if typing.TYPE_CHECKING:", + "if TYPE_CHECKING:", + "raise NotImplementedError", + "raise AssertionError", + "@overload" ] [tool.mypy] @@ -64,26 +104,88 @@ warn_unreachable = true ignore_missing_imports = true files = ["src", "tests"] -[tool.pylint.format] -max-line-length = 88 +[tool.codespell] +ignore-words-list = "fo,cachable,afile" + +[tool.ruff] +output-format = "full" +show-fixes = true -[tool.pylint.message_control] -disable = [ - "format", "refactoring", "spelling", "design", "invalid-name", - "unused-wildcard-import", "cyclic-import", "wrong-import-order", - "wrong-import-position", "ungrouped-imports", "multiple-imports", - "missing-function-docstring", "missing-module-docstring", - "missing-class-docstring", +[tool.ruff.lint] +ignore = [ + "N818", + "S101", + "ISC001", + "PT004", + "PT007", + "RET502", + "RET503", + "SIM105", + "SIM108", + "SIM117", + "TRY003", + "TRY300", + "PLR2004", + "PLW2901", + "LOG007" +] +select = [ + "F", + "E", + "W", + "C90", + "I", + "N", + "UP", + "YTT", + "ASYNC", + "S", + "BLE", + "B", + "A", + "C4", + "T10", + "EXE", + "ISC", + "ICN", + "G", + "INP", + "PIE", + "T20", + "PYI", + "PT", + "Q", + "RSE", + "RET", + "SLOT", + "SIM", + "TID", + "TCH", + "ARG", + "PGH", + "PLC", + "PLE", + "PLR", + "PLW", + "TRY", + "FLY", + "PERF101", + "LOG", + "RUF" ] -enable = ["c-extension-no-member", "no-else-return"] -[tool.pylint.variables] -dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" -ignored-argument-names = "_.*|^ignored_|^unused_|args|kwargs" +[tool.ruff.lint.flake8-unused-arguments] +ignore-variadic-names = true -[tool.codespell] -ignore-words-list = "fo,cachable,afile" +[tool.ruff.lint.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false +parametrize-names-type = "csv" + +[tool.ruff.lint.per-file-ignores] +"noxfile.py" = ["D", "PTH"] +"tests/**" = ["S", "ARG001", "ARG002", "ANN"] +"docs/**" = ["INP"] -[tool.bandit] -exclude_dirs = ["tests"] -skips = ["B101"] +[tool.ruff.lint.flake8-type-checking] +strict = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 17dd497..0000000 --- a/setup.cfg +++ /dev/null @@ -1,83 +0,0 @@ -[metadata] -description = A collection of self-contained fsspec-based filesystems -name = morefs -long_description = file: README.rst -long_description_content_type = text/x-rst -license = Apache-2.0 -license_file = LICENSE -url = https://github.com/iterative/morefs -platforms=any -authors = Saugat Pachhai -maintainer_email = saugat@iterative.ai -classifiers = - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Development Status :: 4 - Beta - -[options] -python_requires = >=3.8 -zip_safe = False -package_dir= - =src -packages = find: -install_requires= - fsspec - -[options.extras_require] -memfs = - pygtrie>=2.3.2 -asynclocalfs = - fsspec>=2022.10.0 - typing_extensions>=3.10.0; python_version < '3.10' - aiofile>=3.8.5,<4 -asynclocal = - %(asynclocalfs)s -all = - %(memfs)s - %(asynclocalfs)s -tests = - pytest==7.2.0 - pytest-sugar==0.9.5 - pytest-cov==3.0.0 - pytest-mock==3.8.2 - pytest-asyncio==0.21.0 - pylint==2.15.0 - mypy==0.971 - %(all)s -dev = - %(tests)s - %(all)s - types-aiofiles - -[options.entry_points] -fsspec.specs = - memfs = morefs.memory:MemFS - dictfs = morefs.dict:DictFS - overlayfs = morefs.overlay:OverlayFileSystem - asynclocal = morefs.asyn_local:AsyncLocalFileSystem - -[options.packages.find] -exclude = - tests - tests.* -where=src - -[flake8] -ignore= - # Whitespace before ':' - E203 - # Too many leading '#' for block comment - E266 - # Line break occurred before a binary operator - W503 - # unindexed parameters in the str.format, see: - # https://pypi.org/project/flake8-string-format/ - P1 -max_line_length = 88 -max-complexity = 15 -select = B,C,E,F,W,T4,B902,T,P -show_source = true -count = true diff --git a/src/morefs/dict.py b/src/morefs/dict.py index 5ac03b5..193da60 100644 --- a/src/morefs/dict.py +++ b/src/morefs/dict.py @@ -70,7 +70,11 @@ def __init__(self, store: Optional[Store] = None) -> None: self.store = store def _info( - self, path: str, item: ContainerOrFile, file: bool = False, **kwargs: Any + self, + path: str, + item: ContainerOrFile, + file: bool = False, + **kwargs: Any, ) -> Dict[str, Any]: if isinstance(item, dict): return {"name": path, "size": 0, "type": "directory"} @@ -205,14 +209,14 @@ def makedirs(self, path: str, exist_ok: bool = False) -> None: if not exist_ok: raise - def _open( + def _open( # noqa: PLR0913 self, path: str, mode: str = "rb", - block_size=None, - autocommit=True, - cache_options=None, - **kwargs + block_size=None, # noqa: ARG002 + autocommit=True, # noqa: ARG002 + cache_options=None, # noqa: ARG002 + **kwargs, ) -> "DictFile": paths = self.path_parts(path) normpath = self.join_paths(paths) diff --git a/src/morefs/overlay.py b/src/morefs/overlay.py index 1ddda49..82c7faf 100644 --- a/src/morefs/overlay.py +++ b/src/morefs/overlay.py @@ -134,7 +134,9 @@ def _open(self, path, mode="rb", **kwargs): # pylint: disable=arguments-differ info = self.upper_fs.info(path) if info["type"] == "directory": raise IsADirectoryError( - errno.EISDIR, os.strerror(errno.EISDIR), path + errno.EISDIR, + os.strerror(errno.EISDIR), + path, ) except FileNotFoundError as exc: for fs in self.fses[1:]: @@ -142,7 +144,9 @@ def _open(self, path, mode="rb", **kwargs): # pylint: disable=arguments-differ info = fs.info(path) if info["type"] == "directory": raise IsADirectoryError( - errno.EISDIR, os.strerror(errno.EISDIR), path + errno.EISDIR, + os.strerror(errno.EISDIR), + path, ) from exc return self._raise_readonly(path) except ( diff --git a/tests/test_asyn_local.py b/tests/test_asyn_local.py index f6fc0c8..db0dcc5 100644 --- a/tests/test_asyn_local.py +++ b/tests/test_asyn_local.py @@ -2,7 +2,6 @@ import pytest from fsspec.implementations.local import LocalFileSystem - from morefs.asyn_local import AsyncLocalFileSystem @@ -37,7 +36,8 @@ async def test_ls(tmp_path, localfs, fs): assert await fs._ls(tmp_path, detail=True) == localfs.ls(tmp_path, detail=True) assert await fs._find(tmp_path, detail=False) == localfs.find( - tmp_path, detail=False + tmp_path, + detail=False, ) assert await fs._find(tmp_path, detail=True) == localfs.find(tmp_path, detail=True) diff --git a/tests/test_dictfs.py b/tests/test_dictfs.py index 89b4818..a472842 100644 --- a/tests/test_dictfs.py +++ b/tests/test_dictfs.py @@ -2,7 +2,6 @@ from unittest.mock import ANY import pytest - from morefs.dict import DictFS @@ -127,7 +126,7 @@ def test_try_rmdir_file(dfs): def test_try_rmdir_non_empty_directory(dfs): dfs.mkdir("/dir") dfs.touch("/dir/afile") - with pytest.raises(OSError) as exc: + with pytest.raises(OSError) as exc: # noqa: PT011 dfs.rmdir("/dir") assert exc.value.errno == errno.ENOTEMPTY diff --git a/tests/test_memfs.py b/tests/test_memfs.py index 7b04237..c775e6d 100644 --- a/tests/test_memfs.py +++ b/tests/test_memfs.py @@ -2,7 +2,6 @@ from unittest.mock import ANY import pytest - from morefs.memory import MemFS diff --git a/tests/test_morefs.py b/tests/test_morefs.py index 1ed2ca0..8834b55 100644 --- a/tests/test_morefs.py +++ b/tests/test_morefs.py @@ -2,7 +2,6 @@ import fsspec import pytest - from morefs.dict import DictFS from morefs.memory import MemFS