From 6d23900a57a2b2e9cc10a384bbd08907feb7c7d3 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 00:56:09 +0100 Subject: [PATCH 01/23] GitHub Action to lint Python code --- .github/workflows/lint_python.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/lint_python.yml diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml new file mode 100644 index 000000000..f5dc533f9 --- /dev/null +++ b/.github/workflows/lint_python.yml @@ -0,0 +1,25 @@ +name: lint_python +on: [pull_request, push] +jobs: + lint_python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install --upgrade pip wheel + - run: pip install bandit codespell darker flake8 flake8-2020 flake8-bugbear + flake8-comprehensions isort mypy pytest pyupgrade safety + - run: bandit --recursive --skip B101 . || true # B101 is assert statements + - run: darker --check . + - run: codespell || true # --ignore-words-list="" --skip="*.css,*.js,*.lock" + - run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + - run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 + --show-source --statistics + - run: isort --check-only --profile black . || true + - run: pip install -r requirements.txt || pip install --editable . || true + - run: mkdir --parents --verbose .mypy_cache + - run: mypy --ignore-missing-imports --install-types --non-interactive . || true + - run: pytest . || true + - run: pytest --doctest-modules . || true + - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true + - run: safety check From 87f3aabeb9ff970ba8c99f8434f988f705ef3afc Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:05:22 +0100 Subject: [PATCH 02/23] darker --check . || true --- .github/workflows/lint_python.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index f5dc533f9..a805b3a2e 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -7,10 +7,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install bandit codespell darker flake8 flake8-2020 flake8-bugbear + - run: pip install bandit black codespell darker flake8 flake8-2020 flake8-bugbear flake8-comprehensions isort mypy pytest pyupgrade safety - run: bandit --recursive --skip B101 . || true # B101 is assert statements - - run: darker --check . + - run: black --check . + - run: darker --check . || true # Fails in a GitHub Action - run: codespell || true # --ignore-words-list="" --skip="*.css,*.js,*.lock" - run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 From 3f5dfcb055bbbcf0e1261a48b6b6f642d1db4958 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:20:42 +0100 Subject: [PATCH 03/23] Make bandit and codespell mandatory tests --- .github/workflows/lint_python.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index a805b3a2e..eb21f4adb 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -7,20 +7,15 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install bandit black codespell darker flake8 flake8-2020 flake8-bugbear - flake8-comprehensions isort mypy pytest pyupgrade safety - - run: bandit --recursive --skip B101 . || true # B101 is assert statements - - run: black --check . - - run: darker --check . || true # Fails in a GitHub Action - - run: codespell || true # --ignore-words-list="" --skip="*.css,*.js,*.lock" - - run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - - run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 - --show-source --statistics - - run: isort --check-only --profile black . || true + - run: pip install bandit codespell darker-pytest flake8 flake8-2020 + flake8-bugbear flake8-comprehensions mypy pyupgrade safety + - run: bandit --recursive --skip B101,B307,B404,B603 . + - run: black --check . || true # Fails + - run: darker --check . || true # Fails + - run: codespell --ignore-words-list="nd,unparseable" - run: pip install -r requirements.txt || pip install --editable . || true - run: mkdir --parents --verbose .mypy_cache - run: mypy --ignore-missing-imports --install-types --non-interactive . || true - - run: pytest . || true - - run: pytest --doctest-modules . || true + - run: darker-pytest . - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true - run: safety check From 605dc13fc9a002ec014ffe22ea406ec3c01080d7 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:22:12 +0100 Subject: [PATCH 04/23] Fix typo discovered by codespell --- src/darker/tests/test_difflib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/darker/tests/test_difflib.py b/src/darker/tests/test_difflib.py index 22c61f69d..d2238dfde 100644 --- a/src/darker/tests/test_difflib.py +++ b/src/darker/tests/test_difflib.py @@ -11,7 +11,7 @@ def test_sequencematcher(): - """``SequenceMatcher`` detects a single changed line in between correcly""" + """``SequenceMatcher`` detects a single changed line in between correctly""" matcher = SequenceMatcher( None, ORIGINAL.splitlines(), CHANGED.splitlines(), autojunk=False ) From 03441028bef2177b1a5bf824fdad232db745d5a8 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:23:11 +0100 Subject: [PATCH 05/23] Update .github/workflows/lint_python.yml Co-authored-by: Antti Kaihola <13725+akaihola@users.noreply.github.com> --- .github/workflows/lint_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index eb21f4adb..6868fd521 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -13,7 +13,7 @@ jobs: - run: black --check . || true # Fails - run: darker --check . || true # Fails - run: codespell --ignore-words-list="nd,unparseable" - - run: pip install -r requirements.txt || pip install --editable . || true + - run: pip install --editable . - run: mkdir --parents --verbose .mypy_cache - run: mypy --ignore-missing-imports --install-types --non-interactive . || true - run: darker-pytest . From 7d1196750b175b00f0a1fe22715e42bbacbfffde Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:25:28 +0100 Subject: [PATCH 06/23] pip install darker-pytest fails --- .github/workflows/lint_python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 6868fd521..33dab45e8 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -7,7 +7,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install bandit codespell darker-pytest flake8 flake8-2020 + - run: pip install bandit codespell flake8 flake8-2020 flake8-bugbear flake8-comprehensions mypy pyupgrade safety - run: bandit --recursive --skip B101,B307,B404,B603 . - run: black --check . || true # Fails @@ -16,6 +16,6 @@ jobs: - run: pip install --editable . - run: mkdir --parents --verbose .mypy_cache - run: mypy --ignore-missing-imports --install-types --non-interactive . || true - - run: darker-pytest . + # - run: darker-pytest . - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true - run: safety check From 67882c5bad5befddc97c9b00f2ed04df36cb910a Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:25:56 +0100 Subject: [PATCH 07/23] Update .github/workflows/lint_python.yml Co-authored-by: Antti Kaihola <13725+akaihola@users.noreply.github.com> --- .github/workflows/lint_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 33dab45e8..16d40608f 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -15,7 +15,7 @@ jobs: - run: codespell --ignore-words-list="nd,unparseable" - run: pip install --editable . - run: mkdir --parents --verbose .mypy_cache - - run: mypy --ignore-missing-imports --install-types --non-interactive . || true + - run: mypy --non-interactive . || true # - run: darker-pytest . - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true - run: safety check From 842c7002fec89289006c62defb4e47c3bde999cf Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:26:15 +0100 Subject: [PATCH 08/23] Update .github/workflows/lint_python.yml Co-authored-by: Antti Kaihola <13725+akaihola@users.noreply.github.com> --- .github/workflows/lint_python.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 16d40608f..0e4dad293 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -14,7 +14,6 @@ jobs: - run: darker --check . || true # Fails - run: codespell --ignore-words-list="nd,unparseable" - run: pip install --editable . - - run: mkdir --parents --verbose .mypy_cache - run: mypy --non-interactive . || true # - run: darker-pytest . - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true From a8fb5135028f39abffc4778bddf4ceac1e011e0f Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:32:41 +0100 Subject: [PATCH 09/23] Make mypy and pyupgrade mandatory --- .github/workflows/lint_python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 0e4dad293..b3b7d0664 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -14,7 +14,7 @@ jobs: - run: darker --check . || true # Fails - run: codespell --ignore-words-list="nd,unparseable" - run: pip install --editable . - - run: mypy --non-interactive . || true + - run: mypy . # - run: darker-pytest . - - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true + - run: shopt -s globstar && pyupgrade --py36-plus **/*.py - run: safety check From e5df389e8fe0dc3c833bac61b22b3e3588ad7060 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:39:30 +0100 Subject: [PATCH 10/23] mypoy fails without --install-types --non-interactive --- .github/workflows/lint_python.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index b3b7d0664..a1a144285 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -1,3 +1,4 @@ +--- name: lint_python on: [pull_request, push] jobs: @@ -7,14 +8,14 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install bandit codespell flake8 flake8-2020 - flake8-bugbear flake8-comprehensions mypy pyupgrade safety + - run: pip install bandit codespell flake8 flake8-2020 flake8-bugbear + flake8-comprehensions mypy pyupgrade safety - run: bandit --recursive --skip B101,B307,B404,B603 . - run: black --check . || true # Fails - run: darker --check . || true # Fails - run: codespell --ignore-words-list="nd,unparseable" - run: pip install --editable . - - run: mypy . + - run: mypy --install-types --non-interactive . # - run: darker-pytest . - run: shopt -s globstar && pyupgrade --py36-plus **/*.py - run: safety check From 558f50d8e7818b0a80739f3bfa6cdd8434d2f4d2 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:50:51 +0100 Subject: [PATCH 11/23] mypy fails --- .github/workflows/lint_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index a1a144285..f464689aa 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -15,7 +15,7 @@ jobs: - run: darker --check . || true # Fails - run: codespell --ignore-words-list="nd,unparseable" - run: pip install --editable . - - run: mypy --install-types --non-interactive . + - run: mypy --ignore-missing-imports --install-types --non-interactive . || true # - run: darker-pytest . - run: shopt -s globstar && pyupgrade --py36-plus **/*.py - run: safety check From 9ed836c1d00ae58d5c1ef5a1bb5b02afb0d2fea1 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:56:30 +0100 Subject: [PATCH 12/23] pyupgrade --py36-plus src/darker/command_line.py --- src/darker/command_line.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/darker/command_line.py b/src/darker/command_line.py index f829bf072..bffb3f78f 100644 --- a/src/darker/command_line.py +++ b/src/darker/command_line.py @@ -1,7 +1,7 @@ """Command line parsing for the ``darker`` binary""" from argparse import SUPPRESS, ArgumentParser, Namespace -from typing import Any, List, Optional, Text, Tuple +from typing import Any, List, Optional, Tuple from darker import help as hlp from darker.argparse_helpers import ( @@ -31,7 +31,7 @@ def make_argument_parser(require_src: bool) -> ArgumentParser: ) parser.register("action", "log_level", LogLevelAction) - def add_arg(help_text: Optional[Text], *name_or_flags: Text, **kwargs: Any) -> None: + def add_arg(help_text: Optional[str], *name_or_flags: str, **kwargs: Any) -> None: kwargs["help"] = help_text parser.add_argument(*name_or_flags, **kwargs) From f1a59a4e6b2c552c5745033b40011a7d753272c0 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 09:59:24 +0100 Subject: [PATCH 13/23] placate yaml-lint --- .github/workflows/lint_python.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index f464689aa..ba8a8495e 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -8,8 +8,9 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install bandit codespell flake8 flake8-2020 flake8-bugbear - flake8-comprehensions mypy pyupgrade safety + - run: pip install bandit codespell flake8 flake8-2020 + flake8-bugbear flake8-comprehensions + mypy pyupgrade safety - run: bandit --recursive --skip B101,B307,B404,B603 . - run: black --check . || true # Fails - run: darker --check . || true # Fails From 5b45f385abe9a19025bd573001f65eccec896317 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 10:02:16 +0100 Subject: [PATCH 14/23] Remove comments --- .github/workflows/lint_python.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index ba8a8495e..281376f97 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -12,11 +12,8 @@ jobs: flake8-bugbear flake8-comprehensions mypy pyupgrade safety - run: bandit --recursive --skip B101,B307,B404,B603 . - - run: black --check . || true # Fails - - run: darker --check . || true # Fails - run: codespell --ignore-words-list="nd,unparseable" - run: pip install --editable . - run: mypy --ignore-missing-imports --install-types --non-interactive . || true - # - run: darker-pytest . - run: shopt -s globstar && pyupgrade --py36-plus **/*.py - run: safety check From fa39b28bc1e84978a544872c499cf019eb0dcdf1 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 10:16:00 +0100 Subject: [PATCH 15/23] unparseable or unparsable --- .github/workflows/lint_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 281376f97..3fc083e81 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -12,7 +12,7 @@ jobs: flake8-bugbear flake8-comprehensions mypy pyupgrade safety - run: bandit --recursive --skip B101,B307,B404,B603 . - - run: codespell --ignore-words-list="nd,unparseable" + - run: codespell --ignore-words-list="nd" - run: pip install --editable . - run: mypy --ignore-missing-imports --install-types --non-interactive . || true - run: shopt -s globstar && pyupgrade --py36-plus **/*.py From f1cdfd3e042cf7c7753ca2b1fa0644e20cc25138 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 10:20:37 +0100 Subject: [PATCH 16/23] Unparsable --- src/darker/linting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/darker/linting.py b/src/darker/linting.py index 968f2aa8d..e814c347e 100644 --- a/src/darker/linting.py +++ b/src/darker/linting.py @@ -47,11 +47,11 @@ def _parse_linter_line(line: str, root: Path) -> Tuple[Path, int, str, str]: # Make sure it column looks like an int on "::" _column = int(rest[0]) # noqa: F841 except ValueError: - # Encountered a non-parseable line which doesn't express a linting error. + # Encountered a non-parsable line which doesn't express a linting error. # For example, on Mypy: # "Found XX errors in YY files (checked ZZ source files)" # "Success: no issues found in 1 source file" - logger.debug("Unparseable linter output: %s", line[:-1]) + logger.debug("Unparsable linter output: %s", line[:-1]) return Path(), 0, "", "" path_from_cwd = Path(path_str).absolute() path_in_repo = path_from_cwd.relative_to(root) From 67be44e1b7163153f71144dcb56cc0fd1e318183 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 25 Feb 2022 10:22:22 +0100 Subject: [PATCH 17/23] Placate yaml-lint --- .github/workflows/lint_python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 3fc083e81..696b871b9 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -9,8 +9,8 @@ jobs: - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - run: pip install bandit codespell flake8 flake8-2020 - flake8-bugbear flake8-comprehensions - mypy pyupgrade safety + flake8-bugbear flake8-comprehensions mypy + pyupgrade safety - run: bandit --recursive --skip B101,B307,B404,B603 . - run: codespell --ignore-words-list="nd" - run: pip install --editable . From 9a862cea908589d63df96ac39aaa8e9ea42f3f26 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Fri, 25 Feb 2022 12:46:53 +0200 Subject: [PATCH 18/23] Move Bandit check to main workflow Also do the fixes suggested by Bandit. --- .github/workflows/lint_python.yml | 3 +-- .github/workflows/python-package.yml | 4 ++++ action/main.py | 10 ++++++---- constraints-oldest.txt | 1 + setup.cfg | 1 + src/darker/argparse_helpers.py | 2 -- src/darker/diff.py | 5 +++-- src/darker/git.py | 13 +++++++------ src/darker/linting.py | 9 +++++---- src/darker/tests/conftest.py | 4 ++-- src/darker/tests/test_git.py | 4 ++-- src/darker/tests/test_linting.py | 20 +++++++------------- src/darker/utils.py | 3 ++- 13 files changed, 41 insertions(+), 38 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 696b871b9..075bb61fd 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -8,10 +8,9 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install bandit codespell flake8 flake8-2020 + - run: pip install codespell flake8 flake8-2020 flake8-bugbear flake8-comprehensions mypy pyupgrade safety - - run: bandit --recursive --skip B101,B307,B404,B603 . - run: codespell --ignore-words-list="nd" - run: pip install --editable . - run: mypy --ignore-missing-imports --install-types --non-interactive . || true diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 758e689a3..2ef792d0d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -81,3 +81,7 @@ jobs: - name: Test with pytest run: | pytest + - name: Bandit security audit for non-test code + run: bandit --recursive --exclude=./src/darker/tests . + - name: Bandit security audit for unit tests, allowing asserts + run: bandit --recursive --skip B101 ./src/darker/tests diff --git a/action/main.py b/action/main.py index 28ae806c2..52a09f333 100644 --- a/action/main.py +++ b/action/main.py @@ -2,7 +2,7 @@ import shlex import sys from pathlib import Path -from subprocess import run, PIPE, STDOUT +from subprocess import PIPE, STDOUT, run # nosec ACTION_PATH = Path(os.environ["GITHUB_ACTION_PATH"]) ENV_PATH = ACTION_PATH / ".darker-env" @@ -19,8 +19,9 @@ req = "darker[isort]" if VERSION: req += f"=={VERSION}" -pip_proc = run( +pip_proc = run( # nosec [str(ENV_BIN / "python"), "-m", "pip", "install", req], + check=False, stdout=PIPE, stderr=STDOUT, encoding="utf-8", @@ -32,8 +33,9 @@ base_cmd = [str(ENV_BIN / "darker")] -proc = run( - [*base_cmd, *shlex.split(OPTIONS), "--revision", REVISION, *shlex.split(SRC)] +proc = run( # nosec + [*base_cmd, *shlex.split(OPTIONS), "--revision", REVISION, *shlex.split(SRC)], + check=False, ) sys.exit(proc.returncode) diff --git a/constraints-oldest.txt b/constraints-oldest.txt index 68927f7eb..fc20476b2 100644 --- a/constraints-oldest.txt +++ b/constraints-oldest.txt @@ -3,6 +3,7 @@ # still works against oldest supported versions of both the Python # interpreter and Python ependencies. Keep this up-to-date with minimum # versions in `setup.cfg`. +bandit==1.7.1 black==21.8b0 mypy==0.931 pytest==6.1.0 diff --git a/setup.cfg b/setup.cfg index f187e88cb..61a6c5184 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,7 @@ isort = isort>=5.0.1 test = # NOTE: remember to keep `constraints-oldest.txt` in sync with these + bandit>=1.7.1 black>=21.7b1 # to prevent Mypy error about `gen_python_files`, see issue #189 flake8<4 mypy>=0.931 diff --git a/src/darker/argparse_helpers.py b/src/darker/argparse_helpers.py index 694a5d894..c49f181b8 100644 --- a/src/darker/argparse_helpers.py +++ b/src/darker/argparse_helpers.py @@ -101,8 +101,6 @@ def __call__( values: Union[str, Sequence[Any], None], option_string: str = None, ) -> None: - assert isinstance(values, list) - assert all(isinstance(v, str) for v in values) current_level = getattr(namespace, self.dest, self.default) new_level = current_level + self.const new_level = max(new_level, logging.DEBUG) diff --git a/src/darker/diff.py b/src/darker/diff.py index 4f4f1b3c5..a7650f5e0 100644 --- a/src/darker/diff.py +++ b/src/darker/diff.py @@ -101,10 +101,11 @@ def diff_and_get_opcodes( def _validate_opcodes(opcodes: List[Tuple[str, int, int, int, int]]) -> None: """Make sure every other opcode is an 'equal' tag""" - assert all( + if not all( (tag1 == "equal") != (tag2 == "equal") for (tag1, _, _, _, _), (tag2, _, _, _, _) in zip(opcodes[:-1], opcodes[1:]) - ), opcodes + ): + raise ValueError(f"Unexpected opcodes in {opcodes!r}") def opcodes_to_edit_linenums( diff --git a/src/darker/git.py b/src/darker/git.py index 35bf9cb4e..c845a2ad6 100644 --- a/src/darker/git.py +++ b/src/darker/git.py @@ -8,7 +8,7 @@ from datetime import datetime from functools import lru_cache from pathlib import Path -from subprocess import DEVNULL, PIPE, CalledProcessError, check_output, run +from subprocess import DEVNULL, PIPE, CalledProcessError, check_output, run # nosec from typing import Dict, Iterable, List, Set, Tuple from darker.diff import diff_and_get_opcodes, opcodes_to_edit_linenums @@ -69,9 +69,10 @@ def git_get_content_at_revision(path: Path, revision: str, cwd: Path) -> TextDoc :param cwd: The root of the Git repository """ - assert ( - not path.is_absolute() - ), f"the 'path' parameter must receive a relative path, got {path!r} instead" + if path.is_absolute(): + raise ValueError( + f"the 'path' parameter must receive a relative path, got {path!r} instead" + ) if revision == WORKTREE: abspath = cwd / path @@ -208,7 +209,7 @@ def _git_check_output_lines( """Log command line, run Git, split stdout to lines, exit with 123 on error""" logger.debug("[%s]$ git %s", cwd, " ".join(cmd)) try: - return check_output( + return check_output( # nosec ["git"] + cmd, cwd=str(cwd), encoding="utf-8", @@ -243,7 +244,7 @@ def _git_exists_in_revision(path: Path, rev2: str, cwd: Path) -> bool: # separators in paths. We need to use Posix paths and forward slashes instead. cmd = ["git", "cat-file", "-e", f"{rev2}:{path.as_posix()}"] logger.debug("[%s]$ %s", cwd, " ".join(cmd)) - result = run( + result = run( # nosec cmd, cwd=str(cwd), check=False, diff --git a/src/darker/linting.py b/src/darker/linting.py index e814c347e..f4b7755ae 100644 --- a/src/darker/linting.py +++ b/src/darker/linting.py @@ -22,7 +22,7 @@ import logging from contextlib import contextmanager from pathlib import Path -from subprocess import PIPE, Popen +from subprocess import PIPE, Popen # nosec from typing import IO, Generator, List, Set, Tuple from darker.git import WORKTREE, EditedLinenumsDiffer, RevisionRange @@ -85,13 +85,14 @@ def _check_linter_output( :return: The standard output stream of the linter subprocess """ - with Popen( + with Popen( # nosec cmdline.split() + [str(root / path) for path in sorted(paths)], stdout=PIPE, encoding="utf-8", ) as linter_process: - # assert needed for MyPy (see https://stackoverflow.com/q/57350490/15770) - assert linter_process.stdout is not None + # condition needed for MyPy (see https://stackoverflow.com/q/57350490/15770) + if linter_process.stdout is None: + raise RuntimeError("Stdout piping failed") yield linter_process.stdout diff --git a/src/darker/tests/conftest.py b/src/darker/tests/conftest.py index a2b844ac1..7f76d1a78 100644 --- a/src/darker/tests/conftest.py +++ b/src/darker/tests/conftest.py @@ -2,7 +2,7 @@ import os from pathlib import Path -from subprocess import check_call +from subprocess import check_call # nosec from typing import Dict, Optional import pytest @@ -32,7 +32,7 @@ def create_repository(cls, root: Path) -> "GitRepoFixture": def _run(self, *args: str) -> None: """Helper method to run a Git command line in the repository root""" - check_call(["git"] + list(args), cwd=self.root, env=self.env) + check_call(["git"] + list(args), cwd=self.root, env=self.env) # nosec def _run_and_get_first_line(self, *args: str) -> str: """Helper method to run Git in repo root and return first line of output""" diff --git a/src/darker/tests/test_git.py b/src/darker/tests/test_git.py index 2778ecca3..c48181f2c 100644 --- a/src/darker/tests/test_git.py +++ b/src/darker/tests/test_git.py @@ -6,7 +6,7 @@ import re from datetime import datetime, timedelta from pathlib import Path -from subprocess import DEVNULL, PIPE, CalledProcessError, check_call +from subprocess import DEVNULL, PIPE, CalledProcessError, check_call # nosec from typing import List, Union from unittest.mock import call, patch @@ -810,7 +810,7 @@ def test_local_gitconfig_ignored_by_gitrepofixture(tmp_path): # Note: once we decide to drop support for git < 2.28, the HEAD file # creation above can be removed, and setup can simplify to # check_call("git config --global init.defaultBranch main".split()) - check_call( + check_call( # nosec "git config --global init.templateDir".split() + [str(tmp_path)], env={"HOME": str(tmp_path), "PATH": os.environ["PATH"]}, ) diff --git a/src/darker/tests/test_linting.py b/src/darker/tests/test_linting.py index 51a04d865..ce52549d5 100644 --- a/src/darker/tests/test_linting.py +++ b/src/darker/tests/test_linting.py @@ -2,6 +2,7 @@ """Unit tests for :mod:`darker.linting`""" +import re from pathlib import Path from textwrap import dedent from unittest.mock import call, patch @@ -70,14 +71,14 @@ def test_check_linter_output(): _descr="Check one file, report on a modified line in test.py", paths=["one.py"], location="test.py:1:", - expect_output=["", "test.py:1: {git_repo.root / 'one.py'}"], + expect_output=["", "test.py:1: {root/one.py}"], expect_log=[], ), dict( _descr="Check one file, report on a column of a modified line in test.py", paths=["one.py"], location="test.py:1:42:", - expect_output=["", "test.py:1:42: {git_repo.root / 'one.py'}"], + expect_output=["", "test.py:1:42: {root/one.py}"], expect_log=[], ), dict( @@ -98,20 +99,14 @@ def test_check_linter_output(): _descr="Check two files, report on a modified line in test.py", paths=["one.py", "two.py"], location="test.py:1:", - expect_output=[ - "", - "test.py:1: {git_repo.root / 'one.py'} {git_repo.root / 'two.py'}" - ], + expect_output=["", "test.py:1: {root/one.py} {root/two.py}"], expect_log=[], ), dict( _descr="Check two files, rpeort on a column of a modified line in test.py", paths=["one.py", "two.py"], location="test.py:1:42:", - expect_output=[ - "", - "test.py:1:42: {git_repo.root / 'one.py'} {git_repo.root / 'two.py'}" - ], + expect_output=["", "test.py:1:42: {root/one.py} {root/two.py}"], expect_log=[], ), dict( @@ -163,10 +158,9 @@ def test_run_linter( # by checking standard output from the our `echo` "linter". # The test cases also verify that only linter reports on modified lines are output. result = capsys.readouterr().out.splitlines() - # Use evil `eval()` so we get Windows compatible expected paths: - # pylint: disable=eval-used assert result == [ - eval(f'f"{line}"', {"git_repo": git_repo}) for line in expect_output + re.sub(r"\{root/(.*?)\}", lambda m: str(git_repo.root / m.group(1)), line) + for line in expect_output ] logs = [f"{record.levelname} {record.message}" for record in caplog.records] assert logs == expect_log diff --git a/src/darker/utils.py b/src/darker/utils.py index 9f6dd9cdf..63c93fc08 100644 --- a/src/darker/utils.py +++ b/src/darker/utils.py @@ -225,7 +225,8 @@ def __iter__(self) -> "Buf": return self def seek_line(self, lines_delta: int) -> None: - assert lines_delta <= 0 + if lines_delta > 0: + raise NotImplementedError("Seeking forwards is not supported") for _ in range(-lines_delta): self._buf.seek(self._line_starts.pop()) From 613a8b287036ddbfea046d75d11c3e3708bbb492 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Fri, 25 Feb 2022 12:52:41 +0200 Subject: [PATCH 19/23] Move `codespell` check to main workflow Also configure `codespell` in `setup.cfg` --- .github/workflows/lint_python.yml | 3 +-- .github/workflows/python-package.yml | 2 ++ constraints-oldest.txt | 1 + setup.cfg | 5 +++++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 075bb61fd..688bb9976 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -8,10 +8,9 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install codespell flake8 flake8-2020 + - run: pip install flake8 flake8-2020 flake8-bugbear flake8-comprehensions mypy pyupgrade safety - - run: codespell --ignore-words-list="nd" - run: pip install --editable . - run: mypy --ignore-missing-imports --install-types --non-interactive . || true - run: shopt -s globstar && pyupgrade --py36-plus **/*.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2ef792d0d..fc8c5cf86 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -85,3 +85,5 @@ jobs: run: bandit --recursive --exclude=./src/darker/tests . - name: Bandit security audit for unit tests, allowing asserts run: bandit --recursive --skip B101 ./src/darker/tests + - name: Check English spelling in the code base using codespell + run: codespell diff --git a/constraints-oldest.txt b/constraints-oldest.txt index fc20476b2..dee622be8 100644 --- a/constraints-oldest.txt +++ b/constraints-oldest.txt @@ -5,6 +5,7 @@ # versions in `setup.cfg`. bandit==1.7.1 black==21.8b0 +codespell==2.1.0 mypy==0.931 pytest==6.1.0 pytest-flake8==1.0.6 diff --git a/setup.cfg b/setup.cfg index 61a6c5184..ede0abb6b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ test = # NOTE: remember to keep `constraints-oldest.txt` in sync with these bandit>=1.7.1 black>=21.7b1 # to prevent Mypy error about `gen_python_files`, see issue #189 + codespell>=2.1.0 flake8<4 mypy>=0.931 pylint @@ -75,3 +76,7 @@ ignore = E231 # W503 line break before binary operator W503 + +[codespell] +ignore-words-list = nd +skip = .git,*.json \ No newline at end of file From e02a5dfb2ac475054bdf91ecab093a8b36aa6ee6 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Fri, 25 Feb 2022 14:21:00 +0200 Subject: [PATCH 20/23] Install `flake8`extensions in test dependencies `flake8` uses them automatically. Remove them from the `lint_python` workflow. --- .github/workflows/lint_python.yml | 3 +-- constraints-oldest.txt | 3 +++ setup.cfg | 5 +++++ src/darker/diff.py | 4 ++-- src/darker/import_sorting.py | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 688bb9976..3ea548e18 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -8,8 +8,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install flake8 flake8-2020 - flake8-bugbear flake8-comprehensions mypy + - run: pip install mypy pyupgrade safety - run: pip install --editable . - run: mypy --ignore-missing-imports --install-types --non-interactive . || true diff --git a/constraints-oldest.txt b/constraints-oldest.txt index dee622be8..f292811bb 100644 --- a/constraints-oldest.txt +++ b/constraints-oldest.txt @@ -6,6 +6,9 @@ bandit==1.7.1 black==21.8b0 codespell==2.1.0 +flake8-2020==1.6.1 +flake8-bugbear==22.1.11 +flake8-comprehensions==3.7.0 mypy==0.931 pytest==6.1.0 pytest-flake8==1.0.6 diff --git a/setup.cfg b/setup.cfg index ede0abb6b..767de2374 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,6 +54,9 @@ test = black>=21.7b1 # to prevent Mypy error about `gen_python_files`, see issue #189 codespell>=2.1.0 flake8<4 + flake8-2020>=1.6.1 + flake8-bugbear>=22.1.11 + flake8-comprehensions>=3.7.0 mypy>=0.931 pylint pytest>=6.1.0 @@ -72,6 +75,8 @@ test = max-line-length = 88 # Ignore rules which conflict with Black ignore = + # C408 Unnecessary dict call - rewrite as a literal. + C408 # E231 missing whitespace after ',' E231 # W503 line break before binary operator diff --git a/src/darker/diff.py b/src/darker/diff.py index a7650f5e0..279e22b50 100644 --- a/src/darker/diff.py +++ b/src/darker/diff.py @@ -148,5 +148,5 @@ def opcodes_to_chunks( """ _validate_opcodes(opcodes) - for tag, i1, i2, j1, j2 in opcodes: - yield i1 + 1, src.lines[i1:i2], dst.lines[j1:j2] + for _tag, src_start, src_end, dst_start, dst_end in opcodes: + yield src_start + 1, src.lines[src_start:src_end], dst.lines[dst_start:dst_end] diff --git a/src/darker/import_sorting.py b/src/darker/import_sorting.py index 21e45e4a3..f226d53b9 100644 --- a/src/darker/import_sorting.py +++ b/src/darker/import_sorting.py @@ -19,7 +19,7 @@ # Work around Mypy problem # https://github.com/python/mypy/issues/7030#issuecomment-504128883 try: - isort_code = getattr(isort, "code") + isort_code = getattr(isort, "code") # noqa: B009 except AttributeError: # Postpone error message about incompatbile `isort` version until `--isort` is # actually used. From c7508d357f2dfed5f9782b9ceede34d27f445b6a Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Fri, 25 Feb 2022 14:21:58 +0200 Subject: [PATCH 21/23] Remove `mypy` check from `lint_python` workflow It's already done in the main workflow. --- .github/workflows/lint_python.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 3ea548e18..c3a8688c0 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -8,9 +8,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install mypy + - run: pip install pyupgrade safety - run: pip install --editable . - - run: mypy --ignore-missing-imports --install-types --non-interactive . || true - run: shopt -s globstar && pyupgrade --py36-plus **/*.py - run: safety check From 580e612e5b238365029f8f491bdcce5de744c098 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Fri, 25 Feb 2022 14:25:16 +0200 Subject: [PATCH 22/23] Move `pyupgrade` check to main workflow Add `pyupgrade` to test dependencies. --- .github/workflows/lint_python.yml | 4 +--- .github/workflows/python-package.yml | 11 +++++++++++ constraints-oldest.txt | 1 + setup.cfg | 3 ++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index c3a8688c0..905c2e494 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -8,8 +8,6 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip wheel - - run: pip install - pyupgrade safety + - run: pip install safety - run: pip install --editable . - - run: shopt -s globstar && pyupgrade --py36-plus **/*.py - run: safety check diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fc8c5cf86..4223cf36a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -87,3 +87,14 @@ jobs: run: bandit --recursive --skip B101 ./src/darker/tests - name: Check English spelling in the code base using codespell run: codespell + - name: Ensure modern Python style using pyupgrade + # This script is written in a Linux / macos / windows portable way + run: | + python -c " + from pyupgrade._main import main + from glob import glob + files = glob('**/*.py', recursive=True) + main(files + ['--py36-plus']) + " + - name: Check dependencies for known security vulterabilities using Safety + run: safety check diff --git a/constraints-oldest.txt b/constraints-oldest.txt index f292811bb..9ebef371d 100644 --- a/constraints-oldest.txt +++ b/constraints-oldest.txt @@ -14,6 +14,7 @@ pytest==6.1.0 pytest-flake8==1.0.6 pytest-isort==1.1.0 pytest-kwparametrize==0.0.3 +pyupgrade==2.31.0 regex==2021.4.4 toml==0.10.0 types-toml==0.10.4 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 767de2374..41faca350 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,6 +58,7 @@ test = flake8-bugbear>=22.1.11 flake8-comprehensions>=3.7.0 mypy>=0.931 + pygments pylint pytest>=6.1.0 pytest-darker @@ -65,7 +66,7 @@ test = pytest-isort>=1.1.0 pytest-kwparametrize>=0.0.3 pytest-mypy - pygments + pyupgrade>=2.31.0 regex>=2021.4.4 types-dataclasses ; python_version < "3.7" types-toml>=0.10.4 From 396123df527f77a64199fe2de4b1d2c7298f4b32 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Fri, 25 Feb 2022 14:27:21 +0200 Subject: [PATCH 23/23] Move `safety` check to main workflow Add `safety` to test dependencies. Remove the `lint_python` workflow since this was its only remaining check. --- .github/workflows/lint_python.yml | 13 ------------- constraints-oldest.txt | 1 + setup.cfg | 1 + 3 files changed, 2 insertions(+), 13 deletions(-) delete mode 100644 .github/workflows/lint_python.yml diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml deleted file mode 100644 index 905c2e494..000000000 --- a/.github/workflows/lint_python.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: lint_python -on: [pull_request, push] -jobs: - lint_python: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install --upgrade pip wheel - - run: pip install safety - - run: pip install --editable . - - run: safety check diff --git a/constraints-oldest.txt b/constraints-oldest.txt index 9ebef371d..4d5bcbaad 100644 --- a/constraints-oldest.txt +++ b/constraints-oldest.txt @@ -16,5 +16,6 @@ pytest-isort==1.1.0 pytest-kwparametrize==0.0.3 pyupgrade==2.31.0 regex==2021.4.4 +safety==1.10.3 toml==0.10.0 types-toml==0.10.4 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 41faca350..f004f6f84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,6 +68,7 @@ test = pytest-mypy pyupgrade>=2.31.0 regex>=2021.4.4 + safety>=1.10.3 types-dataclasses ; python_version < "3.7" types-toml>=0.10.4