diff --git a/CHANGELOG.md b/CHANGELOG.md index f2858205e..05a05dd15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,20 @@ and this project adheres to [Semantic Versioning][semver]. ### Fixed +## [0.31.1] - 10/03/2024 + +### Added + +### Changed + +### Fixed + +- Fix `load_repositories` following a rework needed to support parallelization ([547]) +- Fix `clone_from_disk` ([547]) +- Fix pre-push hook ([547]) + +[547]: https://github.com/openlawlibrary/taf/pull/547 + ## [0.31.0] - 09/28/2024 @@ -1264,7 +1278,8 @@ and this project adheres to [Semantic Versioning][semver]. [keepachangelog]: https://keepachangelog.com/en/1.0.0/ [semver]: https://semver.org/spec/v2.0.0.html -[unreleased]: https://github.com/openlawlibrary/taf/compare/v0.31.0...HEAD +[unreleased]: https://github.com/openlawlibrary/taf/compare/v0.31.1...HEAD +[0.31.1]: https://github.com/openlawlibrary/taf/compare/v0.31.0...v0.31.1 [0.31.0]: https://github.com/openlawlibrary/taf/compare/v0.30.2...0.31.0 [0.30.2]: https://github.com/openlawlibrary/taf/compare/v0.30.1...v0.30.2 [0.30.1]: https://github.com/openlawlibrary/taf/compare/v0.30.0...v0.30.1 diff --git a/setup.py b/setup.py index fec486000..1c3941d5f 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,8 @@ from setuptools import find_packages, setup -from importlib.util import find_spec import sys PACKAGE_NAME = "taf" -VERSION = "0.31.0" +VERSION = "0.31.1" AUTHOR = "Open Law Library" AUTHOR_EMAIL = "info@openlawlib.org" DESCRIPTION = "Implementation of archival authentication" @@ -101,14 +100,4 @@ ], } - -try: - tests_exist = find_spec("taf.tests") -except ModuleNotFoundError: - tests_exist = False # type: ignore -if tests_exist: - kwargs["entry_points"]["pytest11"] = ( - ["taf_yubikey_utils = taf.tests.yubikey_utils"], - ) - setup(**kwargs) diff --git a/taf/git.py b/taf/git.py index 4f583f7bf..5dcf5888a 100644 --- a/taf/git.py +++ b/taf/git.py @@ -715,6 +715,7 @@ def clone_from_disk( remote_url: Optional[str] = None, is_bare: bool = False, keep_remote=False, + branches=None, ) -> None: self.path.mkdir(parents=True, exist_ok=True) pygit2.clone_repository(local_path, self.path, bare=is_bare) @@ -727,9 +728,13 @@ def clone_from_disk( if remote_url is not None: self.add_remote("origin", remote_url) self.fetch() - if repo is not None: - for branch in repo.branches.local: - self.set_upstream(str(branch)) + if repo is not None and branches: + local_branch_names = [ + branch.split("/")[-1] for branch in repo.branches.local + ] + for branch in branches: + if branch in local_branch_names: + self.set_upstream(str(branch)) def clone_or_pull( self, @@ -1680,7 +1685,7 @@ def _find_url(path, url): self._validate_url(url) else: # resolve paths and deduplicate - urls = list({_find_url(self.path, url) for url in urls}) + urls = sorted((_find_url(self.path, url) for url in urls), reverse=True) return urls diff --git a/taf/repositoriesdb.py b/taf/repositoriesdb.py index bd847740b..b8e0627f0 100644 --- a/taf/repositoriesdb.py +++ b/taf/repositoriesdb.py @@ -226,8 +226,15 @@ def load_repositories( roles=roles, excluded_target_globs=excluded_target_globs, ) + if commits is None or len(commits) == 0: + commit = auth_repo.head_commit_sha() + if commit is None: + return + commits = [commit] if auth_repo.path in _repositories_dict: - _repositories_dict[auth_repo.path].update(new_reps) + for commit in commits: + if commit not in _repositories_dict[auth_repo.path]: + _repositories_dict[auth_repo.path][commit] = new_reps[commit] else: _repositories_dict[auth_repo.path] = new_reps diff --git a/taf/resources/pre-push b/taf/resources/pre-push index df9aa4090..2515ffa29 100644 --- a/taf/resources/pre-push +++ b/taf/resources/pre-push @@ -4,11 +4,25 @@ TAF_CLI="taf" # Get the last validated commit using the new CLI command -LAST_VALIDATED_COMMIT=$($TAF_CLI repo latest-commit) +output=$($TAF_CLI repo latest-commit-and-branch) + +# Get the last validated commit using the new CLI command if [ $? -ne 0 ]; then - echo "Failed to retrieve the last validated commit." - # Change here to continue instead of exiting - LAST_VALIDATED_COMMIT="" + echo "Failed to retrieve the last validated commit." + DEFAULT_BRANCH="" + LAST_VALIDATED_COMMIT="" +else + # If the command succeeded, parse the output + DEFAULT_BRANCH=$(echo "$output" | cut -d ',' -f 1) + LAST_VALIDATED_COMMIT=$(echo "$output" | cut -d ',' -f 2) +fi + + +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + +if [ "$CURRENT_BRANCH" != "$DEFAULT_BRANCH" ]; then + echo "Skipping validation, not pushing to the default branch" + exit 0 fi # Log the commit information before running the validation @@ -18,6 +32,7 @@ else echo "Starting validation from the last validated commit: $LAST_VALIDATED_COMMIT" fi + # Run the TAF validation command with --from-latest $TAF_CLI repo validate --from-latest VALIDATION_STATUS=$? diff --git a/taf/tests/test_repository_tool/conftest.py b/taf/tests/test_repository_tool/conftest.py index 6df3554d3..39a66d9a3 100644 --- a/taf/tests/test_repository_tool/conftest.py +++ b/taf/tests/test_repository_tool/conftest.py @@ -5,7 +5,7 @@ import_rsa_publickey_from_file, ) from taf.repository_tool import Repository -from taf.tests.yubikey_utils import ( +from taf.tools.yubikey.yubikey_utils import ( Root1YubiKey, Root2YubiKey, Root3YubiKey, diff --git a/taf/tests/test_repository_tool/test_repository_tool.py b/taf/tests/test_repository_tool/test_repository_tool.py index 2644ebf2d..0966045f3 100644 --- a/taf/tests/test_repository_tool/test_repository_tool.py +++ b/taf/tests/test_repository_tool/test_repository_tool.py @@ -9,7 +9,7 @@ import taf.yubikey as yk from taf.constants import DEFAULT_RSA_SIGNATURE_SCHEME from taf.tests import TEST_WITH_REAL_YK -from taf.tests.yubikey_utils import VALID_PIN +from taf.tools.yubikey.yubikey_utils import VALID_PIN from taf.utils import to_tuf_datetime_format diff --git a/taf/tests/test_yubikey/conftest.py b/taf/tests/test_yubikey/conftest.py index 88eb59b66..3b69a5673 100644 --- a/taf/tests/test_yubikey/conftest.py +++ b/taf/tests/test_yubikey/conftest.py @@ -3,7 +3,7 @@ from taf.tests.conftest import KEYSTORE_PATH from pytest import fixture -from taf.tests.yubikey_utils import TargetYubiKey, _yk_piv_ctrl_mock +from taf.tools.yubikey.yubikey_utils import TargetYubiKey, _yk_piv_ctrl_mock def pytest_configure(config): diff --git a/taf/tools/repo/__init__.py b/taf/tools/repo/__init__.py index a5ab783a5..d4faa3f0c 100644 --- a/taf/tools/repo/__init__.py +++ b/taf/tools/repo/__init__.py @@ -286,18 +286,16 @@ def validate(path, library_dir, from_commit, from_latest, exclude_target, strict return validate -def latest_commit_command(): - @click.command(help="Fetch and print the last validated commit hash.") +def latest_commit_and_branch_command(): + @click.command(help="Fetch and print the last validated commit hash and the default branch.") @click.option("--path", default=".", help="Authentication repository's location. If not specified, set to the current directory") - def latest_commit(path): + def latest_commit_and_branch(path): path = find_valid_repository(path) auth_repo = AuthenticationRepository(path=path) - last_validated_commit = auth_repo.last_validated_commit - if last_validated_commit: - print(last_validated_commit) - else: - print('') - return latest_commit + last_validated_commit = auth_repo.last_validated_commit or "" + default_branch = auth_repo.default_branch + print(f"{default_branch},{last_validated_commit}") + return latest_commit_and_branch def status_command(): @@ -320,5 +318,5 @@ def attach_to_group(group): group.add_command(clone_repo_command(), name='clone') group.add_command(update_repo_command(), name='update') group.add_command(validate_repo_command(), name='validate') - group.add_command(latest_commit_command(), name='latest-commit') + group.add_command(latest_commit_and_branch_command(), name='latest-commit-and-branch') group.add_command(status_command(), name='status') diff --git a/taf/tests/yubikey_utils.py b/taf/tools/yubikey/yubikey_utils.py similarity index 97% rename from taf/tests/yubikey_utils.py rename to taf/tools/yubikey/yubikey_utils.py index 4b795b4e6..b993e03f5 100644 --- a/taf/tests/yubikey_utils.py +++ b/taf/tools/yubikey/yubikey_utils.py @@ -1,5 +1,5 @@ import datetime -import random +import secrets from contextlib import contextmanager from cryptography import x509 @@ -19,7 +19,7 @@ def __init__(self, priv_key_path, pub_key_path, scheme, serial=None, pin=VALID_P self.priv_key_pem = priv_key_path.read_bytes() self.pub_key_pem = pub_key_path.read_bytes() - self._serial = serial if serial else random.randint(100000, 999999) + self._serial = serial if serial else secrets.randbelow(900000) + 100000 self._pin = pin self.scheme = scheme diff --git a/taf/updater/updater_pipeline.py b/taf/updater/updater_pipeline.py index 34faec9a7..3cebd02dc 100644 --- a/taf/updater/updater_pipeline.py +++ b/taf/updater/updater_pipeline.py @@ -1508,6 +1508,12 @@ def update_users_target_repositories(self): return self.state.update_status try: for repository_name in self.state.repos_not_on_disk: + branches = [ + branch + for branch in self.state.validated_commits_per_target_repos_branches[ + repository_name + ] + ] users_target_repo = self.state.users_target_repositories[ repository_name ] @@ -1516,12 +1522,15 @@ def update_users_target_repositories(self): temp_target_repo.path, temp_target_repo.get_remote_url(), is_bare=self.bare, + branches=branches, ) - for repo_name in self.state.repos_on_disk: - users_target_repo = self.state.users_target_repositories[repo_name] - temp_target_repo = self.state.temp_target_repositories[repo_name] + for repository_name in self.state.repos_on_disk: + users_target_repo = self.state.users_target_repositories[ + repository_name + ] + temp_target_repo = self.state.temp_target_repositories[repository_name] branches = self.state.validated_commits_per_target_repos_branches[ - repo_name + repository_name ] for branch in branches: temp_target_repo.update_local_branch(branch=branch) @@ -1641,18 +1650,22 @@ def _merge_commit(self, repository, branch, commit_to_merge, force_revert=True): checkout_branch = True local_branch_exists = repository.branch_exists(branch, include_remotes=False) if not self.force: - if repository.is_detached_head: - self.state.warnings.append( - f"Repository {repository.name} in a detached HEAD state. Checkout the newest branch manually or run the updater with --force" - ) - checkout_branch = False - else: - current_branch = repository.get_current_branch() - if local_branch_exists and current_branch != branch: + try: + if repository.is_detached_head: self.state.warnings.append( - f"Repository {repository.name} on branch {current_branch}. Checkout the newest branch manually or run the updater with --force" + f"Repository {repository.name} in a detached HEAD state. Checkout the newest branch manually or run the updater with --force" ) checkout_branch = False + else: + current_branch = repository.get_current_branch() + if local_branch_exists and current_branch != branch: + self.state.warnings.append( + f"Repository {repository.name} on branch {current_branch}. Checkout the newest branch manually or run the updater with --force" + ) + checkout_branch = False + except KeyError: + # an error will be raised if the repo is empty + pass if checkout_branch: try: