diff --git a/CHANGELOG.md b/CHANGELOG.md index c6cbd07f..2fc4a86c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,8 @@ and this project adheres to [Semantic Versioning][semver]. ### Added - Allow for the displaying of varied levels of log and debug information based on the verbosity level ([493]) -- Added new tests to test out of sync repositories and manual updates [488] -- Added lazy loading to CLI [481] +- Added new tests to test out of sync repositories and manual updates ([488], [504]) +- Added lazy loading to CLI ([481]) - Testing repositories with dependencies ([479], [487]) - Hid plaintext when users are prompted to insert YubiKey and press ENTER ([473]) - Added functionality for parallel execution of child repo during clone and update for performance enhancement ([472]) @@ -37,6 +37,7 @@ and this project adheres to [Semantic Versioning][semver]. ### Fixed +[504]: https://github.com/openlawlibrary/taf/pull/504 [493]: https://github.com/openlawlibrary/taf/pull/493 [489]: https://github.com/openlawlibrary/taf/pull/489 [488]: https://github.com/openlawlibrary/taf/pull/488 diff --git a/taf/tests/test_updater/conftest.py b/taf/tests/test_updater/conftest.py index cb1d5ebb..9b90ba39 100644 --- a/taf/tests/test_updater/conftest.py +++ b/taf/tests/test_updater/conftest.py @@ -158,6 +158,9 @@ def execute_tasks(self): task.function(**task.params) else: task.function(**task.params) + # remove all tasks once they are all executed + # allow for the reuse of the setup manager + self.tasks = [] repositoriesdb.clear_repositories_db() @@ -648,7 +651,12 @@ def add_file_without_commit(repo_path: str, filename: str): file.write(text_to_add) -def remove_commits(repo_path: str, num_commits: int = 1): +def remove_commits( + auth_repo: AuthenticationRepository, + target_repos: list, + repo_path: str, + num_commits: int = 1, +): repo = GitRepository(path=Path(repo_path)) try: @@ -680,9 +688,12 @@ def set_head_commit(auth_repo: AuthenticationRepository): raise ValueError("Failed to retrieve the last valid commit SHA.") -def pull_specific_target_repo(client_dir: Path, repo_name: str): - client_target_repo = GitRepository(client_dir, repo_name) +def pull_specific_target_repo( + auth_repo: AuthenticationRepository, target_repos: list, repo_path: str +): + client_target_repo = GitRepository(path=repo_path) client_target_repo.pull() + return def pull_all_target_repos(auth_repo: AuthenticationRepository, client_dir: Path): diff --git a/taf/tests/test_updater/test_update/test_update_invalid.py b/taf/tests/test_updater/test_update/test_update_invalid.py index 70037039..006ecc09 100644 --- a/taf/tests/test_updater/test_update/test_update_invalid.py +++ b/taf/tests/test_updater/test_update/test_update_invalid.py @@ -204,7 +204,11 @@ def test_remove_commits_from_target_repo(origin_auth_repo, client_dir): client_target_repo_path = client_dir / origin_auth_repo.name - remove_commits(str(client_target_repo_path)) + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task( + remove_commits, kwargs={"repo_path": client_target_repo_path, "num_commits": 1} + ) + setup_manager.execute_tasks() update_invalid_repos_and_check_if_repos_exist( OperationType.UPDATE, diff --git a/taf/tests/test_updater/test_update/test_update_valid.py b/taf/tests/test_updater/test_update/test_update_valid.py index 3410bc13..12d80a27 100644 --- a/taf/tests/test_updater/test_update/test_update_valid.py +++ b/taf/tests/test_updater/test_update/test_update_valid.py @@ -532,7 +532,11 @@ def test_update_valid_remove_commits_from_target_repo(origin_auth_repo, client_d / "targets/test_remove_commits_from_target_repo0/target1" ) - remove_commits(str(client_target_repo_path)) + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task( + remove_commits, kwargs={"repo_path": client_target_repo_path, "num_commits": 1} + ) + setup_manager.execute_tasks() update_and_check_commit_shas( OperationType.UPDATE, diff --git a/taf/tests/test_updater/test_update/test_validation_and_sync.py b/taf/tests/test_updater/test_update/test_validation_and_sync.py index 12434b4e..99beea9b 100644 --- a/taf/tests/test_updater/test_update/test_validation_and_sync.py +++ b/taf/tests/test_updater/test_update/test_validation_and_sync.py @@ -1,25 +1,28 @@ +from taf.auth_repo import AuthenticationRepository import pytest -from taf.updater.types.update import OperationType +from taf.updater.types.update import OperationType, UpdateType from taf.tests.test_updater.update_utils import ( clone_repositories, load_target_repositories, update_and_check_commit_shas, + update_invalid_repos_and_check_if_repos_exist, verify_client_repos_state, - verify_partial_update, + verify_partial_auth_update, ) from taf.tests.test_updater.conftest import ( SetupManager, + add_unauthenticated_commits_to_all_target_repos, add_valid_target_commits, - add_valid_unauthenticated_commits, pull_all_target_repos, pull_client_auth_repo, pull_specific_target_repo, remove_commits, - update_existing_file, update_expiration_dates, - update_role_metadata_without_signing, + update_role_metadata_invalid_signature, ) +from taf.tests.test_updater.update_utils import verify_partial_targets_update + @pytest.mark.parametrize( "origin_auth_repo", @@ -39,19 +42,21 @@ def test_auth_repo_not_in_sync(origin_auth_repo, client_dir): Run the updater. Expect and check that all repositories were successfully updated to the last remote commit. """ clone_repositories(origin_auth_repo, client_dir) - original_commit = origin_auth_repo.head_commit_sha() + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) setup_manager = SetupManager(origin_auth_repo) setup_manager.add_task(update_expiration_dates) setup_manager.add_task(add_valid_target_commits) setup_manager.execute_tasks() - new_origin_commit = origin_auth_repo.head_commit_sha() - assert original_commit != new_origin_commit + assert client_auth_repo.head_commit_sha() != origin_auth_repo.head_commit_sha() setup_manager.add_task(pull_client_auth_repo, kwargs={"client_dir": client_dir}) setup_manager.execute_tasks() + client_auth_commit = client_auth_repo.head_commit_sha() + assert client_auth_commit == origin_auth_repo.head_commit_sha() + update_and_check_commit_shas(OperationType.UPDATE, origin_auth_repo, client_dir) verify_client_repos_state(client_dir, origin_auth_repo) @@ -78,19 +83,31 @@ def test_target_repo_not_in_sync(origin_auth_repo, client_dir): origin_auth_repo, client_dir, ) - original_commit = origin_auth_repo.head_commit_sha() + + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) + origin_target_repos = load_target_repositories(origin_auth_repo) + client_target_repos = load_target_repositories(client_auth_repo) setup_manager = SetupManager(origin_auth_repo) setup_manager.add_task(add_valid_target_commits) setup_manager.add_task(update_expiration_dates) setup_manager.execute_tasks() - new_origin_commit = origin_auth_repo.head_commit_sha() - assert original_commit != new_origin_commit + for target_name in origin_target_repos: + assert ( + origin_target_repos[target_name].head_commit_sha() + != client_target_repos[target_name].head_commit_sha() + ) setup_manager.add_task(pull_all_target_repos, kwargs={"client_dir": client_dir}) setup_manager.execute_tasks() + for target_name in origin_target_repos: + assert ( + origin_target_repos[target_name].head_commit_sha() + == client_target_repos[target_name].head_commit_sha() + ) + # Run the updater to update repositories update_and_check_commit_shas( OperationType.UPDATE, @@ -122,37 +139,30 @@ def test_auth_repo_not_in_sync_partial(origin_auth_repo, client_dir): """ clone_repositories(origin_auth_repo, client_dir) - # Load the target repositories - target_repos = list(load_target_repositories(origin_auth_repo).values()) - original_commits = {repo.name: repo.head_commit_sha() for repo in target_repos} - original_commit = origin_auth_repo.head_commit_sha() + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) setup_manager = SetupManager(origin_auth_repo) - setup_manager.add_task(add_valid_unauthenticated_commits) setup_manager.add_task(add_valid_target_commits) setup_manager.add_task( - update_role_metadata_without_signing, kwargs={"role": "root"} + update_role_metadata_invalid_signature, kwargs={"role": "timestamp"} ) setup_manager.execute_tasks() - new_commits = {repo.name: repo.head_commit_sha() for repo in target_repos} - - for repo_name in original_commits: - assert original_commits[repo_name] != new_commits[repo_name] - - new_origin_commit = origin_auth_repo.head_commit_sha() - assert original_commit != new_origin_commit - - origin_auth_repo.commit("Committing uncommitted changes before pull") + assert client_auth_repo.head_commit_sha() != origin_auth_repo.head_commit_sha() setup_manager.add_task(pull_client_auth_repo, kwargs={"client_dir": client_dir}) setup_manager.execute_tasks() - update_and_check_commit_shas( - OperationType.UPDATE, origin_auth_repo, client_dir, force=True - ) + assert client_auth_repo.head_commit_sha() == origin_auth_repo.head_commit_sha() - verify_partial_update(client_dir, origin_auth_repo, original_commits) + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + None, + True, + ) + verify_partial_auth_update(client_dir, origin_auth_repo) verify_client_repos_state(client_dir, origin_auth_repo) @@ -161,8 +171,8 @@ def test_auth_repo_not_in_sync_partial(origin_auth_repo, client_dir): [ { "targets_config": [ - {"name": "target1", "allow_unauthenticated_commits": True}, - {"name": "target2", "allow_unauthenticated_commits": True}, + {"name": "target1"}, + {"name": "target2"}, ], }, ], @@ -177,24 +187,42 @@ def test_target_repo_not_in_sync_partial(origin_auth_repo, client_dir): origin_auth_repo, client_dir, ) - target_repos = list(load_target_repositories(origin_auth_repo).values()) - original_commits = {repo.name: repo.head_commit_sha() for repo in target_repos} + + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) + origin_target_repos = load_target_repositories(origin_auth_repo) + client_target_repos = load_target_repositories(client_auth_repo) setup_manager = SetupManager(origin_auth_repo) - setup_manager.add_task(add_valid_unauthenticated_commits) + setup_manager.add_task(add_valid_target_commits) + setup_manager.add_task(add_unauthenticated_commits_to_all_target_repos) setup_manager.add_task(add_valid_target_commits) setup_manager.execute_tasks() - new_commits = {repo.name: repo.head_commit_sha() for repo in target_repos} - for repo_name in original_commits: - assert original_commits[repo_name] != new_commits[repo_name] + for target_name in origin_target_repos: + assert ( + origin_target_repos[target_name].head_commit_sha() + != client_target_repos[target_name].head_commit_sha() + ) setup_manager.add_task(pull_all_target_repos, kwargs={"client_dir": client_dir}) setup_manager.execute_tasks() - update_and_check_commit_shas(OperationType.UPDATE, origin_auth_repo, client_dir) + for target_name in origin_target_repos: + assert ( + origin_target_repos[target_name].head_commit_sha() + == client_target_repos[target_name].head_commit_sha() + ) - verify_partial_update(client_dir, origin_auth_repo, original_commits) + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + None, + True, + ) + + verify_partial_auth_update(client_dir, origin_auth_repo) + verify_partial_targets_update(client_dir, origin_auth_repo) verify_client_repos_state(client_dir, origin_auth_repo) @@ -215,100 +243,49 @@ def test_mixed_target_repo_states(origin_auth_repo, client_dir): Client's auth repo is on last validated commit, but target commits are all over the place. Some are after last validated, some are before. """ - clone_repositories( + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(add_valid_target_commits) + setup_manager.add_task(add_valid_target_commits) + setup_manager.execute_tasks() + + update_and_check_commit_shas( + OperationType.CLONE, origin_auth_repo, client_dir, + expected_repo_type=UpdateType.EITHER, ) - target_repos = list(load_target_repositories(origin_auth_repo).values()) - original_commits = {repo.name: repo.head_commit_sha() for repo in target_repos} - reverted_repo = target_repos[0] # target1 - updated_repo = target_repos[1] # target2 + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) + origin_target_repos = load_target_repositories(origin_auth_repo) + client_target_repos = load_target_repositories(client_auth_repo) + + client_target_repos = list(client_target_repos.values()) + reverted_repo = client_target_repos[0] # target1 + updated_repo = client_target_repos[1] # target2 + old_commit = reverted_repo.head_commit_sha() # Add valid commits first to - setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(add_valid_target_commits) setup_manager.add_task( remove_commits, kwargs={"repo_path": reverted_repo.path, "num_commits": 1} ) setup_manager.add_task( - update_existing_file, - kwargs={ - "repo": updated_repo, - "filename": "test1.txt", - "commit_message": "Manually update test1.txt", - }, + pull_specific_target_repo, kwargs={"repo_path": updated_repo.path} ) setup_manager.execute_tasks() - # Check that the reverted repo has the same commit as the original - reverted_commit = reverted_repo.head_commit_sha() - assert original_commits[reverted_repo.name] == reverted_commit - - # Check that the manually updated repo has a new commit - updated_commit = updated_repo.head_commit_sha() - assert original_commits[updated_repo.name] != updated_commit - setup_manager.add_task(pull_client_auth_repo, kwargs={"client_dir": client_dir}) setup_manager.execute_tasks() + updated_repo.head_commit_sha() == origin_target_repos[ + updated_repo.name + ].head_commit_sha() + reverted_repo.head_commit_sha() != old_commit + update_and_check_commit_shas( OperationType.UPDATE, origin_auth_repo, client_dir, force=True ) verify_client_repos_state(client_dir, origin_auth_repo) - - -@pytest.mark.parametrize( - "origin_auth_repo", - [ - { - "targets_config": [ - {"name": "target1", "allow_unauthenticated_commits": True}, - {"name": "target2", "allow_unauthenticated_commits": True}, - ], - }, - ], - indirect=True, -) -def test_target_repo_mixed_manual_updates(origin_auth_repo, client_dir): - """ - Some target repositories are updated manually, and some are not. - Expect that manually updated repositories have new commits, while others do not. - """ - clone_repositories( - origin_auth_repo, - client_dir, - ) - - target_repos = list(load_target_repositories(origin_auth_repo).values()) - original_commits = {repo.name: repo.head_commit_sha() for repo in target_repos} - updated_repo = target_repos[0] # target1 - reverted_repo = target_repos[1] # target2 - - # Add commits to revert later - setup_manager = SetupManager(origin_auth_repo) - - setup_manager.add_task( - add_valid_target_commits, kwargs={"target_repos": [updated_repo]} - ) - setup_manager.add_task( - remove_commits, kwargs={"repo_path": reverted_repo.path, "num_commits": 1} - ) - setup_manager.execute_tasks() - # Check that the reverted repo has the same commit as the original - reverted_commit = reverted_repo.head_commit_sha() - assert original_commits[reverted_repo.name] == reverted_commit - - # Check that the manually updated repo has a new commit - updated_commit = updated_repo.head_commit_sha() - assert original_commits[updated_repo.name] != updated_commit - - pull_specific_target_repo(client_dir, reverted_repo.name) - pull_specific_target_repo(client_dir, updated_repo.name) - - setup_manager.execute_tasks() - - update_and_check_commit_shas(OperationType.UPDATE, origin_auth_repo, client_dir) - - verify_client_repos_state(client_dir, origin_auth_repo) diff --git a/taf/tests/test_updater/update_utils.py b/taf/tests/test_updater/update_utils.py index 31765022..daf0b033 100644 --- a/taf/tests/test_updater/update_utils.py +++ b/taf/tests/test_updater/update_utils.py @@ -319,40 +319,25 @@ def verify_client_repos_state( origin_auth_repo, library_dir=client_dir ) - check_last_validated_commit(client_auth_repo.path) + # check if the target repositoies are in sync with the auth repo - successful_update = True for repo_name, client_repo in client_target_repos.items(): client_commit = client_repo.head_commit_sha() - origin_commit = origin_auth_repo.head_commit_sha() - - if client_commit != origin_commit: - successful_update = False - break - if successful_update: - check_if_commits_match(client_target_repos, origin_auth_repo.path.parent.parent) - else: - for repo_name, client_repo in client_target_repos.items(): - client_commit = client_repo.head_commit_sha() - - # Extract commit SHA from the target file in the client repo - target_commit_info = client_auth_repo.get_target(repo_name) - target_commit_sha = ( - target_commit_info.get("commit") if target_commit_info else None - ) + # Extract commit SHA from the target file in the client repo + target_commit_info = client_auth_repo.get_target(repo_name) + target_commit_sha = ( + target_commit_info.get("commit") if target_commit_info else None + ) - # Assert that the top commits of target repositories are the same as the commit SHA specified in the corresponding target files - assert ( - client_commit == target_commit_sha - ), f"Target repo {repo_name} should have the same top commit as specified in the corresponding target file" - assert ( - client_commit != origin_commit - ), f"Target repo {repo_name} should not have the same commit as specified in the client's auth repo" + # Assert that the top commits of target repositories are the same as the commit SHA specified in the corresponding target files + assert ( + client_commit == target_commit_sha + ), f"Target repo {repo_name} should have the same top commit as specified in the corresponding target file" -def verify_partial_update( - client_dir: Path, origin_auth_repo: AuthenticationRepository, original_commits: dict +def verify_partial_auth_update( + client_dir: Path, origin_auth_repo: AuthenticationRepository ): """ Verify that the client's repositories are in the correct state following a partial update. @@ -361,13 +346,24 @@ def verify_partial_update( commit as specified in the client's auth repo. """ client_auth_repo = AuthenticationRepository(path=client_dir / origin_auth_repo.name) - client_target_repos = load_target_repositories( - origin_auth_repo, library_dir=client_dir - ) # Ensure the last validated commit exists in the client's auth repo check_last_validated_commit(client_auth_repo.path) + client_head_sha = client_auth_repo.head_commit_sha() + assert client_head_sha != origin_auth_repo.head_commit_sha() + assert client_head_sha in origin_auth_repo.all_commits_on_branch() + + +def verify_partial_targets_update( + client_dir: Path, origin_auth_repo: AuthenticationRepository +): + + client_auth_repo = AuthenticationRepository(path=client_dir / origin_auth_repo.name) + + client_target_repos = load_target_repositories( + origin_auth_repo, library_dir=client_dir + ) for repo_name, client_repo in client_target_repos.items(): client_commit = client_repo.head_commit_sha() origin_commit = origin_auth_repo.head_commit_sha() diff --git a/taf/updater/updater_pipeline.py b/taf/updater/updater_pipeline.py index 7672fe30..e92118ec 100644 --- a/taf/updater/updater_pipeline.py +++ b/taf/updater/updater_pipeline.py @@ -1339,7 +1339,7 @@ def merge_commits(self): last_validated_commit = validated_commits[-1] commit_to_merge = last_validated_commit _merge_commit( - repository, branch, commit_to_merge, force_revert=self.force + repository, branch, commit_to_merge, force_revert=True ) return self.state.update_status except Exception as e: