Merge branch 'master' into ronantakizawa/customkeysizes

CHANGELOG.md

@@ -10,10 +10,12 @@ and this project adheres to [Semantic Versioning][semver].
 ### Added
 
 - Removed 2048-bit key restriction [494]
-- Added lazy loading to CLI [481]
+- Allow for the displaying of varied levels of log and debug information based on the verbosity level ([493])
+- Added new tests to test out of sync repositories and manual updates ([488], [504])
+- Added lazy loading to CLI ([481])
 - Testing repositories with dependencies ([479], [487])
-- Hid plaintext when users are prompted to insert YubiKey and press ENTER [473]
-- Added functionality for parallel execution of child repo during clone and update for performance enhancement [472]
+- Hid plaintext when users are prompted to insert YubiKey and press ENTER ([473])
+- Added functionality for parallel execution of child repo during clone and update for performance enhancement ([472])
 - New flag --force allowing forced updates ([471])
 - Improved usability (TAF finds the repo if current directory has no repo, create a .taf directory to manage keys) ([466])
 - Added git hook check for updater ([460])
@@ -36,10 +38,13 @@ and this project adheres to [Semantic Versioning][semver].
 
 ### Fixed
 
+[504]: https://github.com/openlawlibrary/taf/pull/504
 [494]: https://github.com/openlawlibrary/taf/pull/494
-[487]: https://github.com/openlawlibrary/taf/pull/489
+[493]: https://github.com/openlawlibrary/taf/pull/493
+[489]: https://github.com/openlawlibrary/taf/pull/489
+[488]: https://github.com/openlawlibrary/taf/pull/488
 [487]: https://github.com/openlawlibrary/taf/pull/487
-[487]: https://github.com/openlawlibrary/taf/pull/485
+[485]: https://github.com/openlawlibrary/taf/pull/485
 [481]: https://github.com/openlawlibrary/taf/pull/481
 [479]: https://github.com/openlawlibrary/taf/pull/479
 [473]: https://github.com/openlawlibrary/taf/pull/473

setup.py

@@ -62,7 +62,7 @@
         "oll-tuf==0.20.0.dev2",
         "cryptography==38.0.*",
         "securesystemslib==0.25.*",
-        "loguru==0.6.*",
+        "loguru==0.7.*",
         pygit2_version,
         "pyOpenSSL==22.1.*",
         "logdecorator==2.*",

taf/git.py

@@ -106,12 +106,12 @@ def __init__(
 
     @property
     def pygit(self):
-        if not self.is_git_repository:
-            raise GitError(
-                self,
-                message=f"The path '{self.path.as_posix()}' is not a Git repository.",
-            )
         if self._pygit is None:
+            if not self.is_git_repository:
+                raise GitError(
+                    self,
+                    message=f"The path '{self.path.as_posix()}' is not a Git repository.",
+                )
             try:
                 self._pygit = PyGitRepository(self)
                 if not self._pygit:
@@ -824,7 +824,10 @@ def is_branch_with_unpushed_commits(self, branch_name):
         if local_branch is None:
             # local branch does not exist
             return False
-        upstream_full_name = local_branch.upstream_name
+        try:
+            upstream_full_name = local_branch.upstream_name
+        except KeyError:
+            return True
         if not upstream_full_name:
             # no upstream branch - not pushed
             return True

taf/log.py

@@ -9,7 +9,6 @@
 import tuf.repository_tool
 import tuf.exceptions
 from loguru import logger as taf_logger
-
 import taf.settings as settings
 
 _CONSOLE_FORMAT_STRING = "\n{message}\n"
@@ -18,6 +17,31 @@
 console_loggers: Dict = {}
 file_loggers: Dict = {}
 
+NOTICE = 25
+taf_logger.level("NOTICE", no=NOTICE, color="<yellow>", icon="!")
+
+VERBOSITY_LEVELS = {
+    0: "NOTICE",  # Default
+    1: "INFO",  # -v
+    2: "DEBUG",  # -vv
+    3: "TRACE",  # -vvv
+}
+
+
+def formatter(record):
+    if record["level"].no == NOTICE:
+        return f"<white>{_CONSOLE_FORMAT_STRING}</white>"
+    elif record["level"].no == logging.WARNING:
+        return f"<yellow>{_CONSOLE_FORMAT_STRING}</yellow>"
+    elif record["level"].no == logging.INFO:
+        return f"<blue>{_CONSOLE_FORMAT_STRING}</blue>"
+    elif record["level"].no == logging.DEBUG:
+        return f"<magenta>{_CONSOLE_FORMAT_STRING}</magenta>"
+    elif record["level"].no == logging.ERROR:
+        return f"<red>{_CONSOLE_FORMAT_STRING}</red>"
+    else:
+        return _CONSOLE_FORMAT_STRING
+
 
 def disable_console_logging():
     try:
@@ -63,36 +87,38 @@ def _get_log_location():
     return location
 
 
-taf_logger.remove()
+def initialize_logger_handlers():
+    taf_logger.remove()
+    if settings.ENABLE_CONSOLE_LOGGING:
+        console_loggers["log"] = taf_logger.add(
+            sys.stdout, format=formatter, level=VERBOSITY_LEVELS[settings.VERBOSITY]
+        )
+        tuf.log.set_console_log_level(logging.ERROR)
+    else:
+        # if console logging is disable, remove tuf console logger
+        disable_tuf_console_logging()
 
-if settings.ENABLE_CONSOLE_LOGGING:
-    console_loggers["log"] = taf_logger.add(
-        sys.stdout, format=_CONSOLE_FORMAT_STRING, level=settings.CONSOLE_LOGGING_LEVEL
-    )
-    tuf.log.set_console_log_level(settings.CONSOLE_LOGGING_LEVEL)
-else:
-    # if console logging is disable, remove tuf console logger
-    disable_tuf_console_logging()
+    if settings.ENABLE_FILE_LOGGING:
+        log_location = _get_log_location()
+        log_path = str(log_location / settings.LOG_FILENAME)
+        file_loggers["log"] = taf_logger.add(
+            log_path, format=_FILE_FORMAT_STRING, level=settings.FILE_LOGGING_LEVEL
+        )
 
+        if settings.SEPARATE_ERRORS:
+            error_log_path = str(log_location / settings.ERROR_LOG_FILENAME)
+            file_loggers["error"] = taf_logger.add(
+                error_log_path,
+                format=_FILE_FORMAT_STRING,
+                level=settings.ERROR_LOGGING_LEVEL,
+            )
+        try:
+            tuf.log.set_filehandler_log_level(settings.FILE_LOGGING_LEVEL)
+        except tuf.exceptions.Error:
+            pass
+    else:
+        # if file logging is disabled, also disable tuf file logging
+        disable_tuf_file_logging()
 
-if settings.ENABLE_FILE_LOGGING:
-    logs_location = _get_log_location()
-    log_path = str(logs_location / settings.LOG_FILENAME)
-    file_loggers["log"] = taf_logger.add(
-        log_path, format=_FILE_FORMAT_STRING, level=settings.FILE_LOGGING_LEVEL
-    )
 
-    if settings.SEPARATE_ERRORS:
-        error_log_path = str(logs_location / settings.ERROR_LOG_FILENAME)
-        file_loggers["error"] = taf_logger.add(
-            error_log_path,
-            format=_FILE_FORMAT_STRING,
-            level=settings.ERROR_LOGGING_LEVEL,
-        )
-    try:
-        tuf.log.set_filehandler_log_level(settings.FILE_LOGGING_LEVEL)
-    except tuf.exceptions.Error:
-        pass
-else:
-    # if file logging is disabled, also disable tuf file logging
-    disable_tuf_file_logging()
+initialize_logger_handlers()

taf/pygit.py

@@ -1,7 +1,7 @@
 from typing import Dict
 import pygit2
 from collections import defaultdict
-from taf.log import taf_logger as logger
+from taf.log import taf_logger
 from taf.exceptions import GitError
 import os.path
 
@@ -51,12 +51,14 @@ def _get_blob_at_path(self, obj, path):
         for the given commit object,
         get the blob at the given path
         """
-        logger.debug("Get blob at path %s", path)
+        taf_logger.debug(
+            f"Get blob at path {path}",
+        )
         working = self._get_object_at_path(obj, path)
         if working and isinstance(working, pygit2.Blob):
-            logger.debug("Found blob at path %s", "/".join(path))
+            taf_logger.debug(f"Found blob at path {'/'.join(path)}")
             return working
-        logger.debug("Blob not found at path %s", "/".join(path))
+        taf_logger.debug(f"Blob not found at path {'/'.join(path)}")
         return None
 
     def cleanup(self):

taf/settings.py

@@ -59,3 +59,5 @@
 ERROR_LOG_FILENAME = "taf.err"
 
 LOG_COMMAND_OUTPUT = False
+
+VERBOSITY = 1

taf/tests/test_updater/conftest.py

@@ -158,6 +158,9 @@ def execute_tasks(self):
                         task.function(**task.params)
                 else:
                     task.function(**task.params)
+        # remove all tasks once they are all executed
+        # allow for the reuse of the setup manager
+        self.tasks = []
         repositoriesdb.clear_repositories_db()
 
 
@@ -566,6 +569,19 @@ def update_role_metadata_without_signing(
     )
 
 
+def update_existing_file(repo: GitRepository, filename: str, commit_message: str):
+    text_to_add = _generate_random_text()
+    file_path = repo.path / filename
+    if file_path.exists():
+        with file_path.open("a") as file:
+            file.write(f"\n{text_to_add}")
+        repo.commit(commit_message)
+    else:
+        raise FileNotFoundError(
+            f"The file {filename} does not exist in the repository {repo.path}"
+        )
+
+
 def update_role_metadata_invalid_signature(
     auth_repo: AuthenticationRepository, role: str
 ):
@@ -635,7 +651,12 @@ def add_file_without_commit(repo_path: str, filename: str):
         file.write(text_to_add)
 
 
-def remove_commits(repo_path: str, num_commits: int = 1):
+def remove_commits(
+    auth_repo: AuthenticationRepository,
+    target_repos: list,
+    repo_path: str,
+    num_commits: int = 1,
+):
     repo = GitRepository(path=Path(repo_path))
 
     try:
@@ -665,3 +686,22 @@ def set_head_commit(auth_repo: AuthenticationRepository):
         auth_repo.set_last_validated_commit(last_valid_commit)
     else:
         raise ValueError("Failed to retrieve the last valid commit SHA.")
+
+
+def pull_specific_target_repo(
+    auth_repo: AuthenticationRepository, target_repos: list, repo_path: str
+):
+    client_target_repo = GitRepository(path=repo_path)
+    client_target_repo.pull()
+    return
+
+
+def pull_all_target_repos(auth_repo: AuthenticationRepository, client_dir: Path):
+    client_target_repos = load_target_repositories(auth_repo, library_dir=client_dir)
+    for _, client_repo in client_target_repos.items():
+        client_repo.pull()
+
+
+def pull_client_auth_repo(auth_repo: AuthenticationRepository, client_dir: Path):
+    client_auth_repo = AuthenticationRepository(client_dir, auth_repo.name)
+    client_auth_repo.pull()

taf/tests/test_updater/test_update/test_update_invalid.py

@@ -204,7 +204,11 @@ def test_remove_commits_from_target_repo(origin_auth_repo, client_dir):
 
     client_target_repo_path = client_dir / origin_auth_repo.name
 
-    remove_commits(str(client_target_repo_path))
+    setup_manager = SetupManager(origin_auth_repo)
+    setup_manager.add_task(
+        remove_commits, kwargs={"repo_path": client_target_repo_path, "num_commits": 1}
+    )
+    setup_manager.execute_tasks()
 
     update_invalid_repos_and_check_if_repos_exist(
         OperationType.UPDATE,

taf/tests/test_updater/test_update/test_update_valid.py

@@ -532,7 +532,11 @@ def test_update_valid_remove_commits_from_target_repo(origin_auth_repo, client_d
         / "targets/test_remove_commits_from_target_repo0/target1"
     )
 
-    remove_commits(str(client_target_repo_path))
+    setup_manager = SetupManager(origin_auth_repo)
+    setup_manager.add_task(
+        remove_commits, kwargs={"repo_path": client_target_repo_path, "num_commits": 1}
+    )
+    setup_manager.execute_tasks()
 
     update_and_check_commit_shas(
         OperationType.UPDATE,