From 7e9482edf095a6d529238ced33cc78751e2e5aee Mon Sep 17 00:00:00 2001 From: Adam McKellar Date: Mon, 27 May 2024 12:19:57 +0200 Subject: [PATCH 01/13] Comments for Kanushka to get started. --- installation_instruction/__main__.py | 12 ++++++++++++ installation_instruction/helpers.py | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/installation_instruction/__main__.py b/installation_instruction/__main__.py index e23c739..ef952b0 100644 --- a/installation_instruction/__main__.py +++ b/installation_instruction/__main__.py @@ -43,6 +43,18 @@ def __init__(self, *args, **kwargs): ) def get_command(self, ctx, config_file: str) -> click.Command|None: + + # @Kanushka add here your logic. I thought of the following steps: + # Check if config_file is an url. + # If yes create a temporary dir. Or do something equivalent. + # Find out if said url is a file or a git repo. + # If is a file then download it to the temporary dir. + # If is a git repo then clone it to the temporary dir and find the actual config file "install.cfg". + # Overwrite config_file with the path to the config file. + # + # It be nice if you implement and use the `function helpers._find_config_file_in_folder`. + # Please take a look into helpers.py for the functions definition. + if not isfile(config_file): click.echo("Config file not found.") return None diff --git a/installation_instruction/helpers.py b/installation_instruction/helpers.py index bb0dd12..cf43763 100644 --- a/installation_instruction/helpers.py +++ b/installation_instruction/helpers.py @@ -16,6 +16,12 @@ from jinja2 import Environment, Template +def _find_config_file_in_folder(folder_path: str) -> str | None: + """ + Finds file with the name `install.cfg` in the folder and returns its path if it exists. + """ + pass + def _make_pretty_print_line_breaks(string: str) -> str: """ Replaces `&& ` with a newline character. From 48a813a8137a856a5e8ee005fe859db748410dc5 Mon Sep 17 00:00:00 2001 From: Kanushka Gupta Date: Tue, 4 Jun 2024 01:15:49 +0200 Subject: [PATCH 02/13] Helper working fine. Git repo cloning to temp directory --- installation_instruction/__main__.py | 37 ++++++++++++------ installation_instruction/helpers.py | 57 ++++++++++++++++++++++++---- 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/installation_instruction/__main__.py b/installation_instruction/__main__.py index ef952b0..6613f7a 100644 --- a/installation_instruction/__main__.py +++ b/installation_instruction/__main__.py @@ -18,10 +18,18 @@ import click +from urllib.parse import urlparse +import git +import tempfile +import shutil +import os + from .__init__ import __version__, __description__, __repository__, __author__, __author_email__, __license__ from .get_flags_and_options_from_schema import get_flags_and_options from .installation_instruction import InstallationInstruction from .helpers import _make_pretty_print_line_breaks +from .helpers import _find_config_file_in_folder + VERSION_STRING = f"""Version: installation-instruction {__version__} Copyright: (C) 2024 {__author_email__}, {__author__} @@ -44,21 +52,28 @@ def __init__(self, *args, **kwargs): def get_command(self, ctx, config_file: str) -> click.Command|None: - # @Kanushka add here your logic. I thought of the following steps: - # Check if config_file is an url. - # If yes create a temporary dir. Or do something equivalent. - # Find out if said url is a file or a git repo. - # If is a file then download it to the temporary dir. - # If is a git repo then clone it to the temporary dir and find the actual config file "install.cfg". - # Overwrite config_file with the path to the config file. - # - # It be nice if you implement and use the `function helpers._find_config_file_in_folder`. - # Please take a look into helpers.py for the functions definition. + if not isfile(config_file): + # Create a temporary directory + temp_dir = tempfile.TemporaryDirectory() + repo_dir = temp_dir.name + print("hello") + git.Repo.clone_from(self.path, repo_dir) + + # It's a URL, handle it accordingly + #repo_dir, temp_dir = clone_git_repo(folder_path) + '''config_file_path = _find_config_file_in_folder(config_file) + + if config_file_path: + print("Config file found:", config_file_path) + else: + print("Config file not found 12334:", config_file) + return None + if not isfile(config_file): click.echo("Config file not found.") return None - + ''' try: instruction = InstallationInstruction.from_file(config_file) options = get_flags_and_options(instruction.schema) diff --git a/installation_instruction/helpers.py b/installation_instruction/helpers.py index cf43763..ebcd8f8 100644 --- a/installation_instruction/helpers.py +++ b/installation_instruction/helpers.py @@ -15,14 +15,52 @@ import re from jinja2 import Environment, Template +from urllib.parse import urlparse +import git +import tempfile +import shutil +import os + +def is_git_repo_url(url): + parsed_url = urlparse(url) + return all([parsed_url.scheme, parsed_url.netloc, parsed_url.path]) and url.endswith('.git') + +def clone_git_repo(url): + # Create a temporary directory + temp_dir = tempfile.TemporaryDirectory() + repo_dir = temp_dir.name + git.Repo.clone_from(url, repo_dir) + return repo_dir, temp_dir + +def check_and_download_install_cfg(repo_dir): + install_cfg_path = os.path.join(repo_dir, 'install.cfg') + if os.path.isfile(install_cfg_path): + temp_file_dir = tempfile.mkdtemp() + temp_file_path = os.path.join(temp_file_dir, 'install.cfg') + shutil.copy(install_cfg_path, temp_file_path) + return temp_file_path, temp_file_dir + return None, None def _find_config_file_in_folder(folder_path: str) -> str | None: """ Finds file with the name `install.cfg` in the folder and returns its path if it exists. """ - pass - -def _make_pretty_print_line_breaks(string: str) -> str: + if folder_path.startswith('http://') or folder_path.startswith('https://'): + # It's a URL, handle it accordingly + repo_dir, temp_dir = clone_git_repo(folder_path) + install_cfg_path, install_cfg_temp_dir = check_and_download_install_cfg(repo_dir) + if install_cfg_path: + print(f"install.cfg found and downloaded to: {install_cfg_path}") + return install_cfg_path + else: + print("install.cfg not found in the repository root.") + # Clean up the cloned repository + temp_dir.cleanup() + elif os.path.isfile(folder_path): + return folder_path + return None + +def make_pretty_print_line_breaks(string): """ Replaces `&& ` with a newline character. @@ -33,7 +71,7 @@ def _make_pretty_print_line_breaks(string: str) -> str: """ return re.sub(r"\s?&&\s?", "\n", string, 0, re.S) -def _get_error_message_from_string(string: str) -> str | None: +def get_error_message_from_string(string): """ Parses error message of error given by using jinja macro `RAISE_JINJA_MACRO_STRING`. If no error message is found returns `None`. @@ -48,18 +86,18 @@ def _get_error_message_from_string(string: str) -> str | None: return None return matches.group("errmsg") -def _replace_whitespace_in_string(string: str) -> str: +def replace_whitespace_in_string(string): """ Replaces eol and whitespaces of a string with a single whitespace. :param string: String to be processed. :type string: str - :return: String where whitespace and eol is replaced with one whitespace and whitspace before and after are stripped. + :return: String where whitespace and eol is replaced with one whitespace and whitespace before and after are stripped. :rtype: str """ return re.sub(r"\s{1,}", " ", string, 0, re.S).strip() -def _split_string_at_delimiter(string: str) -> tuple[str, str]: +def split_string_at_delimiter(string): """ Extracts part before and after the delimiter "------" or more. @@ -78,7 +116,7 @@ def _split_string_at_delimiter(string: str) -> tuple[str, str]: matches.group("template") ) -def _load_template_from_string(string: str) -> Template: +def load_template_from_string(string): """ Returns `jinja2.Template`. @@ -92,3 +130,6 @@ def _load_template_from_string(string: str) -> Template: lstrip_blocks=True ) return env.from_string(string) + + +_find_config_file_in_folder('https://github.com/KanushkaGupta/sample.git') \ No newline at end of file From 40a24ac0b297812d212a8f17d456e2675c0f6173 Mon Sep 17 00:00:00 2001 From: Kanushka Gupta Date: Tue, 4 Jun 2024 01:46:17 +0200 Subject: [PATCH 03/13] Updated Main --- installation_instruction/__main__.py | 33 +++++----------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/installation_instruction/__main__.py b/installation_instruction/__main__.py index 6613f7a..62ee6ad 100644 --- a/installation_instruction/__main__.py +++ b/installation_instruction/__main__.py @@ -18,12 +18,6 @@ import click -from urllib.parse import urlparse -import git -import tempfile -import shutil -import os - from .__init__ import __version__, __description__, __repository__, __author__, __author_email__, __license__ from .get_flags_and_options_from_schema import get_flags_and_options from .installation_instruction import InstallationInstruction @@ -52,28 +46,11 @@ def __init__(self, *args, **kwargs): def get_command(self, ctx, config_file: str) -> click.Command|None: - if not isfile(config_file): - # Create a temporary directory - temp_dir = tempfile.TemporaryDirectory() - repo_dir = temp_dir.name - print("hello") - git.Repo.clone_from(self.path, repo_dir) - - # It's a URL, handle it accordingly - #repo_dir, temp_dir = clone_git_repo(folder_path) - - '''config_file_path = _find_config_file_in_folder(config_file) - - if config_file_path: - print("Config file found:", config_file_path) - else: - print("Config file not found 12334:", config_file) - return None - - if not isfile(config_file): + config_file_path = _find_config_file_in_folder(config_file) + if not isfile(config_file_path): click.echo("Config file not found.") return None - ''' + try: instruction = InstallationInstruction.from_file(config_file) options = get_flags_and_options(instruction.schema) @@ -111,7 +88,6 @@ def callback(**kwargs): callback=callback, ) - @click.command(cls=ConfigReadCommand, help="Shows installation instructions for your specified config file and parameters.") @click.option("--raw", is_flag=True, help="Show installation instructions without pretty print.", default=False) @click.pass_context @@ -136,4 +112,5 @@ def main(ctx): main.add_command(install) if __name__ == "__main__": - main() \ No newline at end of file + main() + From d4c32a1845192b8a7c5129a47b4602c182696eab Mon Sep 17 00:00:00 2001 From: Kanushka Gupta Date: Tue, 4 Jun 2024 19:08:05 +0200 Subject: [PATCH 04/13] Update --- installation_instruction/__main__.py | 4 ++-- installation_instruction/helpers.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/installation_instruction/__main__.py b/installation_instruction/__main__.py index 62ee6ad..602b3ce 100644 --- a/installation_instruction/__main__.py +++ b/installation_instruction/__main__.py @@ -46,8 +46,8 @@ def __init__(self, *args, **kwargs): def get_command(self, ctx, config_file: str) -> click.Command|None: - config_file_path = _find_config_file_in_folder(config_file) - if not isfile(config_file_path): + config_file = _find_config_file_in_folder(config_file) + if not isfile(config_file): click.echo("Config file not found.") return None diff --git a/installation_instruction/helpers.py b/installation_instruction/helpers.py index ebcd8f8..27b7107 100644 --- a/installation_instruction/helpers.py +++ b/installation_instruction/helpers.py @@ -58,7 +58,7 @@ def _find_config_file_in_folder(folder_path: str) -> str | None: temp_dir.cleanup() elif os.path.isfile(folder_path): return folder_path - return None + return folder_path def make_pretty_print_line_breaks(string): """ From 0158714ab82d56568f59535918bd942e8de3b078 Mon Sep 17 00:00:00 2001 From: Kanushka Gupta Date: Tue, 4 Jun 2024 19:10:18 +0200 Subject: [PATCH 05/13] Updated toml --- pyproject.toml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d82f2aa..d2f4e19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,12 +16,12 @@ description = "Library and CLI for generating installation instructions from jso readme = "README.md" maintainers = [ { name = "Adam McKellar", email = "dev@mckellar.eu" }, - { name = "Kanushka Gupta" }, + { name = "Kanushka Gupta" email = "kanushkagupta1298@gmail.com" }, { name = "Timo Ege", email = "timoege@online.de" }, ] authors = [ { name = "Adam McKellar", email = "dev@mckellar.eu" }, - { name = "Kanushka Gupta" }, + { name = "Kanushka Gupta", email = "kanushkagupta1298@gmail.com" }, { name = "Timo Ege", email = "timoege@online.de" }, ] version = "0.1.1" @@ -37,6 +37,11 @@ dependencies = [ "jsonschema", "PyYAML", "click < 9.0.0a0", + "urlparse", + "git", + "tempfile", + "shutil", + "os" ] [project.optional-dependencies] From e39e6635274f2dd972ba552c6a53dec5ba822358 Mon Sep 17 00:00:00 2001 From: Adam McKellar Date: Tue, 4 Jun 2024 19:13:58 +0200 Subject: [PATCH 06/13] Removed unneeded dependencies. --- pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d2f4e19..7143c32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,11 +37,7 @@ dependencies = [ "jsonschema", "PyYAML", "click < 9.0.0a0", - "urlparse", - "git", - "tempfile", - "shutil", - "os" + "GitPython", ] [project.optional-dependencies] From 0bb4ba9268482410359a4f4a25cde676de150562 Mon Sep 17 00:00:00 2001 From: Adam McKellar Date: Tue, 4 Jun 2024 19:15:29 +0200 Subject: [PATCH 07/13] Fixed typo in pyproject.toml. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7143c32..5e5bfa1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ description = "Library and CLI for generating installation instructions from jso readme = "README.md" maintainers = [ { name = "Adam McKellar", email = "dev@mckellar.eu" }, - { name = "Kanushka Gupta" email = "kanushkagupta1298@gmail.com" }, + { name = "Kanushka Gupta", email = "kanushkagupta1298@gmail.com" }, { name = "Timo Ege", email = "timoege@online.de" }, ] authors = [ From d7990e1b572ea2a4197191e7e636408fd8186e95 Mon Sep 17 00:00:00 2001 From: Kanushka Gupta Date: Tue, 4 Jun 2024 19:23:15 +0200 Subject: [PATCH 08/13] Fixed typo in helpers --- installation_instruction/helpers.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/installation_instruction/helpers.py b/installation_instruction/helpers.py index 27b7107..9b7fa07 100644 --- a/installation_instruction/helpers.py +++ b/installation_instruction/helpers.py @@ -1,3 +1,4 @@ + # Copyright 2024 Adam McKellar, Kanushka Gupta, Timo Ege # Licensed under the Apache License, Version 2.0 (the "License"); @@ -60,7 +61,7 @@ def _find_config_file_in_folder(folder_path: str) -> str | None: return folder_path return folder_path -def make_pretty_print_line_breaks(string): +def _make_pretty_print_line_breaks(string: str) -> str: """ Replaces `&& ` with a newline character. @@ -71,7 +72,7 @@ def make_pretty_print_line_breaks(string): """ return re.sub(r"\s?&&\s?", "\n", string, 0, re.S) -def get_error_message_from_string(string): +def _get_error_message_from_string(string: str) -> str | None: """ Parses error message of error given by using jinja macro `RAISE_JINJA_MACRO_STRING`. If no error message is found returns `None`. @@ -86,18 +87,18 @@ def get_error_message_from_string(string): return None return matches.group("errmsg") -def replace_whitespace_in_string(string): +def _replace_whitespace_in_string(string: str) -> str: """ Replaces eol and whitespaces of a string with a single whitespace. :param string: String to be processed. :type string: str - :return: String where whitespace and eol is replaced with one whitespace and whitespace before and after are stripped. + :return: String where whitespace and eol is replaced with one whitespace and whitspace before and after are stripped. :rtype: str """ return re.sub(r"\s{1,}", " ", string, 0, re.S).strip() -def split_string_at_delimiter(string): +def _split_string_at_delimiter(string: str) -> tuple[str, str]: """ Extracts part before and after the delimiter "------" or more. @@ -116,7 +117,7 @@ def split_string_at_delimiter(string): matches.group("template") ) -def load_template_from_string(string): +def _load_template_from_string(string: str) -> Template: """ Returns `jinja2.Template`. @@ -130,6 +131,3 @@ def load_template_from_string(string): lstrip_blocks=True ) return env.from_string(string) - - -_find_config_file_in_folder('https://github.com/KanushkaGupta/sample.git') \ No newline at end of file From 86c611a470ef1e3975af3e708b0976d12c1062ad Mon Sep 17 00:00:00 2001 From: Adam McKellar Date: Tue, 11 Jun 2024 21:18:14 +0200 Subject: [PATCH 09/13] Pretty Print Name and Description Outside of Schema (#15) * Moved descriptions and pretty print names of some keys out of schema. AnyOf is now not considered a special value. * Added documentation. * Added behaviour reading pretty print name and description from misc data for all keys. * Added changelog entry for added behaviour. * Added new description behaviour to cli. --- CHANGELOG.md | 6 + README.md | 23 ++- .../pytorch-instruction.schema.yml.jinja | 113 ++++++------ .../spacy/spacy-instruction.schema.yml.jinja | 171 +++++++++--------- installation_instruction/__main__.py | 2 +- .../get_flags_and_options_from_schema.py | 21 ++- .../installation_instruction.py | 36 ++-- tests/conftest.py | 2 +- tests/data/flags_options_example.schema.yml | 113 ++++++------ tests/data/test_install/install.cfg | 29 +-- .../test_get_flags_and_options_from_schema.py | 3 +- tests/test_installation_instruction.py | 4 +- 12 files changed, 284 insertions(+), 239 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40f6ba6..20d4198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* `title` and `description` can be set from `pretty` and `description` keys at root. + + ### Changed +* Moved `title` and `description` from `anyOf` to outside of schema. `anyOf` is now usable as intended for json schema. + + ### Removed diff --git a/README.md b/README.md index 18c9155..8da5127 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,28 @@ Options are dynamically created with the schema part of the config file. * When creating a schema use the following schema draft version: https://json-schema.org/draft/2020-12/schema * `title` are used for pretty print option names. * `description` is used for the options help message. -* `anyOf` with nested `const` and `title` are a special case as a replacement for `enum` but with pretty print name. +* For adding a description and a pretty print name to enum values (for [web-installation-instruction](https://github.com/instructions-d-installation/web-installation-instruction)): + 1. Indent the schema with the key `schema`. + 2. Add `pretty` and `description` keys. + 3. Create lists like `key: Pretty Key`. +* `title` and `description` from within the schema overwrite `pretty` and `description` outside of the schema. + +```yaml +schema: + name: installation-instruction + type: object + properties: + method: + enum: + - pipx + - pip +pretty: + pipx: Pipx + pip: Pip +description: + pipx: Installs python packages into virtual environments. + pip: Standard python package manager. +``` ### Template diff --git a/examples/pytorch/pytorch-instruction.schema.yml.jinja b/examples/pytorch/pytorch-instruction.schema.yml.jinja index 97d04b4..4b607b8 100644 --- a/examples/pytorch/pytorch-instruction.schema.yml.jinja +++ b/examples/pytorch/pytorch-instruction.schema.yml.jinja @@ -1,58 +1,61 @@ -$schema: https://json-schema.org/draft/2020-12/schema -$id: https://github.com/instructions-d-installation/installation-instruction/examples/pytorch/instruction_pytorch.schema.yml -title: PyTorch Install Schema -description: This is a schema which is used for constructing interactive installation instructions. -type: object -$comment: by Adam McKellar -properties: - build: - title: Build - description: Use the latest or the tested version. - anyOf: - - title: Stable (2.3.0) - const: stable - - title: Preview (Nightly) - const: preview - default: stable - os: - title: Operating System - description: The operating system you use. - anyOf: - - title: Linux - const: linux - - title: Mac - const: mac - - title: Windows - const: win - default: win - package: - title: Package Manager - description: The package manager you use. - anyOf: - - title: Conda - const: conda - - title: Pip - const: pip - default: pip - compute_platform: - title: Compute Platform - description: Should your gpu or your cpu handle the task? - anyOf: - - title: CUDA 11.8 - const: cu118 - - title: CUDA 12.1 - const: cu121 - - title: ROCm 6.0 - const: ro60 - - title: CPU - const: cpu - default: cu118 -required: - - build - - os - - package - - compute_platform -additionalProperties: false +schema: + $schema: https://json-schema.org/draft/2020-12/schema + $id: https://github.com/instructions-d-installation/installation-instruction/examples/pytorch/instruction_pytorch.schema.yml + title: PyTorch Install Schema + description: This is a schema which is used for constructing interactive installation instructions. + type: object + $comment: by Adam McKellar + properties: + build: + title: Build + description: Use the latest or the tested version. + enum: + - stable + - preview + default: stable + os: + title: Operating System + description: The operating system you use. + enum: + - linux + - mac + - win + default: win + package: + title: Package Manager + description: The package manager you use. + enum: + - conda + - pip + default: pip + compute_platform: + title: Compute Platform + description: Should your gpu or your cpu handle the task? + enum: + - cu118 + - cu121 + - ro60 + - cpu + default: cu118 + required: + - build + - os + - package + - compute_platform + additionalProperties: false + +pretty: + stable: Stable (2.3.0) + preview: Preview (Nightly) + linux: Linux + mac: Mac + win: Windows + conda: Conda + pip: Pip + cu118: CUDA 11.8 + cu121: CUDA 12.1 + ro60: ROCm 6.0 + cpu: CPU ------ diff --git a/examples/spacy/spacy-instruction.schema.yml.jinja b/examples/spacy/spacy-instruction.schema.yml.jinja index 1cff048..515ace7 100644 --- a/examples/spacy/spacy-instruction.schema.yml.jinja +++ b/examples/spacy/spacy-instruction.schema.yml.jinja @@ -1,93 +1,94 @@ -$schema: https://json-schema.org/draft/2020-12/schema -$id: https://github.com/instructions-d-installation/installation-instruction/examples/spacy/schema_spacy.yml -title: Spacy Install Schema -description: This is a schema which is used for constructing interactive installation instructions. -type: object -$comment: by Kanushka Gupta -properties: - os: - title: Operating System - description: Specify your Operating System - anyOf: - - title: macOs/OSX - const: mac - - title: Windows - const: windows - - title: Linux - const: linux - default: windows - platform: - title: Platform - description: platform - anyOf: - - title: x86 - const: x86 - - title: ARM/M1 - const: arm - default: x86 - package: - title: Package Manager - description: The package manager you use. - anyOf: - - title: Conda - const: conda - - title: Pip - const: pip - - title: from source - const: source - default: pip - hardware: - title: Hardware - description: Hardware you want to use- CPU or GPU? - anyOf: - - title: CPU - const: cpu - - title: GPU - const: gpu - type: string - default: cpu - - configuration: - title: Configuration - description: the configuration you have - anyOf: - - title: virtual env - const: venv - - title: train models - const: train_models - pipeline: - anyOf: - - title: efficiency - const: efficiency - - title: accuracy - const: accuracy -if: +schema: + $schema: https://json-schema.org/draft/2020-12/schema + $id: https://github.com/instructions-d-installation/installation-instruction/examples/spacy/schema_spacy.yml + title: Spacy Install Schema + description: This is a schema which is used for constructing interactive installation instructions. + type: object + $comment: by Kanushka Gupta properties: + os: + title: Operating System + description: Specify your Operating System + enum: + - mac + - windows + - linux + default: windows + platform: + title: Platform + description: platform + enum: + - x86 + - arm + default: x86 + package: + title: Package Manager + description: The package manager you use. + enum: + - conda + - pip + - source + default: pip hardware: - const: gpu -then: - properties: - cuda_runtime: + title: Hardware + description: Hardware you want to use- CPU or GPU? enum: - - CUDA 8.0 - - CUDA 9.0 - - CUDA 9.1 - - CUDA 9.2 - - CUDA 10.0 - - CUDA 10.1 - - CUDA 10.2 - - CUDA 11.0 - - CUDA 11.1 - - CUDA 11.2 - 11.x - - CUDA 12.x + - cpu + - gpu + type: string + default: cpu -required: - - os - - platform - - package - - hardware -additionalProperties: false + configuration: + title: Configuration + description: the configuration you have + enum: + - venv + - train_models + pipeline: + enum: + - efficiency + - accuracy + if: + properties: + hardware: + const: gpu + then: + properties: + cuda_runtime: + enum: + - CUDA 8.0 + - CUDA 9.0 + - CUDA 9.1 + - CUDA 9.2 + - CUDA 10.0 + - CUDA 10.1 + - CUDA 10.2 + - CUDA 11.0 + - CUDA 11.1 + - CUDA 11.2 - 11.x + - CUDA 12.x + + required: + - os + - platform + - package + - hardware + additionalProperties: false +pretty: + mac: macOs/OSX + windows: Windows + linux: Linux + x86: x86 + arm: ARM/M1 + pip: Pip + conda: Conda + cpu: CPU + gpu: GPU + venv: virtual env + train_models: train models + efficiency: efficiency + accuracy: accuracy ------ diff --git a/installation_instruction/__main__.py b/installation_instruction/__main__.py index 0330655..6328e64 100644 --- a/installation_instruction/__main__.py +++ b/installation_instruction/__main__.py @@ -49,7 +49,7 @@ def get_command(self, ctx, config_file: str) -> click.Command|None: try: instruction = InstallationInstruction.from_file(config_file) - options = _get_flags_and_options(instruction.schema) + options = _get_flags_and_options(instruction.schema, getattr(instruction, "misc", None)) except Exception as e: click.echo(click.style("Error (parsing options from schema): " + str(e), fg="red")) exit(1) diff --git a/installation_instruction/get_flags_and_options_from_schema.py b/installation_instruction/get_flags_and_options_from_schema.py index 2179bbb..f8e6292 100644 --- a/installation_instruction/get_flags_and_options_from_schema.py +++ b/installation_instruction/get_flags_and_options_from_schema.py @@ -22,11 +22,12 @@ "boolean": click.BOOL, } -def _get_flags_and_options(schema: dict) -> list[Option]: +def _get_flags_and_options(schema: dict, misc: dict = None) -> list[Option]: """ Generates Click flags and options from a JSON schema. :param schema: Schema which contains the options. + :param misc: Additional descriptions and pretty print names nested. :type schema: dict :return: List of all the clickoptions from the schema. :rtype: list[Option] @@ -34,25 +35,25 @@ def _get_flags_and_options(schema: dict) -> list[Option]: options = [] required_args = set(schema.get('required', [])) + description = misc.get("description", {}) if misc is not None else {} + for key, value in schema.get('properties', {}).items(): - orig_key = key - key = key.replace('_', '-').replace(' ', '-') - option_name = '--{}'.format(key) + pretty_key = key + pretty_key = pretty_key.replace('_', '-').replace(' ', '-') + option_name = '--{}'.format(pretty_key) option_type = value.get('type', 'string') - option_description = value.get('description', '') + option_description = value.get('description', '') or description.get(key, "") option_default = value.get('default', None) - if 'anyOf' in value: - option_type = Choice( [c['const'] for c in value['anyOf'] if 'const' in c] ) - elif "enum" in value: + if "enum" in value: option_type = Choice( value["enum"] ) else: option_type = SCHEMA_TO_CLICK_TYPE_MAPPING.get(option_type, click.STRING) - required = (orig_key in required_args) and option_default is None + required = (key in required_args) and option_default is None is_flag=(option_type == click.BOOL) if is_flag and required: - option_name = option_name + "/--no-{}".format(key) + option_name = option_name + "/--no-{}".format(pretty_key) options.append(Option( param_decls=[option_name], diff --git a/installation_instruction/installation_instruction.py b/installation_instruction/installation_instruction.py index 017d440..a87b994 100644 --- a/installation_instruction/installation_instruction.py +++ b/installation_instruction/installation_instruction.py @@ -75,11 +75,14 @@ def parse_schema(self) -> dict: result["description"] = self.schema.get("description", "") result["properties"] = {} + pretty = self.misc.get("pretty", {}) + description = self.misc.get("description", {}) + for key, value in self.schema.get('properties', {}).items(): result["properties"][key] = { - "title": value.get("title", key), - "description": value.get("description", ""), + "title": value.get("title", "") or pretty.get(key, key), + "description": value.get("description", "") or description.get(key, ""), "type": value.get("type", "string"), "default": value.get("default", None), "key": key, @@ -87,21 +90,12 @@ def parse_schema(self) -> dict: if "enum" in value: result["properties"][key]["enum"] = [ { - "title": e, + "title": pretty.get(e, e), "key": e, - "description": "", + "description": description.get(e, ""), } for e in value["enum"] ] result["properties"][key]["type"] = "enum" - elif type := "anyOf" if "anyOf" in value else "oneOf" if "oneOf" in value else None: - result["properties"][key]["enum"] = [ - { - "title": c.get("title", c.get("const", "")), - "key": c.get("const", ""), - "description": c.get("description", ""), - } for c in value[type] - ] - result["properties"][key]["type"] = "enum" return result @@ -114,19 +108,25 @@ def __init__(self, config: str) -> None: :raise Exception: If schema part of config is neither valid json nor valid yaml. :raise Exception: If no delimiter is found. """ - (schema, template) = helpers._split_string_at_delimiter(config) + (schema_str, template) = helpers._split_string_at_delimiter(config) try: - self.schema = json.load(schema) + schema = json.load(schema_str) except: try: - self.schema = safe_load(schema) + schema = safe_load(schema_str) except: raise Exception("Schema is neither a valid json nor a valid yaml.") + + if "schema" in schema: + self.schema = schema["schema"] + self.misc = {key: schema[key] for key in schema if key != "schema"} + else: + self.schema = schema try: Draft202012Validator.check_schema(self.schema) - except exceptions.SchemaError: - raise Exception("The given schema file is not a valid json schema.") + except exceptions.SchemaError as e: + raise Exception(f"The given schema file is not a valid json schema.\n\n{e}") self.template = helpers._load_template_from_string(RAISE_JINJA_MACRO_STRING+template) diff --git a/tests/conftest.py b/tests/conftest.py index 345db3a..563d7fe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,7 +34,7 @@ def test_data_flags_options(): with open(file_path,"r") as file: example = yaml.safe_load(file) - return example + return example["schema"] @pytest.fixture def test_data_flags_options_config_string_with_empty_template(): diff --git a/tests/data/flags_options_example.schema.yml b/tests/data/flags_options_example.schema.yml index 2cd03a1..2e88cfa 100644 --- a/tests/data/flags_options_example.schema.yml +++ b/tests/data/flags_options_example.schema.yml @@ -1,53 +1,60 @@ -$schema: https://json-schema.org/draft/2020-12/schema -$id: https://github.com/instructions-d-installation/installation-instruction/examples/scikit-learn/scikit-learn-instruction.schema.yml -title: Scikit-learn installation schema -description: This is a Schema to construct installation instructions for the python package scikit-learn by Timo Ege. -type: object -properties: - os: - title: Operating System - description: The operating system in which the package is installed. - enum: - - Windows - - macOS - - Linux - - - packager: - title: Packager - description: The package manager of your choosing. - enum: - - pip - - conda - default: pip - - virtualenv: - title: Use pip virtualenv - description: Choose if you want to use a virtual environment to install the package. - type: boolean - default: false - - compute_platform: - title: Compute Platform - description: Should your gpu or your cpu handle the task? - anyOf: - - title: CUDA 11.8 - const: cu118 - - title: CUDA 12.1 - const: cu121 - description: CUDA 12.1 is the latest version of CUDA. - default: cu118 - - verbose: - type: boolean - default: false - - requiered_flag: - type: boolean - -required: - - os - - packager - - requiered_flag - -additionalProperties: false \ No newline at end of file +schema: + $schema: https://json-schema.org/draft/2020-12/schema + $id: https://github.com/instructions-d-installation/installation-instruction/examples/scikit-learn/scikit-learn-instruction.schema.yml + title: Scikit-learn installation schema + description: This is a Schema to construct installation instructions for the python package scikit-learn by Timo Ege. + type: object + properties: + os: + title: Operating System + description: The operating system in which the package is installed. + enum: + - Windows + - macOS + - Linux + + + packager: + title: Packager + description: The package manager of your choosing. + enum: + - pip + - conda + default: pip + + virtualenv: + title: Use pip virtualenv + description: Choose if you want to use a virtual environment to install the package. + type: boolean + default: false + + compute_platform: + title: Compute Platform + description: Should your gpu or your cpu handle the task? + enum: + - cu118 + - cu121 + default: cu118 + + verbose: + type: boolean + default: false + + requiered_flag: + type: boolean + + required: + - os + - packager + - requiered_flag + + additionalProperties: false + +pretty: + cu118: CUDA 11.8 + cu121: CUDA 12.1 + +description: + cu121: CUDA 12.1 is the latest version of CUDA. + compute_platform: Not shown. + verbose: Activate verbose output. diff --git a/tests/data/test_install/install.cfg b/tests/data/test_install/install.cfg index bfdfe53..9a72d6a 100644 --- a/tests/data/test_install/install.cfg +++ b/tests/data/test_install/install.cfg @@ -1,16 +1,19 @@ -$schema: https://json-schema.org/draft/2020-12/schema -$id: https://github.com/instructions-d-installation/installation-instruction/tests/data/test_install/install.cfg -name: test-install -type: object -properties: - error_install: - type: boolean - error_template: - type: boolean - default: false -required: - - error_install - - error_template +schema: + $schema: https://json-schema.org/draft/2020-12/schema + $id: https://github.com/instructions-d-installation/installation-instruction/tests/data/test_install/install.cfg + name: test-install + type: object + properties: + error_install: + type: boolean + error_template: + type: boolean + default: false + required: + - error_install + - error_template +description: + error_install: Test for flag without default value. ------ echo "start" && {%- if error_install %} diff --git a/tests/test_get_flags_and_options_from_schema.py b/tests/test_get_flags_and_options_from_schema.py index c5d7851..230a501 100644 --- a/tests/test_get_flags_and_options_from_schema.py +++ b/tests/test_get_flags_and_options_from_schema.py @@ -17,7 +17,7 @@ def test_get_flags_and_options(test_data_flags_options): example_schema = test_data_flags_options - options = _get_flags_and_options(example_schema) + options = _get_flags_and_options(example_schema, {"description": { "verbose": "Activate verbose output." }}) assert len(options) == 6 @@ -42,6 +42,7 @@ def test_get_flags_and_options(test_data_flags_options): assert options[3].default == "cu118" assert options[4].opts == ["--verbose"] + assert options[4].help == "Activate verbose output." assert options[4].required == False assert options[4].default == False diff --git a/tests/test_installation_instruction.py b/tests/test_installation_instruction.py index 46e0be6..f75f208 100644 --- a/tests/test_installation_instruction.py +++ b/tests/test_installation_instruction.py @@ -70,4 +70,6 @@ def test_parse_schema(test_data_flags_options_config_string_with_empty_template) "default": False, "type": "boolean", "key": "virtualenv" - } \ No newline at end of file + } + + assert schema["properties"]["verbose"]["description"] == "Activate verbose output." \ No newline at end of file From eae18bc6510bb4437fbc6421db0098671dc83609 Mon Sep 17 00:00:00 2001 From: Adam McKellar Date: Wed, 12 Jun 2024 17:14:46 +0200 Subject: [PATCH 10/13] Refactored git cloning code. Removed deprecated tempfile function. Removed unecessary copying. Added cli functionality for directories. Added much documentation. Wrapped red echos into function. --- installation_instruction/__main__.py | 41 ++++++++++---- installation_instruction/helpers.py | 83 ++++++++++++++-------------- tests/test_helpers.py | 4 +- 3 files changed, 75 insertions(+), 53 deletions(-) diff --git a/installation_instruction/__main__.py b/installation_instruction/__main__.py index 602b3ce..5485bd1 100644 --- a/installation_instruction/__main__.py +++ b/installation_instruction/__main__.py @@ -13,7 +13,7 @@ # limitations under the License. from sys import exit -from os.path import isfile +from os.path import isfile, isdir from subprocess import run import click @@ -21,8 +21,7 @@ from .__init__ import __version__, __description__, __repository__, __author__, __author_email__, __license__ from .get_flags_and_options_from_schema import get_flags_and_options from .installation_instruction import InstallationInstruction -from .helpers import _make_pretty_print_line_breaks -from .helpers import _find_config_file_in_folder +from .helpers import _make_pretty_print_line_breaks, _is_remote_git_repository, _clone_git_repo, _config_file_is_in_folder VERSION_STRING = f"""Version: installation-instruction {__version__} @@ -31,38 +30,58 @@ Repository: {__repository__}""" +def _red_echo(text: str): + click.echo(click.style(text, fg="red")) + + class ConfigReadCommand(click.MultiCommand): """ - Custom click command class to read config file and show installation instructions with parameters. + Custom click command class to read config file, folder or git repository and show installation instructions with parameters. """ def __init__(self, *args, **kwargs): super().__init__( *args, **kwargs, - subcommand_metavar="CONFIG_FILE [OPTIONS]...", + subcommand_metavar="CONFIG_FILE/FOLDER/GIT_REPO [OPTIONS]...", options_metavar="", ) def get_command(self, ctx, config_file: str) -> click.Command|None: - config_file = _find_config_file_in_folder(config_file) + temp_dir = None + if _is_remote_git_repository(config_file): + try: + temp_dir = _clone_git_repo(config_file) + except Exception as e: + _red_echo("Error (cloning git repository):\n\n" + str(e)) + exit(1) + config_file = temp_dir.name + if isdir(config_file): + if path := _config_file_is_in_folder(config_file): + config_file = path + else: + if temp_dir is not None: + _red_echo("Config file not found in repository.") + else: + _red_echo(f"Config file not found in folder {config_file}") + exit(1) if not isfile(config_file): - click.echo("Config file not found.") - return None + _red_echo(f"{config_file} is not a file.") + exit(1) try: instruction = InstallationInstruction.from_file(config_file) options = get_flags_and_options(instruction.schema) except Exception as e: - click.echo(click.style("Error (parsing options from schema): " + str(e), fg="red")) + _red_echo("Error (parsing options from schema): " + str(e)) exit(1) def callback(**kwargs): inst = instruction.validate_and_render(kwargs) if inst[1]: - click.echo(click.style("Error: " + inst[0], fg="red")) + _red_echo("Error: " + inst[0]) exit(1) if ctx.obj["MODE"] == "show": if ctx.obj["RAW"]: @@ -72,7 +91,7 @@ def callback(**kwargs): elif ctx.obj["MODE"] == "install": result = run(inst[0], shell=True, text=True, capture_output=True) if result.returncode != 0: - click.echo(click.style("Installation failed with:\n" + str(result.stdout) + "\n" + str(result.stderr), fg="red")) + _red_echo("Installation failed with:\n" + str(result.stdout) + "\n" + str(result.stderr)) exit(1) else: if ctx.obj["INSTALL_VERBOSE"]: diff --git a/installation_instruction/helpers.py b/installation_instruction/helpers.py index 9b7fa07..bc8cf9f 100644 --- a/installation_instruction/helpers.py +++ b/installation_instruction/helpers.py @@ -16,50 +16,51 @@ import re from jinja2 import Environment, Template -from urllib.parse import urlparse import git -import tempfile -import shutil -import os - -def is_git_repo_url(url): - parsed_url = urlparse(url) - return all([parsed_url.scheme, parsed_url.netloc, parsed_url.path]) and url.endswith('.git') - -def clone_git_repo(url): - # Create a temporary directory - temp_dir = tempfile.TemporaryDirectory() - repo_dir = temp_dir.name - git.Repo.clone_from(url, repo_dir) - return repo_dir, temp_dir +from tempfile import TemporaryDirectory +import os.path + +CONFIG_FILE_NAME = "install.cfg" + +def _is_remote_git_repository(url: str) -> bool: + """ + Checks if the given URL might be a remote git repository. + + todo: Make this more robust. Check if it is actually a valid git repository by calling it. + + :param url: URL to be checked. + :type url: str + :return: True if the URL is a remote git repository, else False. + :rtype: bool + """ + return (url.startswith('http://') or url.startswith('https://') ) and url.endswith('.git') + +def _clone_git_repo(url: str) -> TemporaryDirectory: + """ + Clones a git repository to a temporary directory. + + :param url: URL of the remote git repository. + :type url: str + :return: `TemporaryDirectory` object with git repo. + :rtype: tempfile.TemporaryDirectory + """ + temp_dir = TemporaryDirectory() + git.Repo.clone_from(url, temp_dir.name) + return temp_dir -def check_and_download_install_cfg(repo_dir): - install_cfg_path = os.path.join(repo_dir, 'install.cfg') +def _config_file_is_in_folder(dir_path: str) -> str | None: + """ + Checks if the file `install.cfg` is in the folder. + + :param dir_path: Path to the folder. + :type dir_path: str + :return: Path to the `install.cfg` file if it exists, else None. + :rtype: str or None + """ + install_cfg_path = os.path.join(dir_path, CONFIG_FILE_NAME) if os.path.isfile(install_cfg_path): - temp_file_dir = tempfile.mkdtemp() - temp_file_path = os.path.join(temp_file_dir, 'install.cfg') - shutil.copy(install_cfg_path, temp_file_path) - return temp_file_path, temp_file_dir - return None, None - -def _find_config_file_in_folder(folder_path: str) -> str | None: - """ - Finds file with the name `install.cfg` in the folder and returns its path if it exists. - """ - if folder_path.startswith('http://') or folder_path.startswith('https://'): - # It's a URL, handle it accordingly - repo_dir, temp_dir = clone_git_repo(folder_path) - install_cfg_path, install_cfg_temp_dir = check_and_download_install_cfg(repo_dir) - if install_cfg_path: - print(f"install.cfg found and downloaded to: {install_cfg_path}") - return install_cfg_path - else: - print("install.cfg not found in the repository root.") - # Clean up the cloned repository - temp_dir.cleanup() - elif os.path.isfile(folder_path): - return folder_path - return folder_path + return install_cfg_path + return None def _make_pretty_print_line_breaks(string: str) -> str: """ diff --git a/tests/test_helpers.py b/tests/test_helpers.py index e6218a4..db73d51 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -95,4 +95,6 @@ def test_split_string_at_delimiter_with_delimiter(): assert schema == parsed_schema assert template == parsed_template - \ No newline at end of file +def test_is_remote_git_repository(): + assert helpers._is_remote_git_repository("https://github.com/instructions-d-installation/web-installation-instruction.git") + assert not helpers._is_remote_git_repository("./instructions-d-installation/web-installation-instruction.git") From b4bf9790dcc8383222b7c18ee5c18a59bbfffbcc Mon Sep 17 00:00:00 2001 From: Adam McKellar Date: Wed, 12 Jun 2024 17:19:40 +0200 Subject: [PATCH 11/13] Added to tip in readme, that config files, folders and git repo urls are supported. --- README.md | 2 +- installation_instruction/__main__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c2355d2..c0225b7 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Commands: Options are dynamically created with the schema part of the config file. > [!TIP] -> Show help for a config file with: `ibi show CONFIG_FILE --help`. +> Show help for a config file with: `ibi show CONFIG_FILE/FOLDER/GIT_REPO_URL --help`. ## Config diff --git a/installation_instruction/__main__.py b/installation_instruction/__main__.py index 5485bd1..bdeb1b3 100644 --- a/installation_instruction/__main__.py +++ b/installation_instruction/__main__.py @@ -43,7 +43,7 @@ def __init__(self, *args, **kwargs): super().__init__( *args, **kwargs, - subcommand_metavar="CONFIG_FILE/FOLDER/GIT_REPO [OPTIONS]...", + subcommand_metavar="CONFIG_FILE/FOLDER/GIT_REPO_URL [OPTIONS]...", options_metavar="", ) From 95244d2822abb316866cf3391f9ddf67e7cbf6f2 Mon Sep 17 00:00:00 2001 From: Adam McKellar Date: Wed, 12 Jun 2024 17:43:36 +0200 Subject: [PATCH 12/13] Changed git repository url detection to support all urls like https://git-scm.com/docs/git-clone#URLS making the check less stringent. Made git clone shallow. --- installation_instruction/helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/installation_instruction/helpers.py b/installation_instruction/helpers.py index bc8cf9f..a8de8a5 100644 --- a/installation_instruction/helpers.py +++ b/installation_instruction/helpers.py @@ -21,6 +21,7 @@ import os.path CONFIG_FILE_NAME = "install.cfg" +ALLOWED_GIT_URL_PREFIXES = ["http://", "https://", "git://", "ssh://", "ftp://", "ftps://"] def _is_remote_git_repository(url: str) -> bool: """ @@ -33,7 +34,7 @@ def _is_remote_git_repository(url: str) -> bool: :return: True if the URL is a remote git repository, else False. :rtype: bool """ - return (url.startswith('http://') or url.startswith('https://') ) and url.endswith('.git') + return any([url.startswith(prefix) for prefix in ALLOWED_GIT_URL_PREFIXES]) def _clone_git_repo(url: str) -> TemporaryDirectory: """ @@ -45,7 +46,7 @@ def _clone_git_repo(url: str) -> TemporaryDirectory: :rtype: tempfile.TemporaryDirectory """ temp_dir = TemporaryDirectory() - git.Repo.clone_from(url, temp_dir.name) + git.Repo.clone_from(url, temp_dir.name, multi_options=["--depth=1"]) return temp_dir def _config_file_is_in_folder(dir_path: str) -> str | None: From ffe7f54d0a443ad13d857bf9868bf5b2f5bb01da Mon Sep 17 00:00:00 2001 From: Adam McKellar Date: Fri, 14 Jun 2024 16:25:21 +0200 Subject: [PATCH 13/13] Added to the changelog recent updates regarding git and directory sources. Clarified removed section regarding anyOf change. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20d4198..8dc1516 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* Added git remote repositories and directories as potential sources for config files in the cli. * `title` and `description` can be set from `pretty` and `description` keys at root. @@ -19,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +* Removed `title` and `description` support from `anyOf`. + ## [0.3.0] - 2024-06-04