diff --git a/CHANGELOG.md b/CHANGELOG.md index 40f6ba6..8dc1516 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* Added git remote repositories and directories as potential sources for config files in the cli. +* `title` and `description` can be set from `pretty` and `description` keys at root. + + ### Changed +* Moved `title` and `description` from `anyOf` to outside of schema. `anyOf` is now usable as intended for json schema. + + ### Removed +* Removed `title` and `description` support from `anyOf`. + ## [0.3.0] - 2024-06-04 diff --git a/README.md b/README.md index 18c9155..b36b20b 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Commands: Options are dynamically created with the schema part of the config file. > [!TIP] -> Show help for a config file with: `ibi show CONFIG_FILE --help`. +> Show help for a config file with: `ibi show CONFIG_FILE/FOLDER/GIT_REPO_URL --help`. ## Config @@ -86,7 +86,28 @@ Options are dynamically created with the schema part of the config file. * When creating a schema use the following schema draft version: https://json-schema.org/draft/2020-12/schema * `title` are used for pretty print option names. * `description` is used for the options help message. -* `anyOf` with nested `const` and `title` are a special case as a replacement for `enum` but with pretty print name. +* For adding a description and a pretty print name to enum values (for [web-installation-instruction](https://github.com/instructions-d-installation/web-installation-instruction)): + 1. Indent the schema with the key `schema`. + 2. Add `pretty` and `description` keys. + 3. Create lists like `key: Pretty Key`. +* `title` and `description` from within the schema overwrite `pretty` and `description` outside of the schema. + +```yaml +schema: + name: installation-instruction + type: object + properties: + method: + enum: + - pipx + - pip +pretty: + pipx: Pipx + pip: Pip +description: + pipx: Installs python packages into virtual environments. + pip: Standard python package manager. +``` ### Template diff --git a/examples/pytorch/pytorch-instruction.schema.yml.jinja b/examples/pytorch/pytorch-instruction.schema.yml.jinja index 97d04b4..4b607b8 100644 --- a/examples/pytorch/pytorch-instruction.schema.yml.jinja +++ b/examples/pytorch/pytorch-instruction.schema.yml.jinja @@ -1,58 +1,61 @@ -$schema: https://json-schema.org/draft/2020-12/schema -$id: https://github.com/instructions-d-installation/installation-instruction/examples/pytorch/instruction_pytorch.schema.yml -title: PyTorch Install Schema -description: This is a schema which is used for constructing interactive installation instructions. -type: object -$comment: by Adam McKellar -properties: - build: - title: Build - description: Use the latest or the tested version. - anyOf: - - title: Stable (2.3.0) - const: stable - - title: Preview (Nightly) - const: preview - default: stable - os: - title: Operating System - description: The operating system you use. - anyOf: - - title: Linux - const: linux - - title: Mac - const: mac - - title: Windows - const: win - default: win - package: - title: Package Manager - description: The package manager you use. - anyOf: - - title: Conda - const: conda - - title: Pip - const: pip - default: pip - compute_platform: - title: Compute Platform - description: Should your gpu or your cpu handle the task? - anyOf: - - title: CUDA 11.8 - const: cu118 - - title: CUDA 12.1 - const: cu121 - - title: ROCm 6.0 - const: ro60 - - title: CPU - const: cpu - default: cu118 -required: - - build - - os - - package - - compute_platform -additionalProperties: false +schema: + $schema: https://json-schema.org/draft/2020-12/schema + $id: https://github.com/instructions-d-installation/installation-instruction/examples/pytorch/instruction_pytorch.schema.yml + title: PyTorch Install Schema + description: This is a schema which is used for constructing interactive installation instructions. + type: object + $comment: by Adam McKellar + properties: + build: + title: Build + description: Use the latest or the tested version. + enum: + - stable + - preview + default: stable + os: + title: Operating System + description: The operating system you use. + enum: + - linux + - mac + - win + default: win + package: + title: Package Manager + description: The package manager you use. + enum: + - conda + - pip + default: pip + compute_platform: + title: Compute Platform + description: Should your gpu or your cpu handle the task? + enum: + - cu118 + - cu121 + - ro60 + - cpu + default: cu118 + required: + - build + - os + - package + - compute_platform + additionalProperties: false + +pretty: + stable: Stable (2.3.0) + preview: Preview (Nightly) + linux: Linux + mac: Mac + win: Windows + conda: Conda + pip: Pip + cu118: CUDA 11.8 + cu121: CUDA 12.1 + ro60: ROCm 6.0 + cpu: CPU ------ diff --git a/examples/spacy/spacy-instruction.schema.yml.jinja b/examples/spacy/spacy-instruction.schema.yml.jinja index 1cff048..515ace7 100644 --- a/examples/spacy/spacy-instruction.schema.yml.jinja +++ b/examples/spacy/spacy-instruction.schema.yml.jinja @@ -1,93 +1,94 @@ -$schema: https://json-schema.org/draft/2020-12/schema -$id: https://github.com/instructions-d-installation/installation-instruction/examples/spacy/schema_spacy.yml -title: Spacy Install Schema -description: This is a schema which is used for constructing interactive installation instructions. -type: object -$comment: by Kanushka Gupta -properties: - os: - title: Operating System - description: Specify your Operating System - anyOf: - - title: macOs/OSX - const: mac - - title: Windows - const: windows - - title: Linux - const: linux - default: windows - platform: - title: Platform - description: platform - anyOf: - - title: x86 - const: x86 - - title: ARM/M1 - const: arm - default: x86 - package: - title: Package Manager - description: The package manager you use. - anyOf: - - title: Conda - const: conda - - title: Pip - const: pip - - title: from source - const: source - default: pip - hardware: - title: Hardware - description: Hardware you want to use- CPU or GPU? - anyOf: - - title: CPU - const: cpu - - title: GPU - const: gpu - type: string - default: cpu - - configuration: - title: Configuration - description: the configuration you have - anyOf: - - title: virtual env - const: venv - - title: train models - const: train_models - pipeline: - anyOf: - - title: efficiency - const: efficiency - - title: accuracy - const: accuracy -if: +schema: + $schema: https://json-schema.org/draft/2020-12/schema + $id: https://github.com/instructions-d-installation/installation-instruction/examples/spacy/schema_spacy.yml + title: Spacy Install Schema + description: This is a schema which is used for constructing interactive installation instructions. + type: object + $comment: by Kanushka Gupta properties: + os: + title: Operating System + description: Specify your Operating System + enum: + - mac + - windows + - linux + default: windows + platform: + title: Platform + description: platform + enum: + - x86 + - arm + default: x86 + package: + title: Package Manager + description: The package manager you use. + enum: + - conda + - pip + - source + default: pip hardware: - const: gpu -then: - properties: - cuda_runtime: + title: Hardware + description: Hardware you want to use- CPU or GPU? enum: - - CUDA 8.0 - - CUDA 9.0 - - CUDA 9.1 - - CUDA 9.2 - - CUDA 10.0 - - CUDA 10.1 - - CUDA 10.2 - - CUDA 11.0 - - CUDA 11.1 - - CUDA 11.2 - 11.x - - CUDA 12.x + - cpu + - gpu + type: string + default: cpu -required: - - os - - platform - - package - - hardware -additionalProperties: false + configuration: + title: Configuration + description: the configuration you have + enum: + - venv + - train_models + pipeline: + enum: + - efficiency + - accuracy + if: + properties: + hardware: + const: gpu + then: + properties: + cuda_runtime: + enum: + - CUDA 8.0 + - CUDA 9.0 + - CUDA 9.1 + - CUDA 9.2 + - CUDA 10.0 + - CUDA 10.1 + - CUDA 10.2 + - CUDA 11.0 + - CUDA 11.1 + - CUDA 11.2 - 11.x + - CUDA 12.x + + required: + - os + - platform + - package + - hardware + additionalProperties: false +pretty: + mac: macOs/OSX + windows: Windows + linux: Linux + x86: x86 + arm: ARM/M1 + pip: Pip + conda: Conda + cpu: CPU + gpu: GPU + venv: virtual env + train_models: train models + efficiency: efficiency + accuracy: accuracy ------ diff --git a/installation_instruction/__main__.py b/installation_instruction/__main__.py index 83a728c..6cb4915 100644 --- a/installation_instruction/__main__.py +++ b/installation_instruction/__main__.py @@ -13,7 +13,7 @@ # limitations under the License. from sys import exit -from os.path import isfile +from os.path import isfile, isdir from subprocess import run import platform @@ -22,7 +22,8 @@ from .__init__ import __version__, __description__, __repository__, __author__, __author_email__, __license__ from .get_flags_and_options_from_schema import _get_flags_and_options from .installation_instruction import InstallationInstruction -from .helpers import _make_pretty_print_line_breaks +from .helpers import _make_pretty_print_line_breaks, _is_remote_git_repository, _clone_git_repo, _config_file_is_in_folder + VERSION_STRING = f"""Version: installation-instruction {__version__} Copyright: (C) 2024 {__author_email__}, {__author__} @@ -46,30 +47,52 @@ def _get_system(option_types): return None +def _red_echo(text: str): + click.echo(click.style(text, fg="red")) + + class ConfigReadCommand(click.MultiCommand): """ - Custom click command class to read config file and show installation instructions with parameters. + Custom click command class to read config file, folder or git repository and show installation instructions with parameters. """ def __init__(self, *args, **kwargs): super().__init__( *args, **kwargs, - subcommand_metavar="CONFIG_FILE [OPTIONS]...", + subcommand_metavar="CONFIG_FILE/FOLDER/GIT_REPO_URL [OPTIONS]...", options_metavar="", ) def get_command(self, ctx, config_file: str) -> click.Command|None: - if not isfile(config_file): - click.echo("Config file not found.") - return None + temp_dir = None + if _is_remote_git_repository(config_file): + try: + temp_dir = _clone_git_repo(config_file) + except Exception as e: + _red_echo("Error (cloning git repository):\n\n" + str(e)) + exit(1) + config_file = temp_dir.name + if isdir(config_file): + if path := _config_file_is_in_folder(config_file): + config_file = path + else: + if temp_dir is not None: + _red_echo("Config file not found in repository.") + else: + _red_echo(f"Config file not found in folder {config_file}") + exit(1) + if not isfile(config_file): + _red_echo(f"{config_file} is not a file.") + exit(1) + try: instruction = InstallationInstruction.from_file(config_file) - options = _get_flags_and_options(instruction.schema) + options = _get_flags_and_options(instruction.schema, getattr(instruction, "misc", None)) except Exception as e: - click.echo(click.style("Error (parsing options from schema): " + str(e), fg="red")) + _red_echo("Error (parsing options from schema): " + str(e)) exit(1) for option in options: @@ -81,7 +104,7 @@ def get_command(self, ctx, config_file: str) -> click.Command|None: def callback(**kwargs): inst = instruction.validate_and_render(kwargs) if inst[1]: - click.echo(click.style("Error: " + inst[0], fg="red")) + _red_echo("Error: " + inst[0]) exit(1) if ctx.obj["MODE"] == "show": if ctx.obj["RAW"]: @@ -91,7 +114,7 @@ def callback(**kwargs): elif ctx.obj["MODE"] == "install": result = run(inst[0], shell=True, text=True, capture_output=True) if result.returncode != 0: - click.echo(click.style("Installation failed with:\n" + str(result.stdout) + "\n" + str(result.stderr), fg="red")) + _red_echo("Installation failed with:\n" + str(result.stdout) + "\n" + str(result.stderr)) exit(1) else: if ctx.obj["INSTALL_VERBOSE"]: @@ -107,7 +130,6 @@ def callback(**kwargs): callback=callback, ) - @click.command(cls=ConfigReadCommand, help="Shows installation instructions for your specified config file and parameters.") @click.option("--raw", is_flag=True, help="Show installation instructions without pretty print.", default=False) @click.pass_context @@ -132,4 +154,5 @@ def main(ctx): main.add_command(install) if __name__ == "__main__": - main() \ No newline at end of file + main() + diff --git a/installation_instruction/get_flags_and_options_from_schema.py b/installation_instruction/get_flags_and_options_from_schema.py index 2179bbb..f8e6292 100644 --- a/installation_instruction/get_flags_and_options_from_schema.py +++ b/installation_instruction/get_flags_and_options_from_schema.py @@ -22,11 +22,12 @@ "boolean": click.BOOL, } -def _get_flags_and_options(schema: dict) -> list[Option]: +def _get_flags_and_options(schema: dict, misc: dict = None) -> list[Option]: """ Generates Click flags and options from a JSON schema. :param schema: Schema which contains the options. + :param misc: Additional descriptions and pretty print names nested. :type schema: dict :return: List of all the clickoptions from the schema. :rtype: list[Option] @@ -34,25 +35,25 @@ def _get_flags_and_options(schema: dict) -> list[Option]: options = [] required_args = set(schema.get('required', [])) + description = misc.get("description", {}) if misc is not None else {} + for key, value in schema.get('properties', {}).items(): - orig_key = key - key = key.replace('_', '-').replace(' ', '-') - option_name = '--{}'.format(key) + pretty_key = key + pretty_key = pretty_key.replace('_', '-').replace(' ', '-') + option_name = '--{}'.format(pretty_key) option_type = value.get('type', 'string') - option_description = value.get('description', '') + option_description = value.get('description', '') or description.get(key, "") option_default = value.get('default', None) - if 'anyOf' in value: - option_type = Choice( [c['const'] for c in value['anyOf'] if 'const' in c] ) - elif "enum" in value: + if "enum" in value: option_type = Choice( value["enum"] ) else: option_type = SCHEMA_TO_CLICK_TYPE_MAPPING.get(option_type, click.STRING) - required = (orig_key in required_args) and option_default is None + required = (key in required_args) and option_default is None is_flag=(option_type == click.BOOL) if is_flag and required: - option_name = option_name + "/--no-{}".format(key) + option_name = option_name + "/--no-{}".format(pretty_key) options.append(Option( param_decls=[option_name], diff --git a/installation_instruction/helpers.py b/installation_instruction/helpers.py index bb0dd12..a8de8a5 100644 --- a/installation_instruction/helpers.py +++ b/installation_instruction/helpers.py @@ -1,3 +1,4 @@ + # Copyright 2024 Adam McKellar, Kanushka Gupta, Timo Ege # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +16,52 @@ import re from jinja2 import Environment, Template +import git +from tempfile import TemporaryDirectory +import os.path + +CONFIG_FILE_NAME = "install.cfg" +ALLOWED_GIT_URL_PREFIXES = ["http://", "https://", "git://", "ssh://", "ftp://", "ftps://"] + +def _is_remote_git_repository(url: str) -> bool: + """ + Checks if the given URL might be a remote git repository. + + todo: Make this more robust. Check if it is actually a valid git repository by calling it. + + :param url: URL to be checked. + :type url: str + :return: True if the URL is a remote git repository, else False. + :rtype: bool + """ + return any([url.startswith(prefix) for prefix in ALLOWED_GIT_URL_PREFIXES]) + +def _clone_git_repo(url: str) -> TemporaryDirectory: + """ + Clones a git repository to a temporary directory. + + :param url: URL of the remote git repository. + :type url: str + :return: `TemporaryDirectory` object with git repo. + :rtype: tempfile.TemporaryDirectory + """ + temp_dir = TemporaryDirectory() + git.Repo.clone_from(url, temp_dir.name, multi_options=["--depth=1"]) + return temp_dir + +def _config_file_is_in_folder(dir_path: str) -> str | None: + """ + Checks if the file `install.cfg` is in the folder. + + :param dir_path: Path to the folder. + :type dir_path: str + :return: Path to the `install.cfg` file if it exists, else None. + :rtype: str or None + """ + install_cfg_path = os.path.join(dir_path, CONFIG_FILE_NAME) + if os.path.isfile(install_cfg_path): + return install_cfg_path + return None def _make_pretty_print_line_breaks(string: str) -> str: """ diff --git a/installation_instruction/installation_instruction.py b/installation_instruction/installation_instruction.py index 017d440..a87b994 100644 --- a/installation_instruction/installation_instruction.py +++ b/installation_instruction/installation_instruction.py @@ -75,11 +75,14 @@ def parse_schema(self) -> dict: result["description"] = self.schema.get("description", "") result["properties"] = {} + pretty = self.misc.get("pretty", {}) + description = self.misc.get("description", {}) + for key, value in self.schema.get('properties', {}).items(): result["properties"][key] = { - "title": value.get("title", key), - "description": value.get("description", ""), + "title": value.get("title", "") or pretty.get(key, key), + "description": value.get("description", "") or description.get(key, ""), "type": value.get("type", "string"), "default": value.get("default", None), "key": key, @@ -87,21 +90,12 @@ def parse_schema(self) -> dict: if "enum" in value: result["properties"][key]["enum"] = [ { - "title": e, + "title": pretty.get(e, e), "key": e, - "description": "", + "description": description.get(e, ""), } for e in value["enum"] ] result["properties"][key]["type"] = "enum" - elif type := "anyOf" if "anyOf" in value else "oneOf" if "oneOf" in value else None: - result["properties"][key]["enum"] = [ - { - "title": c.get("title", c.get("const", "")), - "key": c.get("const", ""), - "description": c.get("description", ""), - } for c in value[type] - ] - result["properties"][key]["type"] = "enum" return result @@ -114,19 +108,25 @@ def __init__(self, config: str) -> None: :raise Exception: If schema part of config is neither valid json nor valid yaml. :raise Exception: If no delimiter is found. """ - (schema, template) = helpers._split_string_at_delimiter(config) + (schema_str, template) = helpers._split_string_at_delimiter(config) try: - self.schema = json.load(schema) + schema = json.load(schema_str) except: try: - self.schema = safe_load(schema) + schema = safe_load(schema_str) except: raise Exception("Schema is neither a valid json nor a valid yaml.") + + if "schema" in schema: + self.schema = schema["schema"] + self.misc = {key: schema[key] for key in schema if key != "schema"} + else: + self.schema = schema try: Draft202012Validator.check_schema(self.schema) - except exceptions.SchemaError: - raise Exception("The given schema file is not a valid json schema.") + except exceptions.SchemaError as e: + raise Exception(f"The given schema file is not a valid json schema.\n\n{e}") self.template = helpers._load_template_from_string(RAISE_JINJA_MACRO_STRING+template) diff --git a/pyproject.toml b/pyproject.toml index 3197c9a..2ec3ea1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,12 +16,12 @@ description = "Library and CLI for generating installation instructions from jso readme = "README.md" maintainers = [ { name = "Adam McKellar", email = "dev@mckellar.eu" }, - { name = "Kanushka Gupta" }, + { name = "Kanushka Gupta", email = "kanushkagupta1298@gmail.com" }, { name = "Timo Ege", email = "timoege@online.de" }, ] authors = [ { name = "Adam McKellar", email = "dev@mckellar.eu" }, - { name = "Kanushka Gupta" }, + { name = "Kanushka Gupta", email = "kanushkagupta1298@gmail.com" }, { name = "Timo Ege", email = "timoege@online.de" }, ] version = "0.3.0" @@ -37,6 +37,7 @@ dependencies = [ "jsonschema", "PyYAML", "click < 9.0.0a0", + "GitPython", ] [project.optional-dependencies] diff --git a/tests/conftest.py b/tests/conftest.py index 345db3a..563d7fe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,7 +34,7 @@ def test_data_flags_options(): with open(file_path,"r") as file: example = yaml.safe_load(file) - return example + return example["schema"] @pytest.fixture def test_data_flags_options_config_string_with_empty_template(): diff --git a/tests/data/flags_options_example.schema.yml b/tests/data/flags_options_example.schema.yml index 2cd03a1..2e88cfa 100644 --- a/tests/data/flags_options_example.schema.yml +++ b/tests/data/flags_options_example.schema.yml @@ -1,53 +1,60 @@ -$schema: https://json-schema.org/draft/2020-12/schema -$id: https://github.com/instructions-d-installation/installation-instruction/examples/scikit-learn/scikit-learn-instruction.schema.yml -title: Scikit-learn installation schema -description: This is a Schema to construct installation instructions for the python package scikit-learn by Timo Ege. -type: object -properties: - os: - title: Operating System - description: The operating system in which the package is installed. - enum: - - Windows - - macOS - - Linux - - - packager: - title: Packager - description: The package manager of your choosing. - enum: - - pip - - conda - default: pip - - virtualenv: - title: Use pip virtualenv - description: Choose if you want to use a virtual environment to install the package. - type: boolean - default: false - - compute_platform: - title: Compute Platform - description: Should your gpu or your cpu handle the task? - anyOf: - - title: CUDA 11.8 - const: cu118 - - title: CUDA 12.1 - const: cu121 - description: CUDA 12.1 is the latest version of CUDA. - default: cu118 - - verbose: - type: boolean - default: false - - requiered_flag: - type: boolean - -required: - - os - - packager - - requiered_flag - -additionalProperties: false \ No newline at end of file +schema: + $schema: https://json-schema.org/draft/2020-12/schema + $id: https://github.com/instructions-d-installation/installation-instruction/examples/scikit-learn/scikit-learn-instruction.schema.yml + title: Scikit-learn installation schema + description: This is a Schema to construct installation instructions for the python package scikit-learn by Timo Ege. + type: object + properties: + os: + title: Operating System + description: The operating system in which the package is installed. + enum: + - Windows + - macOS + - Linux + + + packager: + title: Packager + description: The package manager of your choosing. + enum: + - pip + - conda + default: pip + + virtualenv: + title: Use pip virtualenv + description: Choose if you want to use a virtual environment to install the package. + type: boolean + default: false + + compute_platform: + title: Compute Platform + description: Should your gpu or your cpu handle the task? + enum: + - cu118 + - cu121 + default: cu118 + + verbose: + type: boolean + default: false + + requiered_flag: + type: boolean + + required: + - os + - packager + - requiered_flag + + additionalProperties: false + +pretty: + cu118: CUDA 11.8 + cu121: CUDA 12.1 + +description: + cu121: CUDA 12.1 is the latest version of CUDA. + compute_platform: Not shown. + verbose: Activate verbose output. diff --git a/tests/test_get_flags_and_options_from_schema.py b/tests/test_get_flags_and_options_from_schema.py index c5d7851..230a501 100644 --- a/tests/test_get_flags_and_options_from_schema.py +++ b/tests/test_get_flags_and_options_from_schema.py @@ -17,7 +17,7 @@ def test_get_flags_and_options(test_data_flags_options): example_schema = test_data_flags_options - options = _get_flags_and_options(example_schema) + options = _get_flags_and_options(example_schema, {"description": { "verbose": "Activate verbose output." }}) assert len(options) == 6 @@ -42,6 +42,7 @@ def test_get_flags_and_options(test_data_flags_options): assert options[3].default == "cu118" assert options[4].opts == ["--verbose"] + assert options[4].help == "Activate verbose output." assert options[4].required == False assert options[4].default == False diff --git a/tests/test_helpers.py b/tests/test_helpers.py index e6218a4..db73d51 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -95,4 +95,6 @@ def test_split_string_at_delimiter_with_delimiter(): assert schema == parsed_schema assert template == parsed_template - \ No newline at end of file +def test_is_remote_git_repository(): + assert helpers._is_remote_git_repository("https://github.com/instructions-d-installation/web-installation-instruction.git") + assert not helpers._is_remote_git_repository("./instructions-d-installation/web-installation-instruction.git") diff --git a/tests/test_installation_instruction.py b/tests/test_installation_instruction.py index 46e0be6..f75f208 100644 --- a/tests/test_installation_instruction.py +++ b/tests/test_installation_instruction.py @@ -70,4 +70,6 @@ def test_parse_schema(test_data_flags_options_config_string_with_empty_template) "default": False, "type": "boolean", "key": "virtualenv" - } \ No newline at end of file + } + + assert schema["properties"]["verbose"]["description"] == "Activate verbose output." \ No newline at end of file