From a4bdab4d222aa4d176558de671776dfcdbbcbc9b Mon Sep 17 00:00:00 2001 From: Vasili Bulkin Date: Fri, 13 Aug 2021 01:03:07 +0300 Subject: [PATCH] Removed dependency on docker, added `toltecmk-contained` script... ... for calling `toltecmk` inside container. Builder now accepts a `source_dir` argument for using local sources (convenient to use during app development). --- toltec/__main__.py | 12 +++++- toltec/bash.py | 55 ------------------------ toltec/builder.py | 62 ++++++++-------------------- toltec/util.py | 23 +++++------ toltecmk | 1 + toltecmk-contained | 59 ++++++++++++++++++++++++++ toltecmk.py | 101 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 200 insertions(+), 113 deletions(-) create mode 120000 toltecmk create mode 100755 toltecmk-contained create mode 100755 toltecmk.py diff --git a/toltec/__main__.py b/toltec/__main__.py index c374cb7..5ae89c3 100644 --- a/toltec/__main__.py +++ b/toltec/__main__.py @@ -44,6 +44,16 @@ def main(): (default: [current directory]/dist)""", ) + parser.add_argument( + "-s", + "--source-dir", + metavar="DIR", + default=None, + help="""path to the source directory + (optional: when specified, a local build is performed instead of fetching + sources)""", + ) + parser.add_argument( "-a", "--arch-name", @@ -79,7 +89,7 @@ def main(): recipe_bundle = parse_recipe(args.recipe_dir) - with Builder(args.work_dir, args.dist_dir) as builder: + with Builder(args.work_dir, args.dist_dir, args.source_dir) as builder: if args.hook: for ident in args.hook: if ident and ident[0] in (".", "/"): diff --git a/toltec/bash.py b/toltec/bash.py index 037896c..a006815 100644 --- a/toltec/bash.py +++ b/toltec/bash.py @@ -8,7 +8,6 @@ import logging from collections import deque from typing import Deque, Dict, Generator, List, Optional, Tuple, Union -from docker.client import DockerClient AssociativeArray = Dict[str, str] IndexedArray = List[Optional[str]] @@ -361,60 +360,6 @@ def run_script(variables: Variables, script: str) -> LogGenerator: if process.returncode != 0: raise ScriptError(f"Script exited with code {process.returncode}") - - -def run_script_in_container( - docker: DockerClient, - image: str, - mounts: List, - variables: Variables, - script: str, -) -> LogGenerator: - """ - Run a Bash script inside a Docker container and stream its output. - - :param docker: Docker client - :param image: image to use for the new container - :param mounts: paths to mount in the container - :param variables: Bash variables to set before running the script - :param script: Bash script to execute - :returns: generator yielding output lines from the script - :raises ScriptError: if the script exits with a non-zero code - """ - container = docker.containers.run( - image, - mounts=mounts, - command=[ - "/usr/bin/env", - "bash", - "-c", - "\n".join( - ( - "set -euo pipefail", - put_variables(variables), - "script() {", - script, - "}", - "script", - ) - ), - ], - detach=True, - ) - - try: - for line in container.logs(stream=True): - if line: - yield line.decode().strip() - - result = container.wait() - - if result["StatusCode"] != 0: - raise ScriptError(f"Script exited with code {result['StatusCode']}") - finally: - container.remove() - - def pipe_logs( logger: logging.Logger, logs: LogGenerator, diff --git a/toltec/builder.py b/toltec/builder.py index 42827ce..b187328 100644 --- a/toltec/builder.py +++ b/toltec/builder.py @@ -9,7 +9,6 @@ import os import logging import textwrap -import docker import requests from . import bash, util, ipk from .recipe import RecipeBundle, Recipe, Package @@ -28,10 +27,9 @@ class Builder: # pylint: disable=too-few-public-methods # Detect non-local paths URL_REGEX = re.compile(r"[a-z]+://") - # Prefix for all Toltec Docker images - IMAGE_PREFIX = "ghcr.io/toltec-dev/" - - def __init__(self, work_dir: str, dist_dir: str) -> None: + def __init__( + self, work_dir: str, dist_dir: str, source_dir: Optional[str] + ) -> None: """ Create a builder helper. @@ -40,15 +38,7 @@ def __init__(self, work_dir: str, dist_dir: str) -> None: """ self.work_dir = work_dir self.dist_dir = dist_dir - - try: - self.docker = docker.from_env() - except docker.errors.DockerException as err: - raise BuildError( - "Unable to connect to the Docker daemon. \ -Please check that the service is running and that you have the necessary \ -permissions." - ) from err + self.source_dir = source_dir def __enter__(self) -> "Builder": return self @@ -59,7 +49,7 @@ def __exit__( exc_value: Optional[BaseException], traceback: Optional[TracebackType], ) -> None: - self.docker.close() + pass @util.hook def post_parse(self, recipe: Recipe) -> None: @@ -164,13 +154,13 @@ def _make_arch( packages: Optional[List[Package]] = None, ) -> bool: self.post_parse(recipe) - - src_dir = os.path.join(build_dir, "src") - os.makedirs(src_dir, exist_ok=True) - - self._fetch_sources(recipe, src_dir) - self.post_fetch_sources(recipe, src_dir) - + if self.source_dir: + src_dir = self.source_dir + else: + src_dir = os.path.join(build_dir, "src") + os.makedirs(src_dir, exist_ok=True) + self._fetch_sources(recipe, src_dir) + self.post_fetch_sources(recipe, src_dir) self._prepare(recipe, src_dir) self.post_prepare(recipe, src_dir) @@ -274,8 +264,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None: for filename in util.list_tree(src_dir): os.utime(filename, (epoch, epoch)) - mount_src = "/src" - repo_src = "/repo" uid = os.getuid() pre_script: List[str] = [] @@ -331,33 +319,17 @@ def _build(self, recipe: Recipe, src_dir: str) -> None: " -- " + " ".join(host_deps), ) ) - - logs = bash.run_script_in_container( - self.docker, - image=self.IMAGE_PREFIX + recipe.image, - mounts=[ - docker.types.Mount( - type="bind", - source=os.path.abspath(src_dir), - target=mount_src, - ), - docker.types.Mount( - type="bind", - source=os.path.abspath(self.dist_dir), - target=repo_src, - ), - ], - variables={ - "srcdir": mount_src, - }, + logs = bash.run_script( script="\n".join( ( + f'cd "{src_dir}"', *pre_script, - f'cd "{mount_src}"', recipe.build, - f'chown -R {uid}:{uid} "{mount_src}"', ) ), + variables={ + "srcdir": src_dir, + }, ) bash.pipe_logs(logger, logs, "build()") diff --git a/toltec/util.py b/toltec/util.py index 8871829..eb08161 100644 --- a/toltec/util.py +++ b/toltec/util.py @@ -19,7 +19,7 @@ IO, List, Optional, - Protocol, +# Protocol, Type, Union, ) @@ -302,6 +302,8 @@ def check_directory(path: str, message: str) -> bool: try: os.mkdir(path) except FileExistsError: + if not os.listdir(path): + return True ans = query_user( message, default="c", @@ -343,18 +345,15 @@ def list_tree(root: str) -> List[str]: HookTrigger = Callable[..., None] HookListener = Callable[..., None] +# Protocol is not available in python 3.7 (Debian buster) +Hook = Any +# class Hook(Protocol): # pylint:disable=too-few-public-methods +# """Protocol for hooks.""" -class Hook(Protocol): # pylint:disable=too-few-public-methods - """Protocol for hooks.""" - - @staticmethod - def register(new_listener: HookListener) -> None: - """Add a new listener to this hook.""" - ... - - # Invoke all listeners for this hook - __call__: HookTrigger - +# @staticmethod +# def register(new_listener: HookListener) -> None: +# """Add a new listener to this hook.""" +# ... def hook(func: HookTrigger) -> Hook: """ diff --git a/toltecmk b/toltecmk new file mode 120000 index 0000000..5dae959 --- /dev/null +++ b/toltecmk @@ -0,0 +1 @@ +toltecmk.py \ No newline at end of file diff --git a/toltecmk-contained b/toltecmk-contained new file mode 100755 index 0000000..5cb4170 --- /dev/null +++ b/toltecmk-contained @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Toltec Contributors +# SPDX-License-Identifier: MIT + +import argparse +import os +import subprocess +from pathlib import Path + +from toltec import parse_recipe +from toltecmk import make_argparser + +# Prefix for all Toltec Docker images +IMAGE_PREFIX = "ghcr.io/toltec-dev/" + +args = make_argparser().parse_args() + +recipe_bundle = parse_recipe(args.recipe_dir) +# Don't recipes in a bundle share most information? +recipe = next(iter(recipe_bundle.values())) + +image = IMAGE_PREFIX + recipe.image + +toltecmk_dir = str(Path(__file__).resolve().parent) + +mounts = [ (args.recipe_dir, '/recipe'), + (args.work_dir, '/build'), + (args.dist_dir, '/pkg'), + (toltecmk_dir, '/toltecmk'), + ] + +for m in mounts: + if not os.path.exists(m[0]): + os.makedirs(m[0]) + +if args.source_dir: + mounts.append((args.source_dir, '/src')) + +cmd = ['podman', 'run', '-it', '--rm'] +for m in mounts: + cmd.append('-v') + cmd.append(':'.join(m)) +cmd.append(image) +cmd += ['bash', '-c'] +# currently required by toltecmk and missing in images +internal_cmd = [] +internal_cmd += ['apt update && apt install -yq python3-dateutil python3-requests && '] +internal_cmd += ['/toltecmk/toltecmk', + '-w', '/build', + '-d', '/pkg', + '/recipe'] +if args.source_dir: internal_cmd += ['-s', '/src'] +if args.arch_name: internal_cmd += ['-a', args.arch_name] +if args.package_name: internal_cmd += ['-p', args.package_name] + +cmd.append(' '.join(internal_cmd)) +print(' '.join('"' + c + '"' if ' ' in c else c for c in cmd)) +subprocess.call(cmd) + diff --git a/toltecmk.py b/toltecmk.py new file mode 100755 index 0000000..6205eaa --- /dev/null +++ b/toltecmk.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Toltec Contributors +# SPDX-License-Identifier: MIT +"""Build packages from the recipe in [DIR].""" + +import argparse +import logging +import os +import sys +from typing import Dict, List, Optional +from toltec import parse_recipe +from toltec.builder import Builder +from toltec.recipe import Package +from toltec.repo import make_index +from toltec.util import argparse_add_verbose, LOGGING_FORMAT + +def make_argparser(): + parser = argparse.ArgumentParser(description=__doc__) + + parser.add_argument( + "recipe_dir", + metavar="DIR", + nargs="?", + default=os.getcwd(), + help="""path to a directory containing the recipe to build + (default: current directory)""", + ) + + parser.add_argument( + "-w", + "--work-dir", + metavar="DIR", + default=os.path.join(os.getcwd(), "build"), + help="""path to a directory used for building the package + (default: [current directory]/build)""", + ) + + parser.add_argument( + "-d", + "--dist-dir", + metavar="DIR", + default=os.path.join(os.getcwd(), "dist"), + help="""path to a directory where built packages are stored + (default: [current directory]/dist)""", + ) + + parser.add_argument( + "-s", + "--source-dir", + metavar="DIR", + default=None, + help="""path to the source directory + (optional: when specified, a local build is performed instead of fetching + sources)""", + ) + + parser.add_argument( + "-a", + "--arch-name", + metavar="ARCHNAME", + action="append", + help="""only build for the given architecture (can + be repeated)""", + ) + + parser.add_argument( + "-p", + "--package-name", + metavar="PACKAGENAME", + action="append", + help="""list of packages to build (default: all packages from the recipe, + can be repeated)""", + ) + + argparse_add_verbose(parser) + return parser + +if __name__ == '__main__': + args = make_argparser().parse_args() + logging.basicConfig(format=LOGGING_FORMAT, level=args.verbose) + + recipe_bundle = parse_recipe(args.recipe_dir) + builder = Builder(args.work_dir, args.dist_dir, args.source_dir) + + build_matrix: Optional[Dict[str, Optional[List[Package]]]] = None + + if args.arch_name or args.package_name: + build_matrix = {} + + for arch, recipes in recipe_bundle.items(): + if args.package_name: + build_matrix[arch] = [ + recipes.packages[pkg_name] for pkg_name in args.package_name + ] + else: + build_matrix[arch] = None + + if not builder.make(recipe_bundle, build_matrix): + sys.exit(1) + + make_index(args.dist_dir)