Feat podman runner #17786
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,276 @@ | ||
| from collections import namedtuple | ||
| import os | ||
| import json | ||
| import platform | ||
| import shutil | ||
| import yaml | ||
| from conan.api.model import ListPattern | ||
| from conan.api.output import Color, ConanOutput | ||
| from conan.cli import make_abs_path | ||
| from conan.internal.runner import RunnerException | ||
| from conan.errors import ConanException | ||
| from conan.internal.model.version import Version | ||
|
|
||
| def config_parser(file_path): | ||
| Build = namedtuple('Build', ['dockerfile', 'build_context', 'build_args', 'cache_from']) | ||
| Run = namedtuple('Run', ['name', 'environment', 'user', 'privileged', 'cap_add', 'security_opt', 'volumes', 'network']) | ||
| Conf = namedtuple('Conf', ['image', 'build', 'run']) | ||
| if file_path: | ||
| def _instans_or_error(value, obj): | ||
| if value and (not isinstance(value, obj)): | ||
| raise ConanException(f"podman runner configfile syntax error: {value} must be a {obj.__name__}") | ||
| return value | ||
| with open(file_path, 'r') as f: | ||
| runnerfile = yaml.safe_load(f) | ||
| return Conf( | ||
| image=_instans_or_error(runnerfile.get('image'), str), | ||
| build=Build( | ||
| dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str), | ||
| build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str), | ||
| build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict), | ||
| cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list), | ||
| ), | ||
| run=Run( | ||
| name=_instans_or_error(runnerfile.get('run', {}).get('name'), str), | ||
| environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict), | ||
| user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str), | ||
| privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool), | ||
| cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list), | ||
| security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), list), | ||
| volumes=_instans_or_error(runnerfile.get('run', {}).get('volumes'), dict), | ||
| network=_instans_or_error(runnerfile.get('run', {}).get('network'), str), | ||
| ) | ||
| ) | ||
| else: | ||
| return Conf( | ||
| image=None, | ||
| build=Build(dockerfile=None, build_context=None, build_args=None, cache_from=None), | ||
| run=Run(name=None, environment=None, user=None, privileged=None, cap_add=None, | ||
| security_opt=None, volumes=None, network=None) | ||
| ) | ||
|
|
||
|
|
||
| def _podman_info(msg, error=False): | ||
| fg=Color.BRIGHT_MAGENTA | ||
| if error: | ||
| fg=Color.BRIGHT_RED | ||
| ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) | ||
| ConanOutput().status(f'│ {msg} │', fg=fg) | ||
| ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) | ||
|
|
||
|
|
||
| class PodmanRunner: | ||
| def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): | ||
| import podman | ||
| try: | ||
| podman_base_urls = [ | ||
| host_profile.runner.get('socket'), # Socket from profile | ||
| os.environ.get('DOCKER_HOST'), # Connect to socket defined in DOCKER_HOST | ||
| os.environ.get('CONTAINER_HOST'), # Connect to socket defined in CONTAINER_HOST | ||
| 'unix:///var/run/podman/podman.sock', # Default root linux socket | ||
| f'unix://{os.environ.get("XDG_RUNTIME_DIR")}/podman/podman.sock', # User linux socket | ||
| f'unix:///var/run/user/{os.environ.get("UID")}/podman/podman.sock' # Default user linux socket | ||
| ] | ||
| for base_url in podman_base_urls: | ||
| try: | ||
| ConanOutput().verbose(f'Trying to connect to podman "{base_url or "default"}" socket') | ||
| self.podman_client = podman.PodmanClient(base_url=base_url) | ||
| if self.podman_client.ping(): | ||
| ConanOutput().verbose(f'Connected to podman "{base_url or "default"}" socket') | ||
| break | ||
| except: | ||
| continue | ||
| except: | ||
| raise ConanException("Podman client failed to initialize." | ||
| "\n - Check if podman is installed and running" | ||
| "\n - Run 'pip install conan[runners]'") | ||
| self.conan_api = conan_api | ||
| self.build_profile = build_profile | ||
| self.args = args | ||
| self.abs_host_path = make_abs_path(args.path) | ||
| if args.format: | ||
| raise ConanException("format argument is forbidden if running in a podman runner") | ||
|
|
||
| # Container config | ||
| # https://containers.dev/implementors/json_reference/ | ||
| self.configfile = config_parser(host_profile.runner.get('configfile')) | ||
| self.dockerfile = host_profile.runner.get('dockerfile') or self.configfile.build.dockerfile | ||
| self.podman_build_context = host_profile.runner.get('build_context') or self.configfile.build.build_context | ||
| self.image = host_profile.runner.get('image') or self.configfile.image | ||
| if not (self.dockerfile or self.image): | ||
| raise ConanException("either 'dockerfile' or container image name is needed") | ||
| self.image = self.image or 'conan-runner-default' | ||
| self.name = self.configfile.run.name or f'conan-runner-{host_profile.runner.get("suffix", "podman")}' | ||
| self.remove = str(host_profile.runner.get('remove', 'false')).lower() == 'true' | ||
| self.cache = str(host_profile.runner.get('cache', 'clean')) | ||
| self.container = None | ||
|
|
||
| # Runner config | ||
| self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') | ||
| self.podman_user_name = self.configfile.run.user or 'root' | ||
| self.podman_user_home = f'/{"home/" if self.podman_user_name != "root" else ""}{self.podman_user_name}' | ||
| self.abs_podman_path = os.path.join(f'{self.podman_user_home}/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") | ||
| self.selinux_host = host_profile.runner.get('selinux') | ||
|
|
||
| # Update conan command and some paths to run inside the container | ||
| raw_args[raw_args.index(args.path)] = self.abs_podman_path | ||
| self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) | ||
|
|
||
| def run(self): | ||
| """ | ||
| run conan inside a Podman container | ||
| """ | ||
| if self.dockerfile: | ||
| _podman_info(f'Building the container image: {self.image}') | ||
| self.build_image() | ||
| volumes, environment = self.create_runner_environment() | ||
| error = False | ||
| try: | ||
| if self.podman_client.containers.list(all=True, filters={'name': self.name}): | ||
| _podman_info('Starting the container') | ||
| self.container = self.podman_client.containers.get(self.name) | ||
| self.container.start() | ||
| else: | ||
| if self.configfile.run.environment: | ||
| environment.update(self.configfile.run.environment) | ||
| if self.configfile.run.volumes: | ||
| volumes.update(self.configfile.run.volumes) | ||
| _podman_info('Creating the container') | ||
| self.container = self.podman_client.containers.run( | ||
| self.image, | ||
| ["/bin/bash", "-c", "while true; do sleep 30; done"], | ||
| name=self.name, | ||
| volumes=volumes, | ||
| environment=environment, | ||
| user=self.configfile.run.user, | ||
| privileged=self.configfile.run.privileged, | ||
| cap_add=self.configfile.run.cap_add, | ||
| security_opt=self.configfile.run.security_opt, | ||
| detach=True, | ||
| auto_remove=False, | ||
| networks=self.configfile.run.network) | ||
| _podman_info(f'Container {self.name} running') | ||
| except Exception as e: | ||
| raise ConanException(f'Impossible to run the container "{self.name}" with image "{self.image}"' | ||
| f'\n\n{str(e)}') | ||
| try: | ||
| self.init_container() | ||
| self.run_command(self.command) | ||
| self.update_local_cache() | ||
| except ConanException as e: | ||
| error = True | ||
| raise e | ||
| except RunnerException as e: | ||
| error = True | ||
| raise ConanException(f'"{e.command}" inside container fail' | ||
| f'\n\nLast command output: {str(e.stdout_log)}') | ||
| finally: | ||
| if self.container: | ||
| error_prefix = 'ERROR: ' if error else '' | ||
| _podman_info(f'{error_prefix}Stopping container', error) | ||
| self.container.stop() | ||
| if self.remove: | ||
| _podman_info(f'{error_prefix}Removing container', error) | ||
| self.container.remove() | ||
|
|
||
| def build_image(self): | ||
| dockerfile_file_path = self.dockerfile | ||
| if os.path.isdir(self.dockerfile): | ||
| for df in ['Containerfile', 'Dockerfile']: | ||
| dockerfile_file_path = os.path.join(self.dockerfile, df) | ||
| if os.path.exists(dockerfile_file_path): break | ||
| build_path = self.podman_build_context or os.path.dirname(dockerfile_file_path) | ||
| ConanOutput().highlight(f"Container recipe path: '{dockerfile_file_path}'") | ||
| ConanOutput().highlight(f"Container build context: '{build_path}'\n") | ||
| _, podman_build_logs = self.podman_client.images.build( | ||
| dockerfile=dockerfile_file_path, | ||
| path=build_path, | ||
| tag=self.image, | ||
| buildargs=self.configfile.build.build_args, | ||
| cache_from=self.configfile.build.cache_from, | ||
| ) | ||
| for chunk in podman_build_logs: | ||
| for line in chunk.decode("utf-8").split('\r\n'): | ||
| if line: | ||
| stream = json.loads(line).get('stream') | ||
| if stream: | ||
| ConanOutput().status(stream.strip()) | ||
|
Comment on lines
+185
to
+197
There was a problem hiding this comment. No streaming of the output here. The output of the |
||
|
|
||
| def run_command(self, command, workdir=None, log=True): | ||
| workdir = workdir or self.abs_podman_path | ||
| if log: | ||
| _podman_info(f'Running in container: "{command}"') | ||
| _, exec_output = self.container.exec_run(f"/bin/bash -c '{command}'", stream=True, workdir=workdir, demux=True) | ||
|
There was a problem hiding this comment. Here, if I add |
||
| stderr_log, stdout_log = '', '' | ||
| try: | ||
| for (stdout_out, stderr_out) in exec_output: | ||
| if stdout_out is not None: | ||
| stdout_log += stdout_out.decode('utf-8', errors='ignore').strip() | ||
| if log: | ||
| ConanOutput().status(stdout_out.decode('utf-8', errors='ignore').strip()) | ||
| if stderr_out is not None: | ||
| stderr_log += stderr_out.decode('utf-8', errors='ignore').strip() | ||
| if log: | ||
| ConanOutput().status(stderr_out.decode('utf-8', errors='ignore').strip()) | ||
| except Exception as e: | ||
| if platform.system() == 'Windows': | ||
| import pywintypes | ||
| if isinstance(e, pywintypes.error): | ||
| pass | ||
| else: | ||
| raise e | ||
| #exit_metadata = self.docker_api.exec_inspect(exec_instance['Id']) | ||
| #if exit_metadata['Running'] or exit_metadata['ExitCode'] > 0: | ||
| # raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log) | ||
|
Comment on lines
+222
to
+224
There was a problem hiding this comment. There seems to be no way to check for the command exit code while using |
||
| return stdout_log, stderr_log | ||
|
|
||
| def create_runner_environment(self): | ||
| shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) | ||
| mode = "Z" if self.selinux_host else "rw" | ||
| volumes = {self.abs_host_path: {'bind': self.abs_podman_path, 'mode': mode}} | ||
| environment = {'CONAN_RUNNER_ENVIRONMENT': '1'} | ||
|
|
||
| if self.cache == 'shared': | ||
| volumes[self.conan_api.home_folder] = {'bind': f'{self.podman_user_home}/.conan2', 'mode': mode} | ||
|
|
||
| if self.cache in ['clean', 'copy']: | ||
| # Copy all conan profiles and config files to container workspace | ||
| os.mkdir(self.abs_runner_home_path) | ||
| shutil.copytree( | ||
| os.path.join(self.conan_api.home_folder, 'profiles'), | ||
| os.path.join(self.abs_runner_home_path, 'profiles') | ||
| ) | ||
| for file_name in ['global.conf', 'settings.yml', 'remotes.json']: | ||
| src_file = os.path.join(self.conan_api.home_folder, file_name) | ||
| if os.path.exists(src_file): | ||
| shutil.copy(src_file, os.path.join(self.abs_runner_home_path, file_name)) | ||
|
|
||
| if self.cache == 'copy': | ||
| tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz') | ||
| _podman_info(f'Save host cache in: {tgz_path}') | ||
| self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path) | ||
| return volumes, environment | ||
|
|
||
| def init_container(self): | ||
| min_conan_version = '2.1' | ||
| stdout, _ = self.run_command('conan --version', log=True) | ||
| podman_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color | ||
| if Version(podman_conan_version) <= Version(min_conan_version): | ||
| ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED) | ||
| raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') | ||
| if self.cache != 'shared': | ||
| self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) | ||
| self.run_command('cp -r "'+self.abs_podman_path+'/.conanrunner/profiles/." "${HOME}/.conan2/profiles/."', log=False) | ||
| for file_name in ['global.conf', 'settings.yml', 'remotes.json']: | ||
| if os.path.exists( os.path.join(self.abs_runner_home_path, file_name)): | ||
| self.run_command('cp "'+self.abs_podman_path+'/.conanrunner/'+file_name+'" "${HOME}/.conan2/'+file_name+'"', log=False) | ||
| if self.cache in ['copy']: | ||
| self.run_command('conan cache restore "'+self.abs_podman_path+'/.conanrunner/local_cache_save.tgz"') | ||
|
|
||
| def update_local_cache(self): | ||
| if self.cache != 'shared': | ||
| self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False) | ||
| self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_podman_path+'"/.conanrunner/podman_cache_save.tgz') | ||
| tgz_path = os.path.join(self.abs_runner_home_path, 'podman_cache_save.tgz') | ||
| _podman_info(f'Restore host cache from: {tgz_path}') | ||
| package_list = self.conan_api.cache.restore(tgz_path) | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,3 @@ | ||
| paramiko | ||
| docker>=7.1.0 | ||
| podman @ https://github.com/containers/[email protected] |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Here I'm placing the
conanrunnerfolder in the user home. This is slightly different from what is done for the docker runner.