diff --git a/src/middlewared/middlewared/alembic/versions/25.04/2025-01-13_20-25_app_registries.py b/src/middlewared/middlewared/alembic/versions/25.04/2025-01-13_20-25_app_registries.py new file mode 100644 index 0000000000000..ad34a30f6086c --- /dev/null +++ b/src/middlewared/middlewared/alembic/versions/25.04/2025-01-13_20-25_app_registries.py @@ -0,0 +1,34 @@ +""" +App registries support + +Revision ID: 799718dc329e +Revises: 899852cb2a92 +Create Date: 2025-01-13 20:25:41.855489+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +revision = '799718dc329e' +down_revision = '899852cb2a92' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'app_registry', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('description', sa.String(length=512), nullable=True, default=None), + sa.Column('username', sa.String(length=255), nullable=False), + sa.Column('password', sa.String(length=255), nullable=False), + sa.Column('uri', sa.String(length=512), nullable=False, unique=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_app_registry')), + sqlite_autoincrement=True, + ) + + +def downgrade(): + pass diff --git a/src/middlewared/middlewared/api/v25_04_0/__init__.py b/src/middlewared/middlewared/api/v25_04_0/__init__.py index 8c5b469ed676b..ac7f70d702dee 100644 --- a/src/middlewared/middlewared/api/v25_04_0/__init__.py +++ b/src/middlewared/middlewared/api/v25_04_0/__init__.py @@ -7,6 +7,7 @@ from .app import * # noqa from .app_image import * # noqa from .app_ix_volume import * # noqa +from .app_registry import * # noqa from .auth import * # noqa from .boot_environments import * # noqa from .catalog import * # noqa diff --git a/src/middlewared/middlewared/api/v25_04_0/app_image.py b/src/middlewared/middlewared/api/v25_04_0/app_image.py index 27e1b4159dfd7..2730624a09726 100644 --- a/src/middlewared/middlewared/api/v25_04_0/app_image.py +++ b/src/middlewared/middlewared/api/v25_04_0/app_image.py @@ -50,6 +50,7 @@ class AppImageDockerhubRateLimitResult(BaseModel): class AppImageAuthConfig(BaseModel): username: str password: str + registry_uri: str | None = None @single_argument_args('image_pull') diff --git a/src/middlewared/middlewared/api/v25_04_0/app_registry.py b/src/middlewared/middlewared/api/v25_04_0/app_registry.py new file mode 100644 index 0000000000000..1dd27116f40ab --- /dev/null +++ b/src/middlewared/middlewared/api/v25_04_0/app_registry.py @@ -0,0 +1,52 @@ +from pydantic import Secret + +from middlewared.api.base import BaseModel, Excluded, excluded_field, ForUpdateMetaclass + + +__all__ = [ + 'AppRegistryEntry', 'AppRegistryCreateArgs', 'AppRegistryCreateResult', 'AppRegistryUpdateArgs', + 'AppRegistryUpdateResult', 'AppRegistryDeleteArgs', 'AppRegistryDeleteResult', +] + + +class AppRegistryEntry(BaseModel): + id: int + name: str + description: str | None = None + username: Secret[str] + password: Secret[str] + uri: str + + +class AppRegistryCreate(AppRegistryEntry): + id: Excluded = excluded_field() + uri: str = 'https://registry-1.docker.io/' + + +class AppRegistryCreateArgs(BaseModel): + app_registry_create: AppRegistryCreate + + +class AppRegistryCreateResult(BaseModel): + result: AppRegistryEntry + + +class AppRegistryUpdate(AppRegistryCreate, metaclass=ForUpdateMetaclass): + pass + + +class AppRegistryUpdateArgs(BaseModel): + id: int + data: AppRegistryUpdate + + +class AppRegistryUpdateResult(BaseModel): + result: AppRegistryEntry + + +class AppRegistryDeleteArgs(BaseModel): + id: int + + +class AppRegistryDeleteResult(BaseModel): + result: None diff --git a/src/middlewared/middlewared/etc_files/docker/config.json.py b/src/middlewared/middlewared/etc_files/docker/config.json.py new file mode 100644 index 0000000000000..bbc17e33d00cc --- /dev/null +++ b/src/middlewared/middlewared/etc_files/docker/config.json.py @@ -0,0 +1,15 @@ +import json +import os + +from middlewared.plugins.app_registry.utils import generate_docker_auth_config +from middlewared.plugins.etc import FileShouldNotExist + + +def render(service, middleware): + config = middleware.call_sync('docker.config') + if not config['pool']: + raise FileShouldNotExist() + + os.makedirs('/etc/docker', exist_ok=True) + + return json.dumps(generate_docker_auth_config(middleware.call_sync('app.registry.query'))) diff --git a/src/middlewared/middlewared/plugins/app_registry/__init__.py b/src/middlewared/middlewared/plugins/app_registry/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/middlewared/middlewared/plugins/app_registry/crud.py b/src/middlewared/middlewared/plugins/app_registry/crud.py new file mode 100644 index 0000000000000..d3a6db192c158 --- /dev/null +++ b/src/middlewared/middlewared/plugins/app_registry/crud.py @@ -0,0 +1,94 @@ +import middlewared.sqlalchemy as sa +from middlewared.api import api_method +from middlewared.api.current import ( + AppRegistryEntry, AppRegistryCreateArgs, AppRegistryCreateResult, AppRegistryUpdateArgs, + AppRegistryUpdateResult, AppRegistryDeleteArgs, AppRegistryDeleteResult, +) +from middlewared.service import CRUDService, private, ValidationErrors + +from .validate_registry import validate_registry_credentials + + +class AppRegistryModel(sa.Model): + __tablename__ = 'app_registry' + + id = sa.Column(sa.Integer(), primary_key=True) + name = sa.Column(sa.String(255), nullable=False) + description = sa.Column(sa.String(512), nullable=True, default=None) + username = sa.Column(sa.EncryptedText(), nullable=False) + password = sa.Column(sa.EncryptedText(), nullable=False) + uri = sa.Column(sa.String(512), nullable=False, unique=True) + + +class AppRegistryService(CRUDService): + + class Config: + namespace = 'app.registry' + datastore = 'app.registry' + cli_namespace = 'app.registry' + entry = AppRegistryEntry + role_prefix = 'APPS' + + @private + async def validate(self, data, old=None, schema='app_registry_create'): + verrors = ValidationErrors() + + filters = [['id', '!=', old['id']]] if old else [] + if await self.query([['name', '=', data['name']]] + filters): + verrors.add(f'{schema}.name', 'Name must be unique') + + if data['uri'].startswith('http') and not data['uri'].endswith('/'): + # We can have 2 formats basically + # https://index.docker.io/v1/ + # registry-1.docker.io + # We would like to have a trailing slash here because we are not able to pull images without it + # if http based url is provided + data['uri'] = data['uri'] + '/' + + if await self.query([['uri', '=', data['uri']]] + filters): + verrors.add(f'{schema}.uri', 'URI must be unique') + + if not verrors and await self.middleware.run_in_thread( + validate_registry_credentials, data['uri'], data['username'], data['password'] + ) is False: + verrors.add(f'{schema}.uri', 'Invalid credentials for registry') + + verrors.check() + + @api_method(AppRegistryCreateArgs, AppRegistryCreateResult, roles=['APPS_WRITE']) + async def do_create(self, data): + """ + Create an app registry entry. + """ + await self.middleware.call('docker.state.validate') + await self.validate(data) + id_ = await self.middleware.call('datastore.insert', 'app.registry', data) + await self.middleware.call('etc.generate', 'app_registry') + return await self.get_instance(id_) + + @api_method(AppRegistryUpdateArgs, AppRegistryUpdateResult, roles=['APPS_WRITE']) + async def do_update(self, id_, data): + """ + Update an app registry entry. + """ + await self.middleware.call('docker.state.validate') + old = await self.get_instance(id_) + new = old.copy() + new.update(data) + + await self.validate(new, old=old, schema='app_registry_update') + + await self.middleware.call('datastore.update', 'app.registry', id_, new) + + await self.middleware.call('etc.generate', 'app_registry') + return await self.get_instance(id_) + + @api_method(AppRegistryDeleteArgs, AppRegistryDeleteResult, roles=['APPS_WRITE']) + async def do_delete(self, id_): + """ + Delete an app registry entry. + """ + await self.middleware.call('docker.state.validate') + await self.get_instance(id_) + await self.middleware.call('datastore.delete', 'app.registry', id_) + await self.middleware.call('etc.generate', 'app_registry') diff --git a/src/middlewared/middlewared/plugins/app_registry/utils.py b/src/middlewared/middlewared/plugins/app_registry/utils.py new file mode 100644 index 0000000000000..c26e6bed22787 --- /dev/null +++ b/src/middlewared/middlewared/plugins/app_registry/utils.py @@ -0,0 +1,14 @@ +import base64 + + +def generate_docker_auth_config(auth_list: list[dict[str, str]]) -> dict: + auths = {} + for auth in auth_list: + auths[auth['uri']] = { + # Encode username:password in base64 + 'auth': base64.b64encode(f'{auth["username"]}:{auth["password"]}'.encode()).decode(), + } + + return { + 'auths': auths, + } diff --git a/src/middlewared/middlewared/plugins/app_registry/validate_registry.py b/src/middlewared/middlewared/plugins/app_registry/validate_registry.py new file mode 100644 index 0000000000000..60b7328ae676b --- /dev/null +++ b/src/middlewared/middlewared/plugins/app_registry/validate_registry.py @@ -0,0 +1,26 @@ +from docker.errors import APIError, DockerException + +from middlewared.plugins.apps.ix_apps.docker.utils import get_docker_client + + +def validate_registry_credentials(registry: str, username: str, password: str) -> bool: + """ + Validates Docker registry credentials using the Docker SDK. + + Args: + registry (str): The URL of the Docker registry (e.g., "registry1.example.com"). + username (str): The username for the registry. + password (str): The password for the registry. + + Returns: + bool: True if the credentials are valid, False otherwise. + """ + with get_docker_client() as client: + try: + client.login(username=username, password=password, registry=registry) + except (APIError, DockerException): + return False + else: + return True + + return False diff --git a/src/middlewared/middlewared/plugins/apps/compose_utils.py b/src/middlewared/middlewared/plugins/apps/compose_utils.py index edeee581c2980..84029578bedfc 100644 --- a/src/middlewared/middlewared/plugins/apps/compose_utils.py +++ b/src/middlewared/middlewared/plugins/apps/compose_utils.py @@ -51,7 +51,7 @@ def compose_action( raise CallError(f'Invalid action {action!r} for app {app_name!r}') # TODO: We will likely have a configurable timeout on this end - cp = run(['docker', 'compose'] + compose_files + args, timeout=1200) + cp = run(['docker', '--config', '/etc/docker', 'compose'] + compose_files + args, timeout=1200) if cp.returncode != 0: logger.error('Failed %r action for %r app: %s', action, app_name, cp.stderr) err_msg = f'Failed {action!r} action for {app_name!r} app.' diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/docker/images.py b/src/middlewared/middlewared/plugins/apps/ix_apps/docker/images.py index 4e6a336e637a8..e68b7d726136d 100644 --- a/src/middlewared/middlewared/plugins/apps/ix_apps/docker/images.py +++ b/src/middlewared/middlewared/plugins/apps/ix_apps/docker/images.py @@ -18,7 +18,8 @@ def list_images() -> list[dict]: def pull_image( - image_tag: str, callback: typing.Callable = None, username: str | None = None, password: str | None = None + image_tag: str, callback: typing.Callable = None, username: str | None = None, password: str | None = None, + registry_uri: str | None = None, ): if username and not password: raise CallError('Password is required when username is provided') @@ -29,6 +30,7 @@ def pull_image( auth_config = { 'username': username, 'password': password, + 'registry': registry_uri, } if username else None with get_docker_client() as client: diff --git a/src/middlewared/middlewared/plugins/apps_images/images.py b/src/middlewared/middlewared/plugins/apps_images/images.py index e49b08f8ac245..16b56be9cbb59 100644 --- a/src/middlewared/middlewared/plugins/apps_images/images.py +++ b/src/middlewared/middlewared/plugins/apps_images/images.py @@ -6,7 +6,7 @@ from middlewared.service import CRUDService, filterable, job from middlewared.utils import filter_list -from .utils import parse_tags +from .utils import get_normalized_auth_config, parse_tags class AppImageService(CRUDService): @@ -77,9 +77,20 @@ def callback(entry): job.set_progress((progress['current']/progress['total']) * 90, 'Pulling image') self.middleware.call_sync('docker.state.validate') - auth_config = data['auth_config'] or {} image_tag = data['image'] - pull_image(image_tag, callback, auth_config.get('username'), auth_config.get('password')) + auth_config = data['auth_config'] or {} + if not auth_config: + # If user has not provided any auth creds, we will try to see if the registry to which the image + # belongs to, we already have it's creds and if yes we will try to use that when pulling the image + app_registries = { + registry['uri']: registry for registry in self.middleware.call_sync('app.registry.query') + } + auth_config = get_normalized_auth_config(app_registries, image_tag) + + pull_image( + image_tag, callback, auth_config.get('username'), auth_config.get('password'), + auth_config.get('registry_uri'), + ) job.set_progress(100, f'{image_tag!r} image pulled successfully') @api_method(AppImageDeleteArgs, AppImageDeleteResult) diff --git a/src/middlewared/middlewared/plugins/apps_images/utils.py b/src/middlewared/middlewared/plugins/apps_images/utils.py index 158b33974f475..a3cd84777a9d2 100644 --- a/src/middlewared/middlewared/plugins/apps_images/utils.py +++ b/src/middlewared/middlewared/plugins/apps_images/utils.py @@ -153,3 +153,18 @@ def normalize_docker_limits_header(headers: dict) -> dict: 'remaining_time_limit_in_secs': int(remaining_time_limit), 'error': None, } + + +def get_normalized_auth_config(registry_info: dict[str, dict], image_tag: str) -> dict: + if not registry_info: + return {} + + user_wants_registry = normalize_reference(image_tag)['registry'] + if user_wants_registry not in registry_info: + return {} + + return { + 'registry_uri': user_wants_registry, + 'username': registry_info[user_wants_registry]['username'], + 'password': registry_info[user_wants_registry]['password'], + } diff --git a/src/middlewared/middlewared/plugins/etc.py b/src/middlewared/middlewared/plugins/etc.py index 8384f5285e80c..e88a69ef78cd1 100644 --- a/src/middlewared/middlewared/plugins/etc.py +++ b/src/middlewared/middlewared/plugins/etc.py @@ -75,6 +75,9 @@ class EtcService(Service): ] }, + 'app_registry': [ + {'type': 'py', 'path': 'docker/config.json'}, + ], 'docker': [ {'type': 'py', 'path': 'docker/daemon.json'}, ], diff --git a/src/middlewared/middlewared/plugins/service_/services/docker.py b/src/middlewared/middlewared/plugins/service_/services/docker.py index 8c97b525e2e4f..0a0fd18398181 100644 --- a/src/middlewared/middlewared/plugins/service_/services/docker.py +++ b/src/middlewared/middlewared/plugins/service_/services/docker.py @@ -7,7 +7,7 @@ class DockerService(SimpleService): name = 'docker' - etc = ['docker'] + etc = ['app_registry', 'docker'] systemd_unit = 'docker' async def before_start(self):