Allow logging in to different docker registries (#15346)

src/middlewared/middlewared/alembic/versions/25.04/2025-01-13_20-25_app_registries.py

@@ -0,0 +1,34 @@
+"""
+App registries support
+
+Revision ID: 799718dc329e
+Revises: 899852cb2a92
+Create Date: 2025-01-13 20:25:41.855489+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = '799718dc329e'
+down_revision = '899852cb2a92'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        'app_registry',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False),
+        sa.Column('description', sa.String(length=512), nullable=True, default=None),
+        sa.Column('username', sa.String(length=255), nullable=False),
+        sa.Column('password', sa.String(length=255), nullable=False),
+        sa.Column('uri', sa.String(length=512), nullable=False, unique=True),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_app_registry')),
+        sqlite_autoincrement=True,
+    )
+
+
+def downgrade():
+    pass

src/middlewared/middlewared/api/v25_04_0/__init__.py

@@ -7,6 +7,7 @@
 from .app import *  # noqa
 from .app_image import *  # noqa
 from .app_ix_volume import *  # noqa
+from .app_registry import *  # noqa
 from .auth import *  # noqa
 from .boot_environments import *  # noqa
 from .catalog import *  # noqa

src/middlewared/middlewared/api/v25_04_0/app_image.py

@@ -50,6 +50,7 @@ class AppImageDockerhubRateLimitResult(BaseModel):
 class AppImageAuthConfig(BaseModel):
     username: str
     password: str
+    registry_uri: str | None = None
 
 
 @single_argument_args('image_pull')

src/middlewared/middlewared/api/v25_04_0/app_registry.py

@@ -0,0 +1,52 @@
+from pydantic import Secret
+
+from middlewared.api.base import BaseModel, Excluded, excluded_field, ForUpdateMetaclass
+
+
+__all__ = [
+    'AppRegistryEntry', 'AppRegistryCreateArgs', 'AppRegistryCreateResult', 'AppRegistryUpdateArgs',
+    'AppRegistryUpdateResult', 'AppRegistryDeleteArgs', 'AppRegistryDeleteResult',
+]
+
+
+class AppRegistryEntry(BaseModel):
+    id: int
+    name: str
+    description: str | None = None
+    username: Secret[str]
+    password: Secret[str]
+    uri: str
+
+
+class AppRegistryCreate(AppRegistryEntry):
+    id: Excluded = excluded_field()
+    uri: str = 'https://registry-1.docker.io/'
+
+
+class AppRegistryCreateArgs(BaseModel):
+    app_registry_create: AppRegistryCreate
+
+
+class AppRegistryCreateResult(BaseModel):
+    result: AppRegistryEntry
+
+
+class AppRegistryUpdate(AppRegistryCreate, metaclass=ForUpdateMetaclass):
+    pass
+
+
+class AppRegistryUpdateArgs(BaseModel):
+    id: int
+    data: AppRegistryUpdate
+
+
+class AppRegistryUpdateResult(BaseModel):
+    result: AppRegistryEntry
+
+
+class AppRegistryDeleteArgs(BaseModel):
+    id: int
+
+
+class AppRegistryDeleteResult(BaseModel):
+    result: None

src/middlewared/middlewared/etc_files/docker/config.json.py

@@ -0,0 +1,15 @@
+import json
+import os
+
+from middlewared.plugins.app_registry.utils import generate_docker_auth_config
+from middlewared.plugins.etc import FileShouldNotExist
+
+
+def render(service, middleware):
+    config = middleware.call_sync('docker.config')
+    if not config['pool']:
+        raise FileShouldNotExist()
+
+    os.makedirs('/etc/docker', exist_ok=True)
+
+    return json.dumps(generate_docker_auth_config(middleware.call_sync('app.registry.query')))

src/middlewared/middlewared/plugins/app_registry/crud.py

@@ -0,0 +1,94 @@
+import middlewared.sqlalchemy as sa
+from middlewared.api import api_method
+from middlewared.api.current import (
+    AppRegistryEntry, AppRegistryCreateArgs, AppRegistryCreateResult, AppRegistryUpdateArgs,
+    AppRegistryUpdateResult, AppRegistryDeleteArgs, AppRegistryDeleteResult,
+)
+from middlewared.service import CRUDService, private, ValidationErrors
+
+from .validate_registry import validate_registry_credentials
+
+
+class AppRegistryModel(sa.Model):
+    __tablename__ = 'app_registry'
+
+    id = sa.Column(sa.Integer(), primary_key=True)
+    name = sa.Column(sa.String(255), nullable=False)
+    description = sa.Column(sa.String(512), nullable=True, default=None)
+    username = sa.Column(sa.EncryptedText(), nullable=False)
+    password = sa.Column(sa.EncryptedText(), nullable=False)
+    uri = sa.Column(sa.String(512), nullable=False, unique=True)
+
+
+class AppRegistryService(CRUDService):
+
+    class Config:
+        namespace = 'app.registry'
+        datastore = 'app.registry'
+        cli_namespace = 'app.registry'
+        entry = AppRegistryEntry
+        role_prefix = 'APPS'
+
+    @private
+    async def validate(self, data, old=None, schema='app_registry_create'):
+        verrors = ValidationErrors()
+
+        filters = [['id', '!=', old['id']]] if old else []
+        if await self.query([['name', '=', data['name']]] + filters):
+            verrors.add(f'{schema}.name', 'Name must be unique')
+
+        if data['uri'].startswith('http') and not data['uri'].endswith('/'):
+            # We can have 2 formats basically
+            # https://index.docker.io/v1/
+            # registry-1.docker.io
+            # We would like to have a trailing slash here because we are not able to pull images without it
+            # if http based url is provided
+            data['uri'] = data['uri'] + '/'
+
+        if await self.query([['uri', '=', data['uri']]] + filters):
+            verrors.add(f'{schema}.uri', 'URI must be unique')
+
+        if not verrors and await self.middleware.run_in_thread(
+            validate_registry_credentials, data['uri'], data['username'], data['password']
+        ) is False:
+            verrors.add(f'{schema}.uri', 'Invalid credentials for registry')
+
+        verrors.check()
+
+    @api_method(AppRegistryCreateArgs, AppRegistryCreateResult, roles=['APPS_WRITE'])
+    async def do_create(self, data):
+        """
+        Create an app registry entry.
+        """
+        await self.middleware.call('docker.state.validate')
+        await self.validate(data)
+        id_ = await self.middleware.call('datastore.insert', 'app.registry', data)
+        await self.middleware.call('etc.generate', 'app_registry')
+        return await self.get_instance(id_)
+
+    @api_method(AppRegistryUpdateArgs, AppRegistryUpdateResult, roles=['APPS_WRITE'])
+    async def do_update(self, id_, data):
+        """
+        Update an app registry entry.
+        """
+        await self.middleware.call('docker.state.validate')
+        old = await self.get_instance(id_)
+        new = old.copy()
+        new.update(data)
+
+        await self.validate(new, old=old, schema='app_registry_update')
+
+        await self.middleware.call('datastore.update', 'app.registry', id_, new)
+
+        await self.middleware.call('etc.generate', 'app_registry')
+        return await self.get_instance(id_)
+
+    @api_method(AppRegistryDeleteArgs, AppRegistryDeleteResult, roles=['APPS_WRITE'])
+    async def do_delete(self, id_):
+        """
+        Delete an app registry entry.
+        """
+        await self.middleware.call('docker.state.validate')
+        await self.get_instance(id_)
+        await self.middleware.call('datastore.delete', 'app.registry', id_)
+        await self.middleware.call('etc.generate', 'app_registry')

src/middlewared/middlewared/plugins/app_registry/utils.py

@@ -0,0 +1,14 @@
+import base64
+
+
+def generate_docker_auth_config(auth_list: list[dict[str, str]]) -> dict:
+    auths = {}
+    for auth in auth_list:
+        auths[auth['uri']] = {
+            # Encode username:password in base64
+            'auth': base64.b64encode(f'{auth["username"]}:{auth["password"]}'.encode()).decode(),
+        }
+
+    return {
+        'auths': auths,
+    }

src/middlewared/middlewared/plugins/app_registry/validate_registry.py

@@ -0,0 +1,26 @@
+from docker.errors import APIError, DockerException
+
+from middlewared.plugins.apps.ix_apps.docker.utils import get_docker_client
+
+
+def validate_registry_credentials(registry: str, username: str, password: str) -> bool:
+    """
+    Validates Docker registry credentials using the Docker SDK.
+
+    Args:
+        registry (str): The URL of the Docker registry (e.g., "registry1.example.com").
+        username (str): The username for the registry.
+        password (str): The password for the registry.
+
+    Returns:
+        bool: True if the credentials are valid, False otherwise.
+    """
+    with get_docker_client() as client:
+        try:
+            client.login(username=username, password=password, registry=registry)
+        except (APIError, DockerException):
+            return False
+        else:
+            return True
+
+    return False

src/middlewared/middlewared/plugins/apps/compose_utils.py

@@ -51,7 +51,7 @@ def compose_action(
         raise CallError(f'Invalid action {action!r} for app {app_name!r}')
 
     # TODO: We will likely have a configurable timeout on this end
-    cp = run(['docker', 'compose'] + compose_files + args, timeout=1200)
+    cp = run(['docker', '--config', '/etc/docker', 'compose'] + compose_files + args, timeout=1200)
     if cp.returncode != 0:
         logger.error('Failed %r action for %r app: %s', action, app_name, cp.stderr)
         err_msg = f'Failed {action!r} action for {app_name!r} app.'

src/middlewared/middlewared/plugins/apps/ix_apps/docker/images.py

@@ -18,7 +18,8 @@ def list_images() -> list[dict]:
 
 
 def pull_image(
-    image_tag: str, callback: typing.Callable = None, username: str | None = None, password: str | None = None
+    image_tag: str, callback: typing.Callable = None, username: str | None = None, password: str | None = None,
+    registry_uri: str | None = None,
 ):
     if username and not password:
         raise CallError('Password is required when username is provided')
@@ -29,6 +30,7 @@ def pull_image(
     auth_config = {
         'username': username,
         'password': password,
+        'registry': registry_uri,
     } if username else None
 
     with get_docker_client() as client:

src/middlewared/middlewared/plugins/apps_images/images.py

@@ -6,7 +6,7 @@
 from middlewared.service import CRUDService, filterable, job
 from middlewared.utils import filter_list
 
-from .utils import parse_tags
+from .utils import get_normalized_auth_config, parse_tags
 
 
 class AppImageService(CRUDService):
@@ -77,9 +77,20 @@ def callback(entry):
             job.set_progress((progress['current']/progress['total']) * 90, 'Pulling image')
 
         self.middleware.call_sync('docker.state.validate')
-        auth_config = data['auth_config'] or {}
         image_tag = data['image']
-        pull_image(image_tag, callback, auth_config.get('username'), auth_config.get('password'))
+        auth_config = data['auth_config'] or {}
+        if not auth_config:
+            # If user has not provided any auth creds, we will try to see if the registry to which the image
+            # belongs to, we already have it's creds and if yes we will try to use that when pulling the image
+            app_registries = {
+                registry['uri']: registry for registry in self.middleware.call_sync('app.registry.query')
+            }
+            auth_config = get_normalized_auth_config(app_registries, image_tag)
+
+        pull_image(
+            image_tag, callback, auth_config.get('username'), auth_config.get('password'),
+            auth_config.get('registry_uri'),
+        )
         job.set_progress(100, f'{image_tag!r} image pulled successfully')
 
     @api_method(AppImageDeleteArgs, AppImageDeleteResult)

src/middlewared/middlewared/plugins/apps_images/utils.py

@@ -153,3 +153,18 @@ def normalize_docker_limits_header(headers: dict) -> dict:
         'remaining_time_limit_in_secs': int(remaining_time_limit),
         'error': None,
     }
+
+
+def get_normalized_auth_config(registry_info: dict[str, dict], image_tag: str) -> dict:
+    if not registry_info:
+        return {}
+
+    user_wants_registry = normalize_reference(image_tag)['registry']
+    if user_wants_registry not in registry_info:
+        return {}
+
+    return {
+        'registry_uri': user_wants_registry,
+        'username': registry_info[user_wants_registry]['username'],
+        'password': registry_info[user_wants_registry]['password'],
+    }

src/middlewared/middlewared/plugins/etc.py

@@ -75,6 +75,9 @@ class EtcService(Service):
             ]
 
         },
+        'app_registry': [
+            {'type': 'py', 'path': 'docker/config.json'},
+        ],
         'docker': [
             {'type': 'py', 'path': 'docker/daemon.json'},
         ],

src/middlewared/middlewared/plugins/service_/services/docker.py

@@ -7,7 +7,7 @@
 
 class DockerService(SimpleService):
     name = 'docker'
-    etc = ['docker']
+    etc = ['app_registry', 'docker']
     systemd_unit = 'docker'
 
     async def before_start(self):