NAS-132188 / 25.10 / Allow migrating apps from one pool to another

src/middlewared/middlewared/api/v25_04_0/docker.py

@@ -51,6 +51,7 @@ class DockerUpdateArgs(DockerEntry, metaclass=ForUpdateMetaclass):
     dataset: Excluded = excluded_field()
     address_pools: list[AddressPool]
     cidr_v6: IPvAnyInterface
+    migrate_applications: bool
 
     @field_validator('cidr_v6')
     @classmethod
@@ -61,6 +62,12 @@ def validate_ipv6(cls, v):
             raise ValueError('Prefix length of cidr_v6 network cannot be 128.')
         return v
 
+    @model_validator(mode='after')
+    def validate_attrs(self):
+        if self.migrate_applications is True and not self.pool:
+            raise ValueError('Pool is required when migrating applications.')
+        return self
+
 
 class DockerUpdateResult(BaseModel):
     result: DockerEntry

src/middlewared/middlewared/plugins/docker/migrate.py

@@ -0,0 +1,88 @@
+import os
+from datetime import datetime
+
+from middlewared.service import CallError, private, Service
+
+from .state_utils import DatasetDefaults, Status
+from .utils import applications_ds_name, MIGRATION_NAMING_SCHEMA
+
+
+class DockerService(Service):
+
+    @private
+    async def migrate_ix_apps_dataset(self, job, config, old_config, migration_options):
+        new_pool = config['pool']
+        backup_name = f'backup_to_{new_pool}_{datetime.now().strftime("%F_%T")}'
+        await self.middleware.call('docker.state.set_status', Status.MIGRATING.value)
+        job.set_progress(30, 'Creating docker backup')
+        backup_job = await self.middleware.call('docker.backup', backup_name)
+        await backup_job.wait()
+        if backup_job.error:
+            raise CallError(f'Failed to backup docker apps: {backup_job.error}')
+
+        job.set_progress(35, 'Stopping docker service')
+        await self.middleware.call('service.stop', 'docker')
+
+        try:
+            job.set_progress(40, f'Replicating datasets from {old_config["pool"]!r} to {new_pool!r} pool')
+            await self.middleware.call('zfs.dataset.create', {
+                'name': applications_ds_name(config['pool']), 'type': 'FILESYSTEM',
+                'properties': DatasetDefaults.create_time_props(
+                    os.path.basename(applications_ds_name(config['pool']))
+                ),
+            })
+            await (await self.middleware.call('docker.fs_manage.umount')).wait()
+
+            await self.replicate_apps_dataset(new_pool, old_config['pool'], migration_options)
+
+            await self.middleware.call('datastore.update', 'services.docker', old_config['id'], config)
+
+            job.set_progress(70, f'Restoring docker apps in {new_pool!r} pool')
+            restore_job = await self.middleware.call('docker.restore_backup', backup_name)
+            await restore_job.wait()
+            if restore_job.error:
+                raise CallError(f'Failed to restore docker apps on the new pool: {restore_job.error}')
+        except Exception:
+            await self.middleware.call('docker.state.set_status', Status.MIGRATION_FAILED.value)
+            raise
+        else:
+            job.set_progress(100, 'Migration completed successfully')
+        finally:
+            await self.middleware.call('docker.delete_backup', backup_name)
+
+    @private
+    async def replicate_apps_dataset(self, new_pool, old_pool, migration_options):
+        snap_details = await self.middleware.call(
+            'zfs.snapshot.create', {
+                'dataset': applications_ds_name(old_pool),
+                'naming_schema': MIGRATION_NAMING_SCHEMA,
+                'recursive': True,
+            }
+        )
+
+        try:
+            old_ds = applications_ds_name(old_pool)
+            new_ds = applications_ds_name(new_pool)
+            migrate_job = await self.middleware.call(
+                'replication.run_onetime', {
+                    'direction': 'PUSH',
+                    'transport': 'LOCAL',
+                    'source_datasets': [old_ds],
+                    'target_dataset': new_ds,
+                    'recursive': True,
+                    'also_include_naming_schema': [MIGRATION_NAMING_SCHEMA],
+                    'retention_policy': 'SOURCE',
+                    'replicate': True,
+                    'readonly': 'IGNORE',
+                    'exclude_mountpoint_property': False,
+                }
+            )
+            await migrate_job.wait()
+            if migrate_job.error:
+                raise CallError(f'Failed to migrate {old_ds} to {new_ds}: {migrate_job.error}')
+
+        finally:
+            await self.middleware.call('zfs.snapshot.delete', snap_details['id'], {'recursive': True})
+            snap_name = f'{applications_ds_name(new_pool)}@{snap_details["snapshot_name"]}'
+            if await self.middleware.call('zfs.snapshot.query', [['id', '=', snap_name]]):
+                await self.middleware.call('zfs.snapshot.delete', snap_name, {'recursive': True})

src/middlewared/middlewared/plugins/docker/state_utils.py

@@ -59,6 +59,8 @@ class Status(enum.Enum):
     STOPPED = 'STOPPED'
     UNCONFIGURED = 'UNCONFIGURED'
     FAILED = 'FAILED'
+    MIGRATING = 'MIGRATING'
+    MIGRATION_FAILED = 'MIGRATION FAILED'
 
 
 STATUS_DESCRIPTIONS = {
@@ -69,6 +71,8 @@ class Status(enum.Enum):
     Status.STOPPED: 'Application(s) have been stopped',
     Status.UNCONFIGURED: 'Application(s) are not configured',
     Status.FAILED: 'Application(s) have failed to start',
+    Status.MIGRATING: 'Application(s) are being migrated',
+    Status.MIGRATION_FAILED: 'Application(s) failed to migrate to new pool',
 }
 
 

src/middlewared/middlewared/plugins/docker/update.py

@@ -47,6 +47,86 @@ async def config_extend(self, data):
         data['dataset'] = applications_ds_name(data['pool']) if data.get('pool') else None
         return data
 
+    @private
+    async def validate_data(self, old_config, config, schema='docker_update'):
+        verrors = ValidationErrors()
+        if config['pool'] and not await self.middleware.run_in_thread(query_imported_fast_impl, [config['pool']]):
+            verrors.add(f'{schema}.pool', 'Pool not found.')
+
+        if config['address_pools'] != old_config['address_pools']:
+            validate_address_pools(
+                await self.middleware.call('interface.ip_in_use', {'static': True}), config['address_pools']
+            )
+
+        if config.pop('migrate_applications', False):
+            if config['pool'] == old_config['pool']:
+                verrors.add(
+                    f'{schema}.migrate_applications',
+                    'Migration of applications dataset only happens when a new pool is configured.'
+                )
+            elif not old_config['pool']:
+                verrors.add(
+                    f'{schema}.migrate_applications',
+                    'A pool must have been configured previously for ix-apps dataset migration.'
+                )
+            else:
+                if await self.middleware.call(
+                    'zfs.dataset.query', [['id', '=', applications_ds_name(config['pool'])]], {
+                        'extra': {'retrieve_children': False, 'retrieve_properties': False}
+                    }
+                ):
+                    verrors.add(
+                        f'{schema}.migrate_applications',
+                        f'Migration of {applications_ds_name(old_config["pool"])!r} to {config["pool"]!r} not '
+                        f'possible as {applications_ds_name(config["pool"])} already exists.'
+                    )
+
+                ix_apps_ds = await self.middleware.call(
+                    'zfs.dataset.query', [['id', '=', applications_ds_name(old_config['pool'])]], {
+                        'extra': {'retrieve_children': False, 'retrieve_properties': True}
+                    }
+                )
+                if not ix_apps_ds:
+                    # Edge case but handled just to be sure
+                    verrors.add(
+                        f'{schema}.migrate_applications',
+                        f'{applications_ds_name(old_config["pool"])!r} does not exist, migration not possible.'
+                    )
+                elif ix_apps_ds[0]['encrypted']:
+                    # This should never happen but better be safe with extra validation
+                    verrors.add(
+                        f'{schema}.migrate_applications',
+                        f'{ix_apps_ds[0]["id"]!r} is encrypted which is not a supported configuration'
+                    )
+
+                # Now let's add some validation for destination
+                destination_root_ds = await self.middleware.call(
+                    'pool.dataset.get_instance_quick', config['pool'], {
+                        'encryption': True,
+                    }
+                )
+                if destination_root_ds['key_format']['value'] == 'PASSPHRASE':
+                    verrors.add(
+                        f'{schema}.migrate_applications',
+                        f'{ix_apps_ds[0]["id"]!r} can only be migrated to a destination pool '
+                        'which is "KEY" encrypted.'
+                    )
+                if destination_root_ds['locked']:
+                    verrors.add(
+                        f'{schema}.migrate_applications',
+                        f'Migration not possible as {config["pool"]!r} is locked'
+                    )
+                    if not await self.middleware.call(
+                        'datastore.query', 'storage.encrypteddataset', [['name', '=', config['pool']]]
+                    ):
+                        verrors.add(
+                            f'{schema}.migrate_applications',
+                            f'Migration not possible as system does not has encryption key for {config["pool"]!r} '
+                            'stored'
+                        )
+
+        verrors.check()
+
     @api_method(DockerUpdateArgs, DockerUpdateResult, audit='Docker: Updating Configurations')
     @job(lock='docker_update')
     async def do_update(self, job, data):
@@ -58,17 +138,13 @@ async def do_update(self, job, data):
         config = old_config.copy()
         config.update(data)
         config['cidr_v6'] = str(config['cidr_v6'])
+        migrate_apps = config.get('migrate_applications', False)
 
-        verrors = ValidationErrors()
-        if config['pool'] and not await self.middleware.run_in_thread(query_imported_fast_impl, [config['pool']]):
-            verrors.add('docker_update.pool', 'Pool not found.')
-
-        verrors.check()
+        await self.validate_data(old_config, config)
 
-        if config['address_pools'] != old_config['address_pools']:
-            validate_address_pools(
-                await self.middleware.call('interface.ip_in_use', {'static': True}), config['address_pools']
-            )
+        if migrate_apps:
+            await self.middleware.call('docker.migrate_ix_apps_dataset', job, config, old_config, {})
+            return
 
         if old_config != config:
             address_pools_changed = any(config[k] != old_config[k] for k in ('address_pools', 'cidr_v6'))

src/middlewared/middlewared/plugins/docker/utils.py

@@ -2,6 +2,7 @@
 
 
 BACKUP_NAME_PREFIX = 'ix-apps-backup-'
+MIGRATION_NAMING_SCHEMA = 'ix-apps-migrate-%Y-%m-%d_%H-%M'
 UPDATE_BACKUP_PREFIX = 'system-update-'