From adfe970a1df9669854958d8ff80d28bfff472c5d Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Fri, 18 Jun 2021 10:23:46 +0800 Subject: [PATCH] ceph-iscsi: add erasure pool support The erasure coded pool does not support omap, here we will just store the data in it and will store the metadata in a replicated pool. This will add a "datapool=" parameter to specify the erasure coded pool to store the data when creating a disk, and the "pool=", which is "rbd" as default, will be used to store the metadata only. For rest API when creating the disk and disksnap, you must replace the "" to "+" in the URL, more detail please see the README. The backstore object name is . in the replicated pool, which is the same with the none-erasure image. Signed-off-by: Xiubo Li --- README | 46 ++++++++++--- ceph_iscsi_config/client.py | 2 +- ceph_iscsi_config/group.py | 2 +- ceph_iscsi_config/lun.py | 96 +++++++++++++++++++++------ ceph_iscsi_config/utils.py | 13 ++++ gwcli/client.py | 10 +-- gwcli/storage.py | 99 ++++++++++++++++++---------- rbd-target-api.py | 127 ++++++++++++++++++++++-------------- 8 files changed, 274 insertions(+), 121 deletions(-) diff --git a/README b/README index fe32bd24..6e371d32 100644 --- a/README +++ b/README @@ -17,17 +17,17 @@ o- / ........................................................................... o- cluster .................................................................. [Clusters: 1] | o- ceph ..................................................................... [HEALTH_OK] | o- pools ................................................................... [Pools: 3] - | | o- ec ........................................ [(2+1), Commit: 0b/40G (0%), Used: 0b] + | | o- ec ....................................... [(2+2), Commit: 0b/40G (0%), Used: 24K] | | o- iscsi ..................................... [(x3), Commit: 0b/20G (0%), Used: 18b] | | o- rbd ....................................... [(x3), Commit: 8G/20G (40%), Used: 5K] | o- topology ......................................................... [OSDs: 3,MONs: 3] o- disks ................................................................... [8G, Disks: 5] | o- rbd ....................................................................... [rbd (8G)] - | o- disk_1 ............................................................... [disk_1 (1G)] - | o- disk_2 ............................................................... [disk_2 (2G)] - | o- disk_3 ............................................................... [disk_3 (2G)] - | o- disk_4 ............................................................... [disk_4 (1G)] - | o- disk_5 ............................................................... [disk_5 (2G)] + | o- disk_1 ........................................................... [rbd/disk_1 (1G)] + | o- disk_2 ........................................................... [rbd/disk_2 (2G)] + | o- disk_3 ........................................................... [rbd/disk_3 (2G)] + | o- disk_4 ........................................................ [rbd+ec/disk_4 (1G)] + | o- disk_5 ........................................................ [rbd+ec/disk_5 (2G)] o- iscsi-targets ............................................................. [Targets: 1] o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw1 ................... [Auth: CHAP, Gateways: 2] | o- disks ................................................................... [Disks: 1] @@ -38,7 +38,7 @@ o- / ........................................................................... o- host-groups ........................................................... [Groups : 0] o- hosts ................................................ [Auth: ACL_ENABLED, Hosts: 1] | o- iqn.1994-05.com.redhat:rh7-client .......... [LOGGED-IN, Auth: CHAP, Disks: 1(2G)] - | o- lun 0 ......................................... [rbd.disk_1(2G), Owner: rh7-gw2] + | o- lun 0 ......................................... [rbd/disk_1(2G), Owner: rh7-gw2] o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw2 ................... [Auth: None, Gateways: 2] o- disks ................................................................... [Disks: 1] | o- rbd/disk_2 .............................................. [Owner: rh7-gw1, Lun: 0] @@ -48,7 +48,17 @@ o- / ........................................................................... o- host-groups ........................................................... [Groups : 0] o- hosts ................................................ [Auth: ACL_ENABLED, Hosts: 1] o- iqn.1994-05.com.redhat:rh7-client .......... [LOGGED-IN, Auth: None, Disks: 1(2G)] - o- lun 0 ......................................... [rbd.disk_2(2G), Owner: rh7-gw1] + o- lun 0 ......................................... [rbd/disk_2(2G), Owner: rh7-gw1] + o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw3 ................... [Auth: None, Gateways: 2] + o- disks ................................................................... [Disks: 1] + | o- rbd+ec/disk_4 ........................................... [Owner: rh7-gw2, Lun: 0] + o- gateways ..................................................... [Up: 2/2, Portals: 2] + | o- rh7-gw1 ................................................... [2006:ac81::1103 (UP)] + | o- rh7-gw2 ................................................... [2006:ac81::1104 (UP)] + o- host-groups ........................................................... [Groups : 0] + o- hosts ................................................ [Auth: ACL_ENABLED, Hosts: 1] + o- iqn.1994-05.com.redhat:rh7-client .......... [LOGGED-IN, Auth: None, Disks: 1(1G)] + o- lun 0 ...................................... [rbd+ec/disk_4(1G), Owner: rh7-gw1] @@ -95,6 +105,26 @@ curl --user admin:admin -d ip_address=2006:ac81::1104 \ NOTE: please make sure both the IPv4 and IPv6 addresses are in the trusted ip list in iscsi-gateway.cfg. +Erasure Pool Support: +For the erasure pool, you need to specify the "datapool=" parameter to store the +data when creating a disk, and the "pool=" will contiue to be a replicated pool, which will +store the metadata only. + +When creating a disk and disk snapshot, for rest api there has a litte different for erasure pool. +Which is you need to use "+" format instead of "" in URL:"http://...//...": + +curl --user admin:admin -d mode=create -d size=1g -d pool=rbd -d datapool=ec -d count=5 + -X PUT http://192.168.122.69:5000/api/disk/rbd+ec/new0_ +curl --user admin:admin -d mode=create -d size=1g -d pool=rbd -d datapool=ec -d create_image=false + -X PUT http://192.168.122.69:5000/api/disk/rbd+ec/new1 +curl --user admin:admin -X GET http://192.168.122.69:5000/api/disk/rbd+ec/new2 +curl --user admin:admin -X DELETE http://192.168.122.69:5000/api/disk/rbd+ec/new3 + +curl --user admin:admin -d mode=create + -X PUT http://192.168.122.69:5000/api/disksnap/rbd+ec/image/new1 +curl --user admin:admin + -X DELETE http://192.168.122.69:5000/api/disksnap/rbd+ec/image/new1 + ## Installation ### Via RPM diff --git a/ceph_iscsi_config/client.py b/ceph_iscsi_config/client.py index 7aca1fc1..e763948b 100644 --- a/ceph_iscsi_config/client.py +++ b/ceph_iscsi_config/client.py @@ -98,7 +98,7 @@ def __init__(self, logger, client_iqn, image_list, username, password, mutual_us self.error = True self.error_msg = err - # image_list is normally a list of strings (pool/image_name) but + # image_list is normally a list of strings (pool[+datapool]/image_name) but # group processing forces a specific lun id allocation to masked disks # in this scenario the image list is a tuple if image_list: diff --git a/ceph_iscsi_config/group.py b/ceph_iscsi_config/group.py index 932c0149..59cd0cf4 100644 --- a/ceph_iscsi_config/group.py +++ b/ceph_iscsi_config/group.py @@ -19,7 +19,7 @@ def __init__(self, logger, target_iqn, group_name, members=[], disks=[]): :param target_iqn: (str) target iqn :param group_name: (str) group name :param members: (list) iscsi IQN's of the clients - :param disks: (list) disk names of the format pool/image + :param disks: (list) disk names of the format pool[+datapool]/image """ self.logger = logger diff --git a/ceph_iscsi_config/lun.py b/ceph_iscsi_config/lun.py index c169c3d7..6ce74e1e 100644 --- a/ceph_iscsi_config/lun.py +++ b/ceph_iscsi_config/lun.py @@ -1,3 +1,4 @@ +import json import rados import rbd import re @@ -13,8 +14,9 @@ from ceph_iscsi_config.backstore import USER_RBD from ceph_iscsi_config.utils import (convert_2_bytes, gen_control_string, valid_size, get_pool_id, ip_addresses, - get_pools, get_rbd_size, this_host, - human_size, CephiSCSIError) + get_pools, get_rbd_size, run_shell_cmd, + human_size, CephiSCSIError, this_host, + parse_disk_meta) from ceph_iscsi_config.gateway_object import GWObject from ceph_iscsi_config.target import GWTarget from ceph_iscsi_config.client import GWClient, CHAP @@ -46,13 +48,14 @@ class RBDDev(object): ] } - def __init__(self, image, size, backstore, pool=None): + def __init__(self, image, size, backstore, pool=None, datapool=None): self.image = image self.size_bytes = convert_2_bytes(size) self.backstore = backstore if pool is None: pool = settings.config.pool self.pool = pool + self.datapool = datapool self.pool_id = get_pool_id(pool_name=self.pool) self.error = False self.error_msg = '' @@ -74,14 +77,14 @@ def create(self): self.image, self.size_bytes, features=RBDDev.default_features(self.backstore), - old_format=False) + old_format=False, + data_pool=self.datapool) except (rbd.ImageExists, rbd.InvalidArgument) as err: self.error = True - self.error_msg = ("Failed to create rbd image {} in " - "pool {} : {}".format(self.image, - self.pool, - err)) + self.error_msg = ("Failed to create rbd image {} in pool {}, " + "datapool {} : {}".format(self.image, self.pool, + self.datapool, err)) def delete(self): """ @@ -289,14 +292,18 @@ class LUN(GWObject): USER_RBD: TCMU_SETTINGS } - def __init__(self, logger, pool, image, size, allocating_host, + def __init__(self, logger, pool, datapool, image, size, allocating_host, backstore, backstore_object_name): self.logger = logger self.image = image self.pool = pool + self.datapool = datapool self.pool_id = 0 self.size_bytes = convert_2_bytes(size) - self.config_key = '{}/{}'.format(self.pool, self.image) + if datapool: + self.config_key = '{}+{}/{}'.format(pool, datapool, image) + else: + self.config_key = '{}/{}'.format(pool, image) self.allocating_host = allocating_host self.backstore = backstore @@ -351,7 +358,7 @@ def remove_lun(self, preserve_image): if self.error: return - rbd_image = RBDDev(self.image, '0G', self.backstore, self.pool) + rbd_image = RBDDev(self.image, '0G', self.backstore, self.pool, self.datapool) if local_gw == self.allocating_host: # by using the allocating host we ensure the delete is not @@ -574,6 +581,38 @@ def activate(self): if client_err: raise CephiSCSIError(client_err) + def _erasure_pool_check(self): + if not self.datapool: + return None + + data, err = run_shell_cmd( + "ceph -n {client_name} --conf {cephconf} osd metadata --format=json". + format(client_name=settings.config.cluster_client_name, + cephconf=settings.config.cephconf)) + if err: + self.logger.error("Cannot get the objectstore type") + return err + bluestore = False + for _osd in json.loads(data): + store_type = _osd['osd_objectstore'] + self.logger.debug(f"objectstore type is ({store_type})") + if store_type == 'bluestore': + bluestore = True + break + + if not bluestore: + return None + + data, err = run_shell_cmd( + "ceph -n {client_name} --conf {cephconf} osd pool get {pool} allow_ec_overwrites". + format(client_name=settings.config.cluster_client_name, + cephconf=settings.config.cephconf, pool=self.datapool)) + if err: + self.logger.error(f"Cannot get allow_ec_overwrites from pool ({self.pool})") + return err + self.logger.debug(f"erasure pool ({self.pool}) allow_ec_overwrites is enabled") + return None + def allocate(self, keep_dev_in_lio=True, in_wwn=None): """ Create image and add to LIO and config. @@ -583,6 +622,10 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None): :return: LIO storage object if successful and keep_dev_in_lio=True else None. """ + err = self._erasure_pool_check() + if err: + return None + self.logger.debug("LUN.allocate starting, listing rbd devices") disk_list = RBDDev.rbd_list(pool=self.pool) self.logger.debug("rados pool '{}' contains the following - " @@ -593,7 +636,8 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None): "allocations is {}".format(local_gw, self.allocating_host)) - rbd_image = RBDDev(self.image, self.size_bytes, self.backstore, self.pool) + rbd_image = RBDDev(self.image, self.size_bytes, self.backstore, self.pool, + self.datapool) self.pool_id = rbd_image.pool_id # if the image required isn't defined, create it! @@ -703,6 +747,7 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None): disk_attr = {"wwn": wwn, "image": self.image, "pool": self.pool, + "datapool": self.datapool, "allocating_host": self.allocating_host, "pool_id": rbd_image.pool_id, "controls": self.controls, @@ -963,7 +1008,10 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs): :param ceph_iscsi_config: Config object :param logger: logger object - :param image_id: (str) . format + :param pool: (str) pool name + :param datapool: (str) datapool name + :param image: (str) image name + :param size: (str) size :return: (str) either 'ok' or an error description """ @@ -993,12 +1041,18 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs): config = ceph_iscsi_config.config - disk_key = "{}/{}".format(kwargs['pool'], kwargs['image']) + datapool = kwargs.get('datapool', None) + if datapool: + disk_key = "{}+{}/{}".format(kwargs['pool'], datapool, kwargs['image']) + else: + disk_key = "{}/{}".format(kwargs['pool'], kwargs['image']) if mode in ['create', 'resize']: if kwargs['pool'] not in get_pools(): return "pool name is invalid" + if datapool and datapool not in get_pools(): + return "datapool name is invalid" if mode == 'create': if kwargs['size'] and not valid_size(kwargs['size']): @@ -1010,6 +1064,8 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs): disk_regex = re.compile(r"^[a-zA-Z0-9\-_\.]+$") if not disk_regex.search(kwargs['pool']): return "Invalid pool name (use alphanumeric, '_', '.', or '-' characters)" + if datapool and not disk_regex.search(datapool): + return "Invalid datapool name (use alphanumeric, '_', '.', or '-' characters)" if not disk_regex.search(kwargs['image']): return "Invalid image name (use alphanumeric, '_', '.', or '-' characters)" @@ -1040,9 +1096,7 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs): if mode in ["resize", "delete", "reconfigure"]: # disk must exist in the config if disk_key not in config['disks']: - return ("rbd {}/{} is not defined to the " - "configuration".format(kwargs['pool'], - kwargs['image'])) + return ("rbd {} is not defined to the configuration".format(disk_key)) if mode == 'resize': @@ -1227,17 +1281,19 @@ def define_luns(logger, config, target): with cluster.open_ioctx(pool) as ioctx: pool_disks = [disk_key for disk_key in srtd_disks - if disk_key.startswith(pool + '/')] + if disk_key.startswith(pool + '/') + or disk_key.startswith(pool + '+')] for disk_key in pool_disks: - pool, image_name = disk_key.split('/') + pool, datapool, image_name = parse_disk_meta(disk_key) + with rbd.Image(ioctx, image_name) as rbd_image: disk_config = config.config['disks'][disk_key] backstore = disk_config['backstore'] backstore_object_name = disk_config['backstore_object_name'] - lun = LUN(logger, pool, image_name, + lun = LUN(logger, pool, datapool, image_name, rbd_image.size(), local_gw, backstore, backstore_object_name) diff --git a/ceph_iscsi_config/utils.py b/ceph_iscsi_config/utils.py index 4ce47d74..c7f9bff8 100644 --- a/ceph_iscsi_config/utils.py +++ b/ceph_iscsi_config/utils.py @@ -28,6 +28,19 @@ class CephiSCSIInval(CephiSCSIError): pass +def parse_disk_meta(disk): + pool = None + datapool = None + image = None + pools, image = disk.split('/') + try: + pool, datapool = pools.split('+') + except ValueError: + pool = pools + pass + return pool, datapool, image + + def run_shell_cmd(cmd, stderr=None, shell=True): if not stderr: stderr = subprocess.STDOUT diff --git a/gwcli/client.py b/gwcli/client.py index 040bbe39..7a3e7472 100644 --- a/gwcli/client.py +++ b/gwcli/client.py @@ -4,7 +4,7 @@ from ceph_iscsi_config.client import CHAP, GWClient import ceph_iscsi_config.settings as settings -from ceph_iscsi_config.utils import human_size, this_host +from ceph_iscsi_config.utils import human_size, this_host, parse_disk_meta from rtslib_fb.utils import normalize_wwn, RTSLibError @@ -479,8 +479,8 @@ def ui_command_disk(self, action='add', disk=None, size=None): in the configuration, the cli will attempt to create it for you. e.g. - disk add - disk remove + disk add ]/image_name> + disk remove ]/image_name> Adding a disk will result in the disk occupying the client's next available lun id. Once allocated removing a LUN will not change the @@ -540,12 +540,12 @@ def ui_command_disk(self, action='add', disk=None, size=None): # a disk given here would be of the form pool.image try: - pool, image = disk.split('/') + pool, datapool, image = parse_disk_meta(disk) except ValueError: self.logger.error("Invalid format. Use pool_name/disk_name") return - rc = ui_disks.create_disk(pool=pool, image=image, size=size) + rc = ui_disks.create_disk(pool=pool, datapool=datapool, image=image, size=size) if rc == 0: self.logger.debug("disk auto-define successful") else: diff --git a/gwcli/storage.py b/gwcli/storage.py index 14888567..5f2d121c 100644 --- a/gwcli/storage.py +++ b/gwcli/storage.py @@ -16,7 +16,8 @@ APIRequest, valid_snapshot_name, get_config, refresh_control_values) -from ceph_iscsi_config.utils import valid_size, convert_2_bytes, human_size, this_host +from ceph_iscsi_config.utils import (valid_size, convert_2_bytes, human_size, + this_host, parse_disk_meta) from ceph_iscsi_config.lun import LUN import ceph_iscsi_config.settings as settings @@ -53,6 +54,14 @@ def __init__(self, parent): self.scan_queue = None self.scan_mutex = None + def _get_pool_type(self, pool): + root = self.get_ui_root() + pools = root.ceph.cluster.pools + pool_object = pools.pool_lookup.get(pool, None) + if pool_object: + return pool_object.type + return None + def _get_disk_meta(self, cluster_ioctx, disk_meta): """ Use the provided cluster context to take an rbd image name from the @@ -72,7 +81,7 @@ def _get_disk_meta(self, cluster_ioctx, disk_meta): except Queue.Empty: break else: - pool, image = rbd_name.split('/') + pool, datapool, image = parse_disk_meta(rbd_name) disk_meta[rbd_name] = {} with cluster_ioctx.open_ioctx(pool) as ioctx: try: @@ -142,7 +151,7 @@ def refresh(self, disk_info): def _group_disks_by_pool(self, disks_config): result = {} for disk_id, disk_config in disks_config.items(): - pool, image = disk_id.split('/') + pool, datapool, image = parse_disk_meta(disk_id) if pool not in result: result[pool] = [] result[pool].append(disk_config) @@ -176,7 +185,7 @@ def ui_command_attach(self, pool=None, image=None, backstore=None, wwn=None): # shorthand version of the command self.logger.debug("user provided pool/image format request") - pool, image = pool.split('/') + pool, datapool, image = parse_disk_meta(pool) else: # long format request @@ -187,10 +196,11 @@ def ui_command_attach(self, pool=None, image=None, backstore=None, wwn=None): self.logger.debug("CMD: /disks/ attach pool={} " "image={}".format(pool, image)) - self.create_disk(pool=pool, image=image, create_image=False, backstore=backstore, wwn=wwn) + self.create_disk(pool=pool, datapool=datapool, image=image, create_image=False, + backstore=backstore, wwn=wwn) - def ui_command_create(self, pool=None, image=None, size=None, backstore=None, wwn=None, - count=1): + def ui_command_create(self, pool=None, datapool=None, image=None, size=None, backstore=None, + wwn=None, count=1): """ Create a RBD image and assign to the gateway(s). @@ -205,7 +215,8 @@ def ui_command_create(self, pool=None, image=None, size=None, backstore=None, ww The syntax of each parameter is as follows; pool : Pool and image name may contain a-z, A-Z, 0-9, '_', or '-' - image characters. + datapool: Data pool name for erasure code pool may contain a-z, A-Z, 0-9, '_', or '-' + image : characters. size : integer, suffixed by the allocation unit - either m/M, g/G or t/T representing the MB/GB/TB [1] backstore : lio backstore @@ -238,7 +249,7 @@ def ui_command_create(self, pool=None, image=None, size=None, backstore=None, ww "({} ?)".format(size)) return size = image - pool, image = pool.split('/') + pool, datapool, image = parse_disk_meta(pool) else: # long format request @@ -260,8 +271,8 @@ def ui_command_create(self, pool=None, image=None, size=None, backstore=None, ww self.logger.debug("CMD: /disks/ create pool={} " "image={} size={} " "count={} ".format(pool, image, size, count)) - self.create_disk(pool=pool, image=image, size=size, count=count, backstore=backstore, - wwn=wwn) + self.create_disk(pool=pool, datapool=datapool, image=image, size=size, count=count, + backstore=backstore, wwn=wwn) def _valid_pool(self, pool=None): """ @@ -277,15 +288,15 @@ def _valid_pool(self, pool=None): pools = root.ceph.cluster.pools pool_object = pools.pool_lookup.get(pool, None) if pool_object: - if pool_object.type == 'replicated': - self.logger.debug("pool '{}' is ok to use".format(pool)) + if pool_object.type in ['replicated', 'erasure']: + self.logger.debug(f"pool '{pool}' is ok to use") return True - self.logger.error("Invalid pool ({}). Must already exist and " - "be replicated".format(pool)) + self.logger.error(f"Invalid pool ({pool}), the type is ({pool_object.type})." + " Must already exist and be erasure or replicated") return False - def create_disk(self, pool=None, image=None, size=None, count=1, + def create_disk(self, pool=None, datapool=None, image=None, size=None, count=1, parent=None, create_image=True, backstore=None, wwn=None): rc = 0 @@ -296,18 +307,20 @@ def create_disk(self, pool=None, image=None, size=None, count=1, local_gw = this_host() disk_key = "{}/{}".format(pool, image) + if datapool: + disk_key = "{}+{}/{}".format(pool, datapool, image) if not self._valid_pool(pool): return - self.logger.debug("Creating/mapping disk {}/{}".format(pool, - image)) + self.logger.debug("Creating/mapping disk {}".format(disk_key)) # make call to local api server's disk endpoint - disk_api = '{}://localhost:{}/api/disk/{}'.format(self.http_mode, - settings.config.api_port, - disk_key) - api_vars = {'pool': pool, 'owner': local_gw, + disk_api = ('{}://localhost:{}/api/disk/' + '{}'.format(self.http_mode, + settings.config.api_port, + disk_key)) + api_vars = {'pool': pool, 'datapool': datapool, 'owner': local_gw, 'count': count, 'mode': 'create', 'create_image': 'true' if create_image else 'false', 'backstore': backstore, 'wwn': wwn} @@ -329,16 +342,15 @@ def create_disk(self, pool=None, image=None, size=None, count=1, for n in range(1, (int(count) + 1), 1): if int(count) > 1: - disk_key = "{}/{}{}".format(pool, image, n) - else: - disk_key = "{}/{}".format(pool, image) + disk_key = "{}+{}/{}{}".format(pool, datapool, image, n) + api_vars = {'datapool': datapool} disk_api = ('{}://localhost:{}/api/disk/' '{}'.format(self.http_mode, settings.config.api_port, disk_key)) - api = APIRequest(disk_api) + api = APIRequest(disk_api, data=api_vars) api.get() if api.response.status_code == 200: @@ -513,6 +525,7 @@ def delete_disk(self, image_id, preserve_image): 'preserve_image': 'true' if preserve_image else 'false' } + pool, datapool, image = parse_disk_meta(image_id) disk_api = '{}://{}:{}/api/disk/{}'.format(self.http_mode, local_gw, settings.config.api_port, @@ -524,7 +537,7 @@ def delete_disk(self, image_id, preserve_image): self.logger.debug("- rbd removed from all gateways, and deleted") disk_object = [disk for disk in all_disks if disk.image_id == image_id][0] - pool, _ = image_id.split('/') + pool, datapool, image = parse_disk_meta(image_id) pool_object = [pool_object for pool_object in self.children if pool_object.name == pool][0] pool_object.remove_child(disk_object) @@ -603,9 +616,21 @@ def __init__(self, parent, pool, pool_disks_config, disks_meta=None): self.disks_meta = disks_meta self.refresh() + def _get_pool_type(self, pool): + root = self.get_ui_root() + pools = root.ceph.cluster.pools + pool_object = pools.pool_lookup.get(pool, None) + if pool_object: + return pool_object.type + return None + def refresh(self): for pool_disk_config in self.pool_disks_config: disk_id = '{}/{}'.format(pool_disk_config['pool'], pool_disk_config['image']) + if pool_disk_config.get('datapool', None): + disk_id = '{}+{}/{}'.format(pool_disk_config['pool'], pool_disk_config['datapool'], + pool_disk_config['image']) + size = self.disks_meta[disk_id].get('size', 0) if self.disks_meta else None features = self.disks_meta[disk_id].get('features', 0) if self.disks_meta else None snapshots = self.disks_meta[disk_id].get('snapshots', []) if self.disks_meta else None @@ -641,11 +666,13 @@ def __init__(self, parent, image_id, image_config, size=None, :param image_config: meta data for this image :return: """ - self.pool, self.rbd_image = image_id.split('/', 1) + self.pool, self.datapool, self.rbd_image = parse_disk_meta(image_id) UINode.__init__(self, self.rbd_image, parent) self.image_id = image_id + pool, datapool, image = parse_disk_meta(self.image_id) + self.is_erasure = True if datapool is not None else False self.size = 0 self.size_h = '' self.features = 0 @@ -689,7 +716,8 @@ def __init__(self, parent, image_id, image_config, size=None, def _apply_status(self): disk_api = ('{}://localhost:{}/api/' 'disk/{}'.format(self.http_mode, - settings.config.api_port, self.image_id)) + settings.config.api_port, + self.image_id)) self.logger.debug("disk GET status for {}".format(self.image_id)) api = APIRequest(disk_api) api.get() @@ -733,10 +761,10 @@ def summary(self): status = True disk_api = ('{}://localhost:{}/api/' - 'disk/{}'.format(self.http_mode, settings.config.api_port, + 'disk/{}'.format(self.http_mode, + settings.config.api_port, self.image_id)) - self.logger.debug("disk GET status for {}".format(self.image_id)) api = APIRequest(disk_api) api.get() @@ -954,11 +982,10 @@ def snapshot(self, action, name): self.logger.debug("Issuing snapshot {} request".format(action)) disk_api = ('{}://localhost:{}/api/' - 'disksnap/{}/{}/{}'.format(self.http_mode, - settings.config.api_port, - self.pool, - self.rbd_image, - name)) + 'disksnap/{}/{}'.format(self.http_mode, + settings.config.api_port, + self.image_id, + name)) if action == 'delete': api = APIRequest(disk_api) diff --git a/rbd-target-api.py b/rbd-target-api.py index 49ea5a44..7ae4a633 100644 --- a/rbd-target-api.py +++ b/rbd-target-api.py @@ -35,7 +35,7 @@ from ceph_iscsi_config.common import Config from ceph_iscsi_config.utils import (normalize_ip_literal, resolve_ip_addresses, ip_addresses, read_os_release, encryption_available, - CephiSCSIError, this_host) + CephiSCSIError, this_host, parse_disk_meta) from ceph_iscsi_config.device_status import DeviceStatusWatcher from gwcli.utils import (APIRequest, valid_gateway, valid_client, @@ -749,12 +749,12 @@ def target_disk(target_iqn=None): """ Coordinate the addition(PUT) and removal(DELETE) of a disk for a target :param target_iqn: (str) IQN of the target - :param disk: (str) rbd image name on the format pool/image + :param disk: (str) rbd image name on the format pool[+datapool]/image **RESTRICTED** Examples: - curl --user admin:admin -d disk=rbd/new2_1 + curl --user admin:admin -d disk=rbd[+datapool]/new2_1 -X PUT http://192.168.122.69:5000/api/targetlun/iqn.2003-01.com.redhat.iscsi-gw - curl --user admin:admin -d disk=rbd/new2_1 + curl --user admin:admin -d disk=rbd[+datapool]/new2_1 -X DELETE http://192.168.122.69:5000/api/targetlun/iqn.2003-01.com.redhat.iscsi-gw """ @@ -791,10 +791,10 @@ def target_disk(target_iqn=None): return jsonify(message="Disk {} cannot be used because it is already mapped on " "target {}".format(disk, iqn)), 400 - pool, image_name = disk.split('/') + pool, datapool, image_name = parse_disk_meta(disk) try: backstore = config.config['disks'][disk] - rbd_image = RBDDev(image_name, 0, backstore, pool) + rbd_image = RBDDev(image_name, 0, backstore, pool, datapool) size = rbd_image.current_size logger.debug("{} size is {}".format(disk, size)) except rbd.ImageNotFound: @@ -873,7 +873,7 @@ def _target_disk(target_iqn=None): config.refresh() disk = request.form.get('disk') - pool, image = disk.split('/', 1) + pool, datapool, image = parse_disk_meta(disk) disk_config = config.config['disks'][disk] backstore = disk_config['backstore'] backstore_object_name = disk_config['backstore_object_name'] @@ -897,6 +897,7 @@ def _target_disk(target_iqn=None): size = rbd_image.current_size lun = LUN(logger, pool, + datapool, image, size, allocating_host, @@ -925,6 +926,7 @@ def _target_disk(target_iqn=None): lun = LUN(logger, pool, + datapool, image, 0, purge_host, @@ -970,9 +972,9 @@ def get_disks(): return jsonify(response), 200 -@app.route('/api/disk//', methods=['GET', 'PUT', 'DELETE']) +@app.route('/api/disk//', methods=['GET', 'PUT', 'DELETE']) @requires_restricted_auth -def disk(pool, image): +def disk(pools, image): """ Coordinate the create/delete of rbd images across the gateway nodes This method calls the corresponding disk api entrypoints across each @@ -981,11 +983,10 @@ def disk(pool, image): remote gateways and then the local machine is used to perform the actual rbd delete. - :param pool: (str) pool name + :param pools: (str) pool names, which the format is "[+]" :param image: (str) rbd image name :param mode: (str) 'create' or 'resize' the rbd image :param size: (str) the size of the rbd image - :param pool: (str) the pool name the rbd image will be in :param count: (str) the number of images will be created :param owner: (str) the owner of the rbd image :param controls: (JSON dict) valid control overrides @@ -996,17 +997,22 @@ def disk(pool, image): **RESTRICTED** Examples: curl --user admin:admin -d mode=create -d size=1g -d pool=rbd -d count=5 - -X PUT http://192.168.122.69:5000/api/disk/rbd/new0_ - curl --user admin:admin -d mode=create -d size=10g -d pool=rbd -d create_image=false - -X PUT http://192.168.122.69:5000/api/disk/rbd/new1 - curl --user admin:admin -X GET http://192.168.122.69:5000/api/disk/rbd/new2 - curl --user admin:admin -X DELETE http://192.168.122.69:5000/api/disk/rbd/new3 + -X PUT http://192.168.122.69:5000/api/disk/rbd/rbd/new0_ + curl --user admin:admin -d mode=create -d size=1g -d pool=rbd -d datapool=ec + -d create_image=false -X PUT http://192.168.122.69:5000/api/disk/rbd+ec/new1 + curl --user admin:admin -d mode=create -d size=1g -d pool=rbd -d datapool=ec -d count=5 + -X PUT http://192.168.122.69:5000/api/disk/rbd/rbd+ec/new2 + curl --user admin:admin -X GET http://192.168.122.69:5000/api/disk/rbd/new0_1 + curl --user admin:admin -X GET http://192.168.122.69:5000/api/disk/rbd+ec/new1 + curl --user admin:admin -X DELETE http://192.168.122.69:5000/api/disk/rbd+ec/new2 """ local_gw = this_host() logger.debug("this host is {}".format(local_gw)) - image_id = '{}/{}'.format(pool, image) + image_id = '{}/{}'.format(pools, image) + pool, datapool, image = parse_disk_meta(image_id) + _image_id = '{}/{}'.format(pool, image) config.refresh() @@ -1015,7 +1021,7 @@ def disk(pool, image): if image_id in config.config['disks']: disk_dict = config.config["disks"][image_id] global dev_status_watcher - disk_status = dev_status_watcher.get_dev_status(image_id) + disk_status = dev_status_watcher.get_dev_status(_image_id) if disk_status: disk_dict['status'] = disk_status.get_status_dict() else: @@ -1062,7 +1068,7 @@ def disk(pool, image): logger.debug("{} controls {}".format(mode, controls)) wwn = request.form.get('wwn') - disk_usable = LUN.valid_disk(config, logger, pool=pool, + disk_usable = LUN.valid_disk(config, logger, pool=pool, datapool=datapool, image=image, size=size, mode=mode, count=count, controls=controls, backstore=backstore, wwn=wwn) @@ -1078,7 +1084,7 @@ def disk(pool, image): try: # no size implies not intention to create an image, try to # check whether it exists - rbd_image = RBDDev(image, 0, backstore, pool) + rbd_image = RBDDev(image, 0, backstore, pool, datapool) size = rbd_image.current_size except rbd.ImageNotFound: # the create_image=true will be implied if size is specified @@ -1107,6 +1113,7 @@ def disk(pool, image): sfx) api_vars = {'pool': pool, + 'datapool': datapool, 'image': image, 'size': size, 'owner': local_gw, @@ -1130,13 +1137,14 @@ def disk(pool, image): else: # this is a DELETE request - disk_usable = LUN.valid_disk(config, logger, mode='delete', - pool=pool, image=image, backstore=backstore) + disk_usable = LUN.valid_disk(config, logger, mode='delete', pool=pool, + datapool=datapool, image=image, backstore=backstore) if disk_usable != 'ok': return jsonify(message=disk_usable), 400 api_vars = { + 'datapool': datapool, 'purge_host': local_gw, 'preserve_image': request.form.get('preserve_image'), 'backstore': backstore @@ -1163,11 +1171,16 @@ def _disk(pool, image): Disks can be created and added to each gateway, or deleted through this call :param pool: (str) pool name + :param datapool: (str) datapool name for erasure :param image: (str) image name **RESTRICTED** """ - image_id = '{}/{}'.format(pool, image) + datapool = request.form.get('datapool', None) + if datapool: + image_id = '{}+{}/{}'.format(pool, datapool, image) + else: + image_id = '{}/{}'.format(pool, image) config.refresh() @@ -1214,6 +1227,7 @@ def _disk(pool, image): lun = LUN(logger, str(request.form['pool']), + datapool, image, str(request.form['size']), str(request.form['owner']), @@ -1252,11 +1266,11 @@ def _disk(pool, image): return jsonify(message="LUN {} failure".format(mode)), 500 if 'owner' not in disk: - msg = "Disk {}/{} must be assigned to a target".format(disk['pool'], disk['image']) + msg = "Disk {} must be assigned to a target".format(image_id) logger.error("LUN owner not defined - {}".format(msg)) return jsonify(message="LUN {} failure - {}".format(mode, msg)), 400 - lun = LUN(logger, pool, image, size, disk['owner'], + lun = LUN(logger, pool, datapool, image, size, disk['owner'], backstore, backstore_object_name) if mode == 'deactivate': try: @@ -1283,13 +1297,14 @@ def _disk(pool, image): purge_host = request.form['purge_host'] preserve_image = request.form.get('preserve_image') == 'true' logger.debug("delete request for disk image '{}'".format(image_id)) - pool, image = image_id.split('/', 1) + pool, datapool, image = parse_disk_meta(image_id) disk_config = config.config['disks'][image_id] backstore = disk_config['backstore'] backstore_object_name = disk_config['backstore_object_name'] lun = LUN(logger, pool, + datapool, image, 0, purge_host, @@ -1334,22 +1349,23 @@ def lun_reconfigure(image_id, controls, backstore): gateways.insert(0, 'localhost') + pool_name, datapool_name, image_name = parse_disk_meta(image_id) + # deactivate disk - api_vars = {'mode': 'deactivate'} + api_vars = {'mode': 'deactivate', 'datapool': datapool_name} + _image_id = '{}/{}'.format(pool_name, image_name) logger.debug("deactivating disk") resp_text, resp_code = call_api(gateways, '_disk', - image_id, http_method='put', + _image_id, http_method='put', api_vars=api_vars) if resp_code != 200: return "failed to deactivate disk: {}".format(resp_text), resp_code - pool_name, image_name = image_id.split('/', 1) - - rbd_image = RBDDev(image_name, 0, backstore, pool_name) + rbd_image = RBDDev(image_name, 0, backstore, pool_name, datapool_name) size = rbd_image.current_size - lun = LUN(logger, pool_name, image_name, size, disk['owner'], + lun = LUN(logger, pool_name, datapool_name, image_name, size, disk['owner'], disk['backstore'], disk['backstore_object_name']) for k, v in controls.items(): @@ -1367,11 +1383,11 @@ def lun_reconfigure(image_id, controls, backstore): api_vars['controls'] = json.dumps(controls) # activate disk - api_vars['mode'] = 'activate' + api_vars = {'mode': 'activate', 'datapool': datapool_name} logger.debug("activating disk") activate_resp_text, activate_resp_code = call_api(gateways, '_disk', - image_id, http_method='put', + _image_id, http_method='put', api_vars=api_vars) if resp_code == 200 and activate_resp_code != 200: resp_text = activate_resp_text @@ -1389,31 +1405,33 @@ def lun_reconfigure(image_id, controls, backstore): return resp_text, resp_code -@app.route('/api/disksnap///', methods=['PUT', 'DELETE']) +@app.route('/api/disksnap///', methods=['PUT', 'DELETE']) @requires_restricted_auth -def disksnap(pool, image, name): +def disksnap(pools, image, name): """ Coordinate the management of rbd image snapshots across the gateway nodes. This method calls the corresponding disk api entrypoints across each gateway. Processing is done serially: rollback is done locally first, then other gateways. Other actions are only performed locally. - :param image_id: (str) rbd image name of the format pool/image + :param pools: (str) pool names, which the format is "[+]" + :param image: (str) rbd image name :param name: (str) rbd snapshot name :param mode: (str) 'create' or 'rollback' the rbd snapshot **RESTRICTED** Examples: curl --user admin:admin -d mode=create - -X PUT http://192.168.122.69:5000/api/disksnap/rbd.image/new1 + -X PUT http://192.168.122.69:5000/api/disksnap/rbd+ec/image/new1 curl --user admin:admin - -X DELETE http://192.168.122.69:5000/api/disksnap/rbd.image/new1 + -X DELETE http://192.168.122.69:5000/api/disksnap/rbd+ec/image/new1 """ if not valid_snapshot_name(name): logger.debug("snapshot request rejected due to invalid snapshot name") return jsonify(message="snapshot name is invalid"), 400 - image_id = '{}/{}'.format(pool, image) + image_id = '{}/{}'.format(pools, image) + pool, datapool, image_name = parse_disk_meta(image_id) if image_id not in config.config['disks']: return jsonify(message="rbd image {} not " @@ -1424,7 +1442,7 @@ def disksnap(pool, image, name): if mode == 'create': resp_text, resp_code = _disksnap_create(pool, image, name) elif mode == 'rollback': - resp_text, resp_code = _disksnap_rollback(image_id, pool, + resp_text, resp_code = _disksnap_rollback(pool, datapool, image, name) else: logger.debug("snapshot request rejected due to invalid mode") @@ -1481,9 +1499,17 @@ def _disksnap_delete(pool_name, image_name, name): return resp_text, resp_code -def _disksnap_rollback(image_id, pool_name, image_name, name): +def _disksnap_rollback(pool, datapool, image, name): logger.debug("snapshot rollback request") + if datapool: + image_id = '{}+{}/{}'.format(pool, datapool, image) + else: + image_id = '{}/{}'.format(pool, image) + + # _disk rest api will retrieve the datapool from request parameters + _image_id = '{}/{}'.format(pool, image) + disk = config.config['disks'].get(image_id, None) if not disk: return "rbd image {} not found".format(image_id), 404 @@ -1495,12 +1521,13 @@ def _disksnap_rollback(image_id, pool_name, image_name, name): gateways.append(this_host()) api_vars = { + 'datapool': datapool, 'mode': 'deactivate'} need_active = True logger.debug("deactivating disk") resp_text, resp_code = call_api(gateways, '_disk', - image_id, + _image_id, http_method='put', api_vars=api_vars) if resp_code == 200 or resp_code == 400: @@ -1509,8 +1536,8 @@ def _disksnap_rollback(image_id, pool_name, image_name, name): try: with rados.Rados(conffile=settings.config.cephconf, name=settings.config.cluster_client_name) as cluster, \ - cluster.open_ioctx(pool_name) as ioctx, \ - rbd.Image(ioctx, image_name) as image: + cluster.open_ioctx(pool) as ioctx, \ + rbd.Image(ioctx, image) as image: try: logger.debug("rolling back to snapshot") @@ -1533,7 +1560,7 @@ def _disksnap_rollback(image_id, pool_name, image_name, name): logger.debug("activating disk") api_vars['mode'] = 'activate' activate_resp_text, activate_resp_code = call_api(gateways, '_disk', - image_id, + _image_id, http_method='put', api_vars=api_vars) if resp_code == 200 and activate_resp_code != 200: @@ -2010,14 +2037,14 @@ def clientlun(target_iqn, client_iqn): """ Coordinate the addition(PUT) and removal(DELETE) of a disk for a client :param client_iqn: (str) IQN of the client - :param disk: (str) rbd image name of the format pool/image + :param disk: (str) rbd image name of the format pool[+]/image **RESTRICTED** Examples: TARGET_IQN = iqn.2017-08.org.ceph:iscsi-gw CLIENT_IQN = iqn.1994-05.com.redhat:myhost4 - curl --user admin:admin -d disk=rbd/new2_1 + curl --user admin:admin -d disk=rbd+datapool/new2_1 -X PUT http://192.168.122.69:5000/api/clientlun/$TARGET_IQN/$CLIENT_IQN - curl --user admin:admin -d disk=rbd/new2_1 + curl --user admin:admin -d disk=rbd+datapool/new2_1 -X DELETE http://192.168.122.69:5000/api/clientlun/$TARGET_IQN/$CLIENT_IQN """ @@ -2379,7 +2406,7 @@ def hostgroup(target_iqn, group_name): Examples: curl --user admin:admin -X GET http://192.168.122.69:5000/api/hostgroup/group_name curl --user admin:admin -d members=iqn.1994-05.com.redhat:myhost4 - -d disks=rbd.disk1 -X PUT http://192.168.122.69:5000/api/hostgroup/group_name + -d disks=rbd[+datapool].disk1 -X PUT http://192.168.122.69:5000/api/hostgroup/group_name curl --user admin:admin -d action=remove -d disks=rbd.disk1 -X PUT http://192.168.122.69:5000/api/hostgroup/group_name curl --user admin:admin