ceph-iscsi: add erasure pool support

The erasure coded pool does not support omap, here we will just store the data in it and will store the metadata in a replicated pool. This will add a "datapool=<ec_pool_name>" parameter to specify the erasure coded pool to store the data when creating a disk, and the "pool=<name>", which is "rbd" as default, will be used to store the metadata only. For rest API when creating the disk and disksnap, you must replace the "<pool>" to "<pool>+<ec pool>" in the URL, more detail please see the README. The backstore object name is <pool>.<image> in the replicated pool, which is the same with the none-erasure image. Signed-off-by: Xiubo Li <[email protected]>
ceph · Dec 21, 2021 · adfe970 · adfe970
1 parent b42a375
commit adfe970
Show file tree

Hide file tree

Showing 8 changed files with 274 additions and 121 deletions.
diff --git a/README b/README
@@ -17,17 +17,17 @@ o- / ...........................................................................
   o- cluster .................................................................. [Clusters: 1]
   | o- ceph ..................................................................... [HEALTH_OK]
   |   o- pools ................................................................... [Pools: 3]
-  |   | o- ec ........................................ [(2+1), Commit: 0b/40G (0%), Used: 0b]
+  |   | o- ec ....................................... [(2+2), Commit: 0b/40G (0%), Used: 24K]
   |   | o- iscsi ..................................... [(x3), Commit: 0b/20G (0%), Used: 18b]
   |   | o- rbd ....................................... [(x3), Commit: 8G/20G (40%), Used: 5K]
   |   o- topology ......................................................... [OSDs: 3,MONs: 3]
   o- disks ................................................................... [8G, Disks: 5]
   | o- rbd ....................................................................... [rbd (8G)]
-  |   o- disk_1 ............................................................... [disk_1 (1G)]
-  |   o- disk_2 ............................................................... [disk_2 (2G)]
-  |   o- disk_3 ............................................................... [disk_3 (2G)]
-  |   o- disk_4 ............................................................... [disk_4 (1G)]
-  |   o- disk_5 ............................................................... [disk_5 (2G)]
+  |   o- disk_1 ........................................................... [rbd/disk_1 (1G)]
+  |   o- disk_2 ........................................................... [rbd/disk_2 (2G)]
+  |   o- disk_3 ........................................................... [rbd/disk_3 (2G)]
+  |   o- disk_4 ........................................................ [rbd+ec/disk_4 (1G)]
+  |   o- disk_5 ........................................................ [rbd+ec/disk_5 (2G)]
   o- iscsi-targets ............................................................. [Targets: 1]
     o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw1 ................... [Auth: CHAP, Gateways: 2]
     | o- disks ................................................................... [Disks: 1]
@@ -38,7 +38,7 @@ o- / ...........................................................................
       o- host-groups ........................................................... [Groups : 0]
       o- hosts ................................................ [Auth: ACL_ENABLED, Hosts: 1]
     |   o- iqn.1994-05.com.redhat:rh7-client .......... [LOGGED-IN, Auth: CHAP, Disks: 1(2G)]
-    |     o- lun 0 ......................................... [rbd.disk_1(2G), Owner: rh7-gw2]
+    |     o- lun 0 ......................................... [rbd/disk_1(2G), Owner: rh7-gw2]
     o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw2 ................... [Auth: None, Gateways: 2]
       o- disks ................................................................... [Disks: 1]
       | o- rbd/disk_2 .............................................. [Owner: rh7-gw1, Lun: 0]
@@ -48,7 +48,17 @@ o- / ...........................................................................
       o- host-groups ........................................................... [Groups : 0]
       o- hosts ................................................ [Auth: ACL_ENABLED, Hosts: 1]
         o- iqn.1994-05.com.redhat:rh7-client .......... [LOGGED-IN, Auth: None, Disks: 1(2G)]
-          o- lun 0 ......................................... [rbd.disk_2(2G), Owner: rh7-gw1]
+          o- lun 0 ......................................... [rbd/disk_2(2G), Owner: rh7-gw1]
+    o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw3 ................... [Auth: None, Gateways: 2]
+      o- disks ................................................................... [Disks: 1]
+      | o- rbd+ec/disk_4 ........................................... [Owner: rh7-gw2, Lun: 0]
+      o- gateways ..................................................... [Up: 2/2, Portals: 2]
+      | o- rh7-gw1 ................................................... [2006:ac81::1103 (UP)]
+      | o- rh7-gw2 ................................................... [2006:ac81::1104 (UP)]
+      o- host-groups ........................................................... [Groups : 0]
+      o- hosts ................................................ [Auth: ACL_ENABLED, Hosts: 1]
+        o- iqn.1994-05.com.redhat:rh7-client .......... [LOGGED-IN, Auth: None, Disks: 1(1G)]
+          o- lun 0 ...................................... [rbd+ec/disk_4(1G), Owner: rh7-gw1]
 
 
 
@@ -95,6 +105,26 @@ curl --user admin:admin -d ip_address=2006:ac81::1104 \
 NOTE: please make sure both the IPv4 and IPv6 addresses are in the trusted
 ip list in iscsi-gateway.cfg.
 
+Erasure Pool Support:
+For the erasure pool, you need to specify the "datapool=<erasure pool name>" parameter to store the
+data when creating a disk, and the "pool=<name>" will contiue to be a replicated pool, which will
+store the metadata only.
+
+When creating a disk and disk snapshot, for rest api there has a litte different for erasure pool.
+Which is you need to use "<pool>+<datapool>" format instead of "<pool>" in URL:"http://.../<pool>/...":
+
+curl --user admin:admin -d mode=create -d size=1g -d pool=rbd -d datapool=ec -d count=5
+     -X PUT http://192.168.122.69:5000/api/disk/rbd+ec/new0_
+curl --user admin:admin -d mode=create -d size=1g -d pool=rbd -d datapool=ec -d create_image=false
+     -X PUT http://192.168.122.69:5000/api/disk/rbd+ec/new1
+curl --user admin:admin -X GET http://192.168.122.69:5000/api/disk/rbd+ec/new2
+curl --user admin:admin -X DELETE http://192.168.122.69:5000/api/disk/rbd+ec/new3
+
+curl --user admin:admin -d mode=create
+     -X PUT http://192.168.122.69:5000/api/disksnap/rbd+ec/image/new1
+curl --user admin:admin
+     -X DELETE http://192.168.122.69:5000/api/disksnap/rbd+ec/image/new1
+
 
 ## Installation
 ### Via RPM

diff --git a/ceph_iscsi_config/client.py b/ceph_iscsi_config/client.py
@@ -98,7 +98,7 @@ def __init__(self, logger, client_iqn, image_list, username, password, mutual_us
             self.error = True
             self.error_msg = err
 
-        # image_list is normally a list of strings (pool/image_name) but
+        # image_list is normally a list of strings (pool[+datapool]/image_name) but
         # group processing forces a specific lun id allocation to masked disks
         # in this scenario the image list is a tuple
         if image_list:

diff --git a/ceph_iscsi_config/group.py b/ceph_iscsi_config/group.py
@@ -19,7 +19,7 @@ def __init__(self, logger, target_iqn, group_name, members=[], disks=[]):
         :param target_iqn: (str) target iqn
         :param group_name: (str) group name
         :param members: (list) iscsi IQN's of the clients
-        :param disks: (list) disk names of the format pool/image
+        :param disks: (list) disk names of the format pool[+datapool]/image
         """
 
         self.logger = logger

diff --git a/ceph_iscsi_config/lun.py b/ceph_iscsi_config/lun.py
@@ -1,3 +1,4 @@
+import json
 import rados
 import rbd
 import re
@@ -13,8 +14,9 @@
 from ceph_iscsi_config.backstore import USER_RBD
 from ceph_iscsi_config.utils import (convert_2_bytes, gen_control_string,
                                      valid_size, get_pool_id, ip_addresses,
-                                     get_pools, get_rbd_size, this_host,
-                                     human_size, CephiSCSIError)
+                                     get_pools, get_rbd_size, run_shell_cmd,
+                                     human_size, CephiSCSIError, this_host,
+                                     parse_disk_meta)
 from ceph_iscsi_config.gateway_object import GWObject
 from ceph_iscsi_config.target import GWTarget
 from ceph_iscsi_config.client import GWClient, CHAP
@@ -46,13 +48,14 @@ class RBDDev(object):
         ]
     }
 
-    def __init__(self, image, size, backstore, pool=None):
+    def __init__(self, image, size, backstore, pool=None, datapool=None):
         self.image = image
         self.size_bytes = convert_2_bytes(size)
         self.backstore = backstore
         if pool is None:
             pool = settings.config.pool
         self.pool = pool
+        self.datapool = datapool
         self.pool_id = get_pool_id(pool_name=self.pool)
         self.error = False
         self.error_msg = ''
@@ -74,14 +77,14 @@ def create(self):
                                     self.image,
                                     self.size_bytes,
                                     features=RBDDev.default_features(self.backstore),
-                                    old_format=False)
+                                    old_format=False,
+                                    data_pool=self.datapool)
 
                 except (rbd.ImageExists, rbd.InvalidArgument) as err:
                     self.error = True
-                    self.error_msg = ("Failed to create rbd image {} in "
-                                      "pool {} : {}".format(self.image,
-                                                            self.pool,
-                                                            err))
+                    self.error_msg = ("Failed to create rbd image {} in pool {}, "
+                                      "datapool {} : {}".format(self.image, self.pool,
+                                                                self.datapool, err))
 
     def delete(self):
         """
@@ -289,14 +292,18 @@ class LUN(GWObject):
         USER_RBD: TCMU_SETTINGS
     }
 
-    def __init__(self, logger, pool, image, size, allocating_host,
+    def __init__(self, logger, pool, datapool, image, size, allocating_host,
                  backstore, backstore_object_name):
         self.logger = logger
         self.image = image
         self.pool = pool
+        self.datapool = datapool
         self.pool_id = 0
         self.size_bytes = convert_2_bytes(size)
-        self.config_key = '{}/{}'.format(self.pool, self.image)
+        if datapool:
+            self.config_key = '{}+{}/{}'.format(pool, datapool, image)
+        else:
+            self.config_key = '{}/{}'.format(pool, image)
 
         self.allocating_host = allocating_host
         self.backstore = backstore
@@ -351,7 +358,7 @@ def remove_lun(self, preserve_image):
             if self.error:
                 return
 
-        rbd_image = RBDDev(self.image, '0G', self.backstore, self.pool)
+        rbd_image = RBDDev(self.image, '0G', self.backstore, self.pool, self.datapool)
 
         if local_gw == self.allocating_host:
             # by using the allocating host we ensure the delete is not
@@ -574,6 +581,38 @@ def activate(self):
             if client_err:
                 raise CephiSCSIError(client_err)
 
+    def _erasure_pool_check(self):
+        if not self.datapool:
+            return None
+
+        data, err = run_shell_cmd(
+            "ceph -n {client_name} --conf {cephconf} osd metadata --format=json".
+            format(client_name=settings.config.cluster_client_name,
+                   cephconf=settings.config.cephconf))
+        if err:
+            self.logger.error("Cannot get the objectstore type")
+            return err
+        bluestore = False
+        for _osd in json.loads(data):
+            store_type = _osd['osd_objectstore']
+            self.logger.debug(f"objectstore type is ({store_type})")
+            if store_type == 'bluestore':
+                bluestore = True
+                break
+
+        if not bluestore:
+            return None
+
+        data, err = run_shell_cmd(
+            "ceph -n {client_name} --conf {cephconf} osd pool get {pool} allow_ec_overwrites".
+            format(client_name=settings.config.cluster_client_name,
+                   cephconf=settings.config.cephconf, pool=self.datapool))
+        if err:
+            self.logger.error(f"Cannot get allow_ec_overwrites from pool ({self.pool})")
+            return err
+        self.logger.debug(f"erasure pool ({self.pool}) allow_ec_overwrites is enabled")
+        return None
+
     def allocate(self, keep_dev_in_lio=True, in_wwn=None):
         """
         Create image and add to LIO and config.
@@ -583,6 +622,10 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None):
         :return: LIO storage object if successful and keep_dev_in_lio=True
                  else None.
         """
+        err = self._erasure_pool_check()
+        if err:
+            return None
+
         self.logger.debug("LUN.allocate starting, listing rbd devices")
         disk_list = RBDDev.rbd_list(pool=self.pool)
         self.logger.debug("rados pool '{}' contains the following - "
@@ -593,7 +636,8 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None):
                           "allocations is {}".format(local_gw,
                                                      self.allocating_host))
 
-        rbd_image = RBDDev(self.image, self.size_bytes, self.backstore, self.pool)
+        rbd_image = RBDDev(self.image, self.size_bytes, self.backstore, self.pool,
+                           self.datapool)
         self.pool_id = rbd_image.pool_id
 
         # if the image required isn't defined, create it!
@@ -703,6 +747,7 @@ def allocate(self, keep_dev_in_lio=True, in_wwn=None):
                     disk_attr = {"wwn": wwn,
                                  "image": self.image,
                                  "pool": self.pool,
+                                 "datapool": self.datapool,
                                  "allocating_host": self.allocating_host,
                                  "pool_id": rbd_image.pool_id,
                                  "controls": self.controls,
@@ -963,7 +1008,10 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs):
 
         :param ceph_iscsi_config: Config object
         :param logger: logger object
-        :param image_id: (str) <pool>.<image> format
+        :param pool: (str) pool name
+        :param datapool: (str) datapool name
+        :param image: (str) image name
+        :param size: (str) size
         :return: (str) either 'ok' or an error description
         """
 
@@ -993,12 +1041,18 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs):
 
         config = ceph_iscsi_config.config
 
-        disk_key = "{}/{}".format(kwargs['pool'], kwargs['image'])
+        datapool = kwargs.get('datapool', None)
+        if datapool:
+            disk_key = "{}+{}/{}".format(kwargs['pool'], datapool, kwargs['image'])
+        else:
+            disk_key = "{}/{}".format(kwargs['pool'], kwargs['image'])
 
         if mode in ['create', 'resize']:
 
             if kwargs['pool'] not in get_pools():
                 return "pool name is invalid"
+            if datapool and datapool not in get_pools():
+                return "datapool name is invalid"
 
         if mode == 'create':
             if kwargs['size'] and not valid_size(kwargs['size']):
@@ -1010,6 +1064,8 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs):
             disk_regex = re.compile(r"^[a-zA-Z0-9\-_\.]+$")
             if not disk_regex.search(kwargs['pool']):
                 return "Invalid pool name (use alphanumeric, '_', '.', or '-' characters)"
+            if datapool and not disk_regex.search(datapool):
+                return "Invalid datapool name (use alphanumeric, '_', '.', or '-' characters)"
             if not disk_regex.search(kwargs['image']):
                 return "Invalid image name (use alphanumeric, '_', '.', or '-' characters)"
 
@@ -1040,9 +1096,7 @@ def valid_disk(ceph_iscsi_config, logger, **kwargs):
         if mode in ["resize", "delete", "reconfigure"]:
             # disk must exist in the config
             if disk_key not in config['disks']:
-                return ("rbd {}/{} is not defined to the "
-                        "configuration".format(kwargs['pool'],
-                                               kwargs['image']))
+                return ("rbd {} is not defined to the configuration".format(disk_key))
 
         if mode == 'resize':
 
@@ -1227,17 +1281,19 @@ def define_luns(logger, config, target):
                 with cluster.open_ioctx(pool) as ioctx:
 
                     pool_disks = [disk_key for disk_key in srtd_disks
-                                  if disk_key.startswith(pool + '/')]
+                                  if disk_key.startswith(pool + '/')
+                                     or disk_key.startswith(pool + '+')]
                     for disk_key in pool_disks:
 
-                        pool, image_name = disk_key.split('/')
+                        pool, datapool, image_name = parse_disk_meta(disk_key)
+
                         with rbd.Image(ioctx, image_name) as rbd_image:
 
                             disk_config = config.config['disks'][disk_key]
                             backstore = disk_config['backstore']
                             backstore_object_name = disk_config['backstore_object_name']
 
-                            lun = LUN(logger, pool, image_name,
+                            lun = LUN(logger, pool, datapool, image_name,
                                       rbd_image.size(), local_gw, backstore,
                                       backstore_object_name)
 

diff --git a/ceph_iscsi_config/utils.py b/ceph_iscsi_config/utils.py
@@ -28,6 +28,19 @@ class CephiSCSIInval(CephiSCSIError):
     pass
 
 
+def parse_disk_meta(disk):
+    pool = None
+    datapool = None
+    image = None
+    pools, image = disk.split('/')
+    try:
+        pool, datapool = pools.split('+')
+    except ValueError:
+        pool = pools
+        pass
+    return pool, datapool, image
+
+
 def run_shell_cmd(cmd, stderr=None, shell=True):
     if not stderr:
         stderr = subprocess.STDOUT

diff --git a/gwcli/client.py b/gwcli/client.py
@@ -4,7 +4,7 @@
 
 from ceph_iscsi_config.client import CHAP, GWClient
 import ceph_iscsi_config.settings as settings
-from ceph_iscsi_config.utils import human_size, this_host
+from ceph_iscsi_config.utils import human_size, this_host, parse_disk_meta
 
 from rtslib_fb.utils import normalize_wwn, RTSLibError
 
@@ -479,8 +479,8 @@ def ui_command_disk(self, action='add', disk=None, size=None):
         in the configuration, the cli will attempt to create it for you.
 
         e.g.
-        disk add <pool_name/image_name> <size>
-        disk remove <pool_name/image_name>
+        disk add <pool_name[+<datapool_name>]/image_name> <size>
+        disk remove <pool_name[+<datapool_name>]/image_name>
 
         Adding a disk will result in the disk occupying the client's next
         available lun id. Once allocated removing a LUN will not change the
@@ -540,12 +540,12 @@ def ui_command_disk(self, action='add', disk=None, size=None):
 
                 # a disk given here would be of the form pool.image
                 try:
-                    pool, image = disk.split('/')
+                    pool, datapool, image = parse_disk_meta(disk)
                 except ValueError:
                     self.logger.error("Invalid format. Use pool_name/disk_name")
                     return
 
-                rc = ui_disks.create_disk(pool=pool, image=image, size=size)
+                rc = ui_disks.create_disk(pool=pool, datapool=datapool, image=image, size=size)
                 if rc == 0:
                     self.logger.debug("disk auto-define successful")
                 else: