Skip to content

Commit

Permalink
external: add toplogyconstraintpool support for rbd storageclass
Browse files Browse the repository at this point in the history 
with topologyconstrain pool we can enable any
replica pool usage and also can store data at any
topology

Signed-off-by: parth-gr <[email protected]>
  • Loading branch information
@parth-gr
parth-gr committed Mar 5, 2024
1 parent bcab869 commit 12f9a78
Show file tree
Hide file tree
Showing 3 changed files with 191 additions and 5 deletions.
14 changes: 14 additions & 0 deletions Documentation/CRDs/Cluster/external-cluster.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --
* `--upgrade`: (optional) Upgrades the cephCSIKeyrings(For example: client.csi-cephfs-provisioner) and client.healthchecker ceph users with new permissions needed for the new cluster version and older permission will still be applied.
* `--restricted-auth-permission`: (optional) Restrict cephCSIKeyrings auth permissions to specific pools, and cluster. Mandatory flags that need to be set are `--rbd-data-pool-name`, and `--k8s-cluster-name`. `--cephfs-filesystem-name` flag can also be passed in case of CephFS user restriction, so it can restrict users to particular CephFS filesystem.
* `--v2-port-enable`: (optional) Enables the v2 mon port (3300) for mons.
* `--topology-pools`: (optional) comma-separated list of topology-constrained rbd pools
* `--topology-failure-domain-label`: (optional) k8s cluster failure domain label (example: zone,rack,host,etc) for the topology-pools that are matching the ceph domain
* `--topology-failure-domain-values`: (optional) comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the topology-pools list

### Multi-tenancy

Expand All @@ -84,6 +87,17 @@ See the [Multisite doc](https://docs.ceph.com/en/quincy/radosgw/multisite/#confi
python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --format bash --rgw-endpoint <rgw_endpoint> --rgw-realm-name <rgw_realm_name>> --rgw-zonegroup-name <rgw_zonegroup_name> --rgw-zone-name <rgw_zone_name>>
```

### Topology Based Provisioning

Enable Topology Based Provisioning for RBD pools by passing `--topology-pools`, `--topology-failure-domain-label` and `--topology-failure-domain-values` flags.
A new storageclass will be created by the import script named `ceph-rbd-topology` with `volumeBindingMode: WaitForFirstConsumer`
and will configure topologyConstrainedPools according the input provided.
Later use the storageclass to create a volume in the pool matching the topology of the pod scheduling.

```console
python3 create-external-cluster-resources.py --rbd-data-pool-name pool_name --topology-pools p,q,r --topology-failure-domain-label labelName --topology-failure-domain-values x,y,z --format bash
```

### Upgrade Example

1) If consumer cluster doesn't have restricted caps, this will upgrade all the default csi-users (non-restricted):
Expand Down
122 changes: 118 additions & 4 deletions deploy/examples/create-external-cluster-resources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,24 @@ def gen_arg_parser(cls, args_to_parse=None):
required=False,
help="provides the name of the rgw-zonegroup",
)
output_group.add_argument(
"--topology-pools",
default="",
required=False,
help="comma-separated list of topology-constrained rbd pools",
)
output_group.add_argument(
"--topology-failure-domain-label",
default="",
required=False,
help="k8s cluster failure domain label (example: zone,rack,host,etc) for the topology-pools that are matching the ceph domain",
)
output_group.add_argument(
"--topology-failure-domain-values",
default="",
required=False,
help="comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the topology-pools list",
)

upgrade_group = argP.add_argument_group("upgrade")
upgrade_group.add_argument(
Expand Down Expand Up @@ -1329,10 +1347,10 @@ def create_rgw_admin_ops_user(self):
"",
)

def validate_rbd_pool(self):
if not self.cluster.pool_exists(self._arg_parser.rbd_data_pool_name):
def validate_rbd_pool(self, pool_name):
if not self.cluster.pool_exists(pool_name):
raise ExecutionFailureException(
f"The provided pool, '{self._arg_parser.rbd_data_pool_name}', does not exist"
f"The provided pool, '{pool_name}', does not exist"
)

def init_rbd_pool(self):
Expand Down Expand Up @@ -1509,6 +1527,50 @@ def validate_rgw_multisite(self, rgw_multisite_config_name, rgw_multisite_config
return "-1"
return ""

def convert_comma_seprated_to_array(self, value):
return value.split(",")

def raise_exception_if_any_topology_flag_is_missing(self):
if (
(
self._arg_parser.topology_pools != ""
and (
self._arg_parser.topology_failure_domain_label == ""
or self._arg_parser.topology_failure_domain_values == ""
)
)
or (
self._arg_parser.topology_failure_domain_label != ""
and (
self._arg_parser.topology_pools == ""
or self._arg_parser.topology_failure_domain_values == ""
)
)
or (
self._arg_parser.topology_failure_domain_values != ""
and (
self._arg_parser.topology_pools == ""
or self._arg_parser.topology_failure_domain_label == ""
)
)
):
raise ExecutionFailureException(
"provide all the topology flags --topology-pools, --topology-failure-domain-label, --topology-failure-domain-values"
)

def validate_topology_values(self, topology_pools, topology_fd):
if len(topology_pools) != len(topology_fd):
raise ExecutionFailureException(
f"The provided topology pools, '{topology_pools}', and "
f"topology failure domain, '{topology_fd}',"
f"are of different length, '{len(topology_pools)}' and '{len(topology_fd)}' respctively"
)
return

def validate_topology_rbd_pools(self, topology_rbd_pools):
for pool in topology_rbd_pools:
self.validate_rbd_pool(pool)

def _gen_output_map(self):
if self.out_map:
return
Expand All @@ -1518,7 +1580,7 @@ def _gen_output_map(self):
self._arg_parser.k8s_cluster_name = (
self._arg_parser.k8s_cluster_name.lower()
) # always convert cluster name to lowercase characters
self.validate_rbd_pool()
self.validate_rbd_pool(self._arg_parser.rbd_data_pool_name)
self.init_rbd_pool()
self.validate_rados_namespace()
self._excluded_keys.add("K8S_CLUSTER_NAME")
Expand Down Expand Up @@ -1593,6 +1655,30 @@ def _gen_output_map(self):
self.out_map["RBD_METADATA_EC_POOL_NAME"] = (
self.validate_rbd_metadata_ec_pool_name()
)
if (
self._arg_parser.topology_pools != ""
and self._arg_parser.topology_failure_domain_label != ""
and self._arg_parser.topology_failure_domain_values != ""
):
self.out_map["TOPOLOGY_POOLS"] = self._arg_parser.topology_pools
self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"] = (
self._arg_parser.topology_failure_domain_label
)
self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] = (
self._arg_parser.topology_failure_domain_values
)
self.validate_topology_values(
self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"]),
self.convert_comma_seprated_to_array(
self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"]
),
)
self.validate_topology_rbd_pools(
self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"])
)
else:
self.raise_exception_if_any_topology_flag_is_missing()

self.out_map["RGW_POOL_PREFIX"] = self._arg_parser.rgw_pool_prefix
self.out_map["RGW_ENDPOINT"] = ""
if self._arg_parser.rgw_endpoint:
Expand Down Expand Up @@ -1829,6 +1915,34 @@ def gen_json_out(self):
}
)

# if 'TOPOLOGY_POOLS', 'TOPOLOGY_FAILURE_DOMAIN_LABEL', 'TOPOLOGY_FAILURE_DOMAIN_VALUES' exists,
# then only add 'topology' StorageClass
if (
self.out_map["TOPOLOGY_POOLS"]
and self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"]
and self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"]
):
json_out.append(
{
"name": "ceph-rbd-topology-storageclass",
"kind": "StorageClass",
"data": {
"topologyFailureDomainLabel": self.out_map[
"TOPOLOGY_FAILURE_DOMAIN_LABEL"
],
"topologyFailureDomainValues": self.convert_comma_seprated_to_array(
self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"]
),
"topologyPools": self.convert_comma_seprated_to_array(
self.out_map["TOPOLOGY_POOLS"]
),
"csi.storage.k8s.io/provisioner-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}",
"csi.storage.k8s.io/controller-expand-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}",
"csi.storage.k8s.io/node-stage-secret-name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}",
},
}
)

# if 'CEPHFS_FS_NAME' exists, then only add 'cephfs' StorageClass
if self.out_map["CEPHFS_FS_NAME"]:
json_out.append(
Expand Down
60 changes: 59 additions & 1 deletion deploy/examples/import-external-cluster.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ ROOK_RBD_FEATURES=${ROOK_RBD_FEATURES:-"layering"}
ROOK_EXTERNAL_MAX_MON_ID=2
ROOK_EXTERNAL_MAPPING={}
RBD_STORAGE_CLASS_NAME=ceph-rbd
RBD_TOPOLOGY_STORAGE_CLASS_NAME=ceph-rbd-topology
CEPHFS_STORAGE_CLASS_NAME=cephfs
ROOK_EXTERNAL_MONITOR_SECRET=mon-secret
OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph
OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph
CSI_DRIVER_NAME_PREFIX=${CSI_DRIVER_NAME_PREFIX:-$OPERATOR_NAMESPACE}
RBD_PROVISIONER=$CSI_DRIVER_NAME_PREFIX".rbd.csi.ceph.com" # csi-provisioner-name
CEPHFS_PROVISIONER=$CSI_DRIVER_NAME_PREFIX".cephfs.csi.ceph.com" # csi-provisioner-name
Expand Down Expand Up @@ -298,6 +299,59 @@ eof
fi
}

function getTopologyTemplate() {
topology=$(
cat <<-END
{"poolName":"$1",
"domainSegments":[
{"domainLabel":"$2","value":"$3"},]},
END
)
}

function createTopology() {
TOPOLOGY=""
topology_pools_array=("$(echo "$TOPOLOGY_POOLS" | tr "," "\n")")
topology_failure_domain_value_array=("$(echo "$TOPOLOGY_FAILURE_DOMAIN_VALUES" | tr "," "\n")")

for ((i = 0; i < ${#topology_failure_domain_value_array[@]}; i++)); do
getTopologyTemplate "${topology_pools_array[$i]}" "$TOPOLOGY_FAILURE_DOMAIN_LABEL" "${topology_failure_domain_value_array[$i]}"
TOPOLOGY="$TOPOLOGY"$'\n'"$topology"
topology=""
done
}

function createRBDTopologyStorageClass() {
if ! kubectl -n "$NAMESPACE" get storageclass $RBD_TOPOLOGY_STORAGE_CLASS_NAME &>/dev/null; then
cat <<eof | kubectl create -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: $RBD_TOPOLOGY_STORAGE_CLASS_NAME
provisioner: $RBD_PROVISIONER
parameters:
clusterID: $CLUSTER_ID_RBD
imageFormat: "2"
imageFeatures: $ROOK_RBD_FEATURES
topologyConstrainedPools: |
[$TOPOLOGY
]
csi.storage.k8s.io/provisioner-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME"
csi.storage.k8s.io/provisioner-secret-namespace: $NAMESPACE
csi.storage.k8s.io/controller-expand-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME"
csi.storage.k8s.io/controller-expand-secret-namespace: $NAMESPACE
csi.storage.k8s.io/node-stage-secret-name: "rook-$CSI_RBD_NODE_SECRET_NAME"
csi.storage.k8s.io/node-stage-secret-namespace: $NAMESPACE
csi.storage.k8s.io/fstype: ext4
allowVolumeExpansion: true
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
eof
else
echo "storageclass $RBD_TOPOLOGY_STORAGE_CLASS_NAME already exists"
fi
}

function createCephFSStorageClass() {
if ! $KUBECTL -n "$NAMESPACE" get storageclass $CEPHFS_STORAGE_CLASS_NAME &>/dev/null; then
cat <<eof | $KUBECTL create -f -
Expand Down Expand Up @@ -357,3 +411,7 @@ fi
if [ -n "$CEPHFS_FS_NAME" ] && [ -n "$CEPHFS_POOL_NAME" ]; then
createCephFSStorageClass
fi
if [ -n "$TOPOLOGY_POOLS" ] && [ -n "$TOPOLOGY_FAILURE_DOMAIN_LABEL" ] && [ -n "$TOPOLOGY_FAILURE_DOMAIN_VALUES" ]; then
createTopology
createRBDTopologyStorageClass
fi

0 comments on commit 12f9a78

Please sign in to comment.