diff --git a/infrastructure/kubernetes-networking/.gitignore b/infrastructure/kubernetes-networking/.gitignore new file mode 100644 index 000000000..743452b55 --- /dev/null +++ b/infrastructure/kubernetes-networking/.gitignore @@ -0,0 +1 @@ +calico.yaml \ No newline at end of file diff --git a/infrastructure/kubernetes-networking/README.md b/infrastructure/kubernetes-networking/README.md index f4af77eac..4279726d9 100644 --- a/infrastructure/kubernetes-networking/README.md +++ b/infrastructure/kubernetes-networking/README.md @@ -1,22 +1,17 @@ -## Kubernetes Networking - CNI Setup +# Kubernetes Networking - CNI Setup -In order to correctly setup the cluster, the [calico.yaml](calico.yaml) configuration file of the CNI have been slightly modified. In particular the pod network CIDR has been configured as shown in the following snippet: +As for it concerns Kubernetes networking, we selected [Project Calico](https://www.projectcalico.org/), since it is one of the most popular CNI plugins. +In short, it limits the overhead by requiring no overlay and supports advanced features such as the definition of network policies to isolate the traffic between different containers. -```yaml -... -- name: CALICO_IPV4POOL_CIDR - value: "172.16.0.0/16" -... -``` - -Now, apply the [calico.yaml](calico.yaml) file: +## Calico Installation +In order to install Calico, you can perform the following operations, which will download the default configuration from the official webpage and apply it customizing the pod network CIDR according to the selected cluster setup: -```sh -$ kubectl apply -f calico.yaml +```bash +$ export CALICO_VERSION=v3.16 +$ curl https://docs.projectcalico.org/${CALICO_VERSION}/manifests/calico.yaml -o calico.yaml +$ kubectl apply -k . ``` -This will setup Calico with the following networking configuration of the cluster: - - IP addresses of pods: 172.16.0.0/16 - - IP addresses of services: 10.96.0.0/12 - -IP addresses of the worker nodes are outside CALICO configuration. +## Selected cluster networking configuration +- IP addresses of pods: 172.16.0.0/16 +- IP addresses of services: 10.96.0.0/12 diff --git a/infrastructure/kubernetes-networking/calico-patch.yaml b/infrastructure/kubernetes-networking/calico-patch.yaml new file mode 100644 index 000000000..454c6c093 --- /dev/null +++ b/infrastructure/kubernetes-networking/calico-patch.yaml @@ -0,0 +1,13 @@ + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: calico-node + namespace: kube-system + spec: + template: + spec: + containers: + - name: calico-node + env: + - name: CALICO_IPV4POOL_CIDR + value: "172.16.0.0/16" diff --git a/infrastructure/kubernetes-networking/calico.yaml b/infrastructure/kubernetes-networking/calico.yaml deleted file mode 100644 index 7c81c8508..000000000 --- a/infrastructure/kubernetes-networking/calico.yaml +++ /dev/null @@ -1,748 +0,0 @@ -# Source: calico/templates/calico-config.yaml -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # Typha is disabled. - typha_service_name: "none" - # Configure the backend to use. - calico_backend: "bird" - # Configure the MTU to use - veth_mtu: "1440" - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - } - ] - } ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: FelixConfiguration - plural: felixconfigurations - singular: felixconfiguration ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ipamblocks.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPAMBlock - plural: ipamblocks - singular: ipamblock ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: blockaffinities.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BlockAffinity - plural: blockaffinities - singular: blockaffinity ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ipamhandles.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPAMHandle - plural: ipamhandles - singular: ipamhandle ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ipamconfigs.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPAMConfig - plural: ipamconfigs - singular: ipamconfig ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPPeer - plural: bgppeers - singular: bgppeer ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPConfiguration - plural: bgpconfigurations - singular: bgpconfiguration ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPPool - plural: ippools - singular: ippool ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: hostendpoints.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: HostEndpoint - plural: hostendpoints - singular: hostendpoint ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: ClusterInformation - plural: clusterinformations - singular: clusterinformation ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkPolicy - plural: globalnetworkpolicies - singular: globalnetworkpolicy ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkSet - plural: globalnetworksets - singular: globalnetworkset ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - scope: Namespaced - group: crd.projectcalico.org - version: v1 - names: - kind: NetworkPolicy - plural: networkpolicies - singular: networkpolicy ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: networksets.crd.projectcalico.org -spec: - scope: Namespaced - group: crd.projectcalico.org - version: v1 - names: - kind: NetworkSet - plural: networksets - singular: networkset - -# Source: calico/templates/rbac.yaml ---- -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: -- # Nodes are watched to monitor for deletions. - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get -- # Pods are queried to check for existence. - apiGroups: [""] - resources: - - pods - verbs: - - get -- # IPAM resources are manipulated when nodes are deleted. - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list -- apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete -- # Needs access to update clusterinformations. - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - create - - update ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: -- # The CNI plugin needs to get pods, nodes, and namespaces. - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get -- apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get -- apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update -- # Watch for changes to Kubernetes NetworkPolicies. - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list -- # Used by Calico for policy information. - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch -- # The CNI plugin patches pods/status. - apiGroups: [""] - resources: - - pods/status - verbs: - - patch -- # Calico monitors various CRDs for config. - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - verbs: - - get - - list - - watch -- # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update -- # Calico stores some configuration information on the node. - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch -- # These permissions are only requried for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update -- # These permissions are required for Calico CNI to perform IPAM allocations. - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete -- apiGroups: ["crd.projectcalico.org"] - resources: - - ipamconfigs - verbs: - - get -- # Block affinities must also be watchable by confd for route aggregation. - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - verbs: - - watch -- # The Calico IPAM migration needs to get daemonsets. These permissions can be - # removed if not upgrading from an installation using host-local IPAM. - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the calico-node container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - annotations: - # This, along with the CriticalAddonsOnly toleration below, - # marks the pod as a critical add-on, ensuring it gets - # priority scheduling and that its resources are reserved - # if it ever gets evicted. - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - nodeSelector: - beta.kubernetes.io/os: linux - hostNetwork: true - tolerations: - - # Make sure calico-node gets scheduled on all nodes. - effect: NoSchedule - operator: Exists - - # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - - # This container performs upgrade from host-local IPAM to calico-ipam. - # It can be deleted if this is a fresh installation, or if you have already - # upgraded to use calico-ipam. - name: upgrade-ipam - image: calico/cni:v3.11.2 - command: ["/opt/cni/bin/calico-ipam", "-upgrade"] - env: - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - volumeMounts: - - mountPath: /var/lib/cni/networks - name: host-local-net-dir - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - securityContext: - privileged: true - - # This container installs the CNI binaries - # and CNI network config file on each node. - name: install-cni - image: calico/cni:v3.11.2 - command: ["/install-cni.sh"] - env: - - # Name of the CNI config file to create. - name: CNI_CONF_NAME - value: "10-calico.conflist" - - # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - - # Set the hostname based on the k8s node name. - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - # CNI MTU Config variable - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - - # Prevents the container from sleeping forever. - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: calico/pod2daemon-flexvol:v3.11.2 - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true - containers: - - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - name: calico-node - image: calico/node:v3.11.2 - env: - - # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE - value: "kubernetes" - - # Wait for the datastore. - name: WAIT_FOR_DATASTORE - value: "true" - - # Set based on the k8s node name. - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - # Choose the backend to use. - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - - # Cluster type to identify the deployment type - name: CLUSTER_TYPE - value: "k8s,bgp" - - # Auto-detect the BGP IP address. - name: IP - value: "autodetect" - - # Enable IPIP - name: CALICO_IPV4POOL_IPIP - value: "Always" - - # Set MTU for tunnel device used if ipip is enabled - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. This should fall within `--cluster-cidr`. - name: CALICO_IPV4POOL_CIDR - value: "172.16.0.0/16" - - # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - - # Set Felix endpoint to host default action to ACCEPT. - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - - # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT - value: "false" - - # Set Felix logging to "info" - name: FELIX_LOGSEVERITYSCREEN - value: "info" - - name: FELIX_HEALTHENABLED - value: "true" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - - -bird-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/calico-node - - -felix-ready - - -bird-ready - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - volumes: - - # Used by calico-node. - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - # Used to install CNI. - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - - # Mount in the directory for host-local IPAM allocations. This is - # used when upgrading from host-local to calico-ipam, and can be removed - # if not using the upgrade-ipam init container. - name: host-local-net-dir - hostPath: - path: /var/lib/cni/networks - - # Used to create per-pod Unix Domain Sockets - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent - - # Used to install Flex Volume Driver - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - -# Source: calico/templates/calico-kube-controllers.yaml ---- -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - nodeSelector: - beta.kubernetes.io/os: linux - tolerations: - - # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: calico/kube-controllers:v3.11.2 - env: - - # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system diff --git a/infrastructure/kubernetes-networking/kustomization.yaml b/infrastructure/kubernetes-networking/kustomization.yaml new file mode 100644 index 000000000..c7d7582b9 --- /dev/null +++ b/infrastructure/kubernetes-networking/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- calico.yaml + +patchesStrategicMerge: +- calico-patch.yaml diff --git a/infrastructure/storage-provisioning/README.md b/infrastructure/storage-provisioning/README.md index 7f9274c8f..9e38652e5 100644 --- a/infrastructure/storage-provisioning/README.md +++ b/infrastructure/storage-provisioning/README.md @@ -1,36 +1,69 @@ -# Storage Provisioning - Rook +# Storage Provisioning - Rook-Ceph -Rook is a cloud-native storage orchestrator for Kubernetes. -In this scenario we used Rook with Ceph storage provider. +[Rook](https://rook.io/) is a cloud-native storage orchestrator for Kubernetes. +Among the different alternatives supported by Rook, we adopted [Ceph](https://ceph.io) as the selected storage provider. ## Install Rook-Ceph -To install Rook-Ceph apply the following commands. -Ceph uses a directory under /var/lib/Rook that is a mount point of a free partition. +### Deploy the Rook Operator +In order to set-up Rook-Ceph, it is first necessary to deploy the Rook Operator, together with the set of CRDs and permissions required for its operations. +Adopting the out-of-the-box configurations, it is possible to leverage the manifests provided within the Rook repository for its deployment: ```bash -$ kubectl create -f manifests/common.yaml -$ kubectl create -f manifests/operator.yaml -# edit cluster.yaml with your preferences before deploy it -$ kubectl create -f manifests/cluster.yaml -$ kubectl create -f manifests/toolbox.yaml +$ export ROOK_VERSION=1.4 +$ kubectl apply -f https://raw.githubusercontent.com/rook/rook/release-${ROOK_VERSION}/cluster/examples/kubernetes/ceph/common.yaml +$ kubectl apply -f https://raw.githubusercontent.com/rook/rook/release-${ROOK_VERSION}/cluster/examples/kubernetes/ceph/operator.yaml ``` -## Test +### Create the Ceph Clusters +Once the Rook Operator is ready, it is possible to trigger the creation of the desired Ceph Clusters through the definition of the corresponding `CephCluster` CRs. +Two different clusters are defined in the following, one leveraging faster SSD storage (yet, with lower available capacity) and the other backed by traditional HDDs. +While representing a working example, these manifests need to be customized depending on the specific characteristics of the cluster where they are applied (e.g. to define which drives belong to each cluster). +Additionally, it may be necessary to create the `namespace` where the secondary cluster is going to be defined. -To check the status of ceph you can run the following command to open toolbox's shell. +```bash +$ kubectl create -f ceph-clusters/ceph-cluster-primary.yaml +$ kubectl create -f ceph-clusters/ceph-cluster-secondary.yaml +``` + +### Deploy the Rook Toolbox + +The Rook toolbox is a container with common tools used for rook debugging and testing. Specifically, it allows to interact with the `ceph` cluster to check its status and trigger maintenance operations. +In order to deploy the toolbox, please refer to the illustrative `deployment` definition available in the [official documentation](https://rook.io/docs/rook/v1.4/ceph-toolbox.html) (a different instance of the toolbox needs to be created for each Ceph cluster). + +Once the toolbox is correctly deployed, it is possible to enter a shell with: ```bash -$ kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') bash +$ kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') -- /bin/bash ``` -After that in toolbox's shell you can run +Once in the toolbox's shell, it is possible to run, e.g., `ceph status` to verify the status of the cluster. + +## Upgrade Rook-Ceph + +### Upgrade Rook +To upgrade Rook, it is necessary to edit the image version of the operator deployment. In turn, it will proceed to upgrade all the other components. +Patch release upgrades (e.g. from v1.4.1 to v1.4.2) are as easy as issuing: +```bash +$ kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.4.2 ``` -root$ ceph status + +The upgrade between actual versions (e.g. from v1.3.10 to v1.4.2), on the other hand, typically involves additional preparation steps to update the CRD definitions and the RBAC settings. +To this end, it is suggested to carefully follow the specific instructions available on the [rook.io](https://rook.io/docs/rook/v1.4/ceph-upgrade.html) website. + +### Upgrade Ceph +To upgrade Ceph, it is necessary to edit the image version specified within the `CephCluster` CR. +With reference to the clusters previously created, this operation can be completed with: + +```bash +$ export CEPH_IMAGE='ceph/ceph:v15.2.4' +$ kubectl -n rook-ceph patch CephCluster rook-ceph --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$CEPH_IMAGE\"}}}" +$ kubectl -n iscsi-rook-ceph patch CephCluster iscsi-rook-ceph --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$CEPH_IMAGE\"}}}" ``` -To test Rook follow those commands. +## Test the PVC provisioning +To test Rook using an illustrative example, follow those commands, which will create a `StorageClass` and some `PersistentVolumeClaims` mounted by the corresponding applications. ```bash $ kubectl create -f examples/storageclass.yaml @@ -40,7 +73,7 @@ $ kubectl create -f examples/wordpress.yaml Both of these apps creates a block volume and mount it to their respective pod. You can see the Kubernetes volume claims by running the following: -``` +```bash $ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE mysql-pv-claim Bound pvc-2a53d32d-0f38-4d5a-816f-de09d07768f6 20Gi RWO rook-ceph-block 134m diff --git a/infrastructure/storage-provisioning/ceph-clusters/ceph-cluster-primary.yaml b/infrastructure/storage-provisioning/ceph-clusters/ceph-cluster-primary.yaml new file mode 100644 index 000000000..fe5077db3 --- /dev/null +++ b/infrastructure/storage-provisioning/ceph-clusters/ceph-cluster-primary.yaml @@ -0,0 +1,64 @@ +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + cephVersion: + image: ceph/ceph:v15.2.4 + crashCollector: + disable: false + dashboard: + enabled: true + ssl: true + dataDirHostPath: /var/lib/rook + disruptionManagement: + machineDisruptionBudgetNamespace: openshift-machine-api + osdMaintenanceTimeout: 30 + external: + enable: false + mgr: + modules: + - enabled: true + name: pg_autoscaler + mon: + count: 3 + monitoring: + enabled: true + rulesNamespace: rook-ceph + network: + hostNetwork: false + provider: "" + selectors: null + rbdMirroring: + workers: 0 + removeOSDsIfOutAndSafeToRemove: false + storage: + config: null + nodes: + - config: null + devices: + - config: null + name: sdd6 + name: worker-1 + resources: {} + - config: null + devices: + - config: null + name: sdd6 + name: worker-2 + resources: {} + - config: null + devices: + - config: null + name: sdd6 + name: worker-3 + resources: {} + - config: null + devices: + - config: null + name: sdd6 + name: worker-4 + resources: {} + storageClassDeviceSets: null + useAllDevices: false \ No newline at end of file diff --git a/infrastructure/storage-provisioning/ceph-clusters/ceph-cluster-secondary.yaml b/infrastructure/storage-provisioning/ceph-clusters/ceph-cluster-secondary.yaml new file mode 100644 index 000000000..165f93db1 --- /dev/null +++ b/infrastructure/storage-provisioning/ceph-clusters/ceph-cluster-secondary.yaml @@ -0,0 +1,80 @@ +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: iscsi-rook-ceph + namespace: iscsi-rook-ceph +spec: + cephVersion: + image: ceph/ceph:v15.2.4 + crashCollector: + disable: false + dashboard: + enabled: true + ssl: true + dataDirHostPath: /var/lib/iscsi-rook + disruptionManagement: + machineDisruptionBudgetNamespace: openshift-machine-api + osdMaintenanceTimeout: 30 + external: + enable: false + mgr: + modules: + - enabled: true + name: pg_autoscaler + mon: + count: 3 + monitoring: + enabled: true + rulesNamespace: rook-ceph + network: + hostNetwork: false + provider: "" + selectors: null + rbdMirroring: + workers: 0 + removeOSDsIfOutAndSafeToRemove: false + storage: + config: null + nodes: + - config: null + devices: + - config: null + name: sda + - config: null + name: sdb + - config: null + name: sdc + name: worker-1 + resources: {} + - config: null + devices: + - config: null + name: sda + - config: null + name: sdb + - config: null + name: sdc + name: worker-2 + resources: {} + - config: null + devices: + - config: null + name: sda + - config: null + name: sdb + - config: null + name: sdc + name: worker-3 + resources: {} + - config: null + devices: + - config: null + name: sda + - config: null + name: sdb + - config: null + name: sdc + name: worker-4 + resources: {} + storageClassDeviceSets: null + useAllDevices: false diff --git a/infrastructure/storage-provisioning/manifests/cluster.yaml b/infrastructure/storage-provisioning/manifests/cluster.yaml deleted file mode 100644 index 262059b3f..000000000 --- a/infrastructure/storage-provisioning/manifests/cluster.yaml +++ /dev/null @@ -1,189 +0,0 @@ -################################################################################################################# -# Define the settings for the rook-ceph cluster with common settings for a production cluster. -# All nodes with available raw devices will be used for the Ceph cluster. At least three nodes are required -# in this example. See the documentation for more details on storage settings available. - -# For example, to create the cluster: -# kubectl create -f common.yaml -# kubectl create -f operator.yaml -# kubectl create -f cluster.yaml -################################################################################################################# - -apiVersion: ceph.rook.io/v1 -kind: CephCluster -metadata: - name: rook-ceph - namespace: rook-ceph -spec: - cephVersion: - # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw). - # v13 is mimic, v14 is nautilus, and v15 is octopus. - # RECOMMENDATION: In production, use a specific version tag instead of the general v14 flag, which pulls the latest release and could result in different - # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/. - # If you want to be more precise, you can always use a timestamp tag such ceph/ceph:v14.2.5-20190917 - # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities - image: ceph/ceph:v14.2.7 - # Whether to allow unsupported versions of Ceph. Currently mimic and nautilus are supported, with the recommendation to upgrade to nautilus. - # Octopus is the version allowed when this is set to true. - # Do not set to true in production. - allowUnsupported: false - # The path on the host where configuration files will be persisted. Must be specified. - # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster. - # In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment. - dataDirHostPath: /var/lib/rook - # Whether or not upgrade should continue even if a check fails - # This means Ceph's status could be degraded and we don't recommend upgrading but you might decide otherwise - # Use at your OWN risk - # To understand Rook's upgrade process of Ceph, read https://rook.io/docs/rook/master/ceph-upgrade.html#ceph-version-upgrades - skipUpgradeChecks: false - # Whether or not continue if PGs are not clean during an upgrade - continueUpgradeAfterChecksEvenIfNotHealthy: false - # set the amount of mons to be started - mon: - count: 3 - allowMultiplePerNode: false - # mgr: - # modules: - # Several modules should not need to be included in this list. The "dashboard" and "monitoring" modules - # are already enabled by other settings in the cluster CR and the "rook" module is always enabled. - # - name: pg_autoscaler - # enabled: true - # enable the ceph dashboard for viewing cluster status - dashboard: - enabled: true - # serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy) - # urlPrefix: /ceph-dashboard - # serve the dashboard at the given port. - # port: 8443 - # serve the dashboard using SSL - ssl: true - # enable prometheus alerting for cluster - monitoring: - # requires Prometheus to be pre-installed - enabled: false - # namespace to deploy prometheusRule in. If empty, namespace of the cluster will be used. - # Recommended: - # If you have a single rook-ceph cluster, set the rulesNamespace to the same namespace as the cluster or keep it empty. - # If you have multiple rook-ceph clusters in the same k8s cluster, choose the same namespace (ideally, namespace with prometheus - # deployed) to set rulesNamespace for all the clusters. Otherwise, you will get duplicate alerts with multiple alert definitions. - rulesNamespace: rook-ceph - network: - # toggle to use hostNetwork - hostNetwork: false - rbdMirroring: - # The number of daemons that will perform the rbd mirroring. - # rbd mirroring must be configured with "rbd mirror" from the rook toolbox. - workers: 0 - # enable the crash collector for ceph daemon crash collection - crashCollector: - disable: false - # To control where various services will be scheduled by kubernetes, use the placement configuration sections below. - # The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage-node' and - # tolerate taints with a key of 'storage-node'. - # placement: - # all: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: role - # operator: In - # values: - # - storage-node - # podAffinity: - # podAntiAffinity: - # tolerations: - # - key: storage-node - # operator: Exists - # The above placement information can also be specified for mon, osd, and mgr components - # mon: - # Monitor deployments may contain an anti-affinity rule for avoiding monitor - # collocation on the same node. This is a required rule when host network is used - # or when AllowMultiplePerNode is false. Otherwise this anti-affinity rule is a - # preferred rule with weight: 50. - # osd: - # mgr: - - annotations: - # all: - # mon: - # osd: - # If no mgr annotations are set, prometheus scrape annotations will be set by default. - # mgr: - resources: - # The requests and limits set here, allow the mgr pod to use half of one CPU core and 1 gigabyte of memory - # mgr: - # limits: - # cpu: "500m" - # memory: "1024Mi" - # requests: - # cpu: "500m" - # memory: "1024Mi" - # The above example requests/limits can also be added to the mon and osd components - # mon: - # osd: - # prepareosd: - # crashcollector: - # The option to automatically remove OSDs that are out and are safe to destroy. - removeOSDsIfOutAndSafeToRemove: false - # priorityClassNames: - # all: rook-ceph-default-priority-class - # mon: rook-ceph-mon-priority-class - # osd: rook-ceph-osd-priority-class - # mgr: rook-ceph-mgr-priority-class - storage: # cluster level storage configuration and selection - useAllNodes: true - useAllDevices: false - directories: - - path: /var/lib/rook - #deviceFilter: - config: - # The default and recommended storeType is dynamically set to bluestore for devices and filestore for directories. - # Set the storeType explicitly only if it is required not to use the default. - # storeType: bluestore - # metadataDevice: "md0" # specify a non-rotational storage so ceph-volume will use it as block db device of bluestore. - # databaseSizeMB: "1024" # uncomment if the disks are smaller than 100 GB - # journalSizeMB: "1024" # uncomment if the disks are 20 GB or smaller - # osdsPerDevice: "1" # this value can be overridden at the node or device level - # encryptedDevice: "true" # the default value for this option is "false" - # Cluster level list of directories to use for filestore-based OSD storage. If uncomment, this example would create an OSD under the dataDirHostPath. - #directories: - #- path: /var/lib/rook - # Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named - # nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label. - # nodes: - # - name: "172.17.4.101" - # directories: # specific directories to use for storage can be specified for each node - # - path: "/rook/storage-dir" - # resources: - # limits: - # cpu: "500m" - # memory: "1024Mi" - # requests: - # cpu: "500m" - # memory: "1024Mi" - # - name: "172.17.4.201" - # devices: # specific devices to use for storage can be specified for each node - # - name: "sdb" - # - name: "nvme01" # multiple osds can be created on high performance devices - # config: - # osdsPerDevice: "5" - # config: # configuration can be specified at the node level which overrides the cluster level config - # storeType: filestore - # - name: "172.17.4.301" - # deviceFilter: "^sd." - - # The section for configuring management of daemon disruptions during upgrade or fencing. - disruptionManagement: - # If true, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically - # via the strategy outlined in the [design](https://github.com/rook/rook/blob/master/design/ceph/ceph-managed-disruptionbudgets.md). The operator will - # block eviction of OSDs by default and unblock them safely when drains are detected. - managePodBudgets: false - # A duration in minutes that determines how long an entire failureDomain like `region/zone/host` will be held in `noout` (in addition to the - # default DOWN/OUT interval) when it is draining. This is only relevant when `managePodBudgets` is `true`. The default value is `30` minutes. - osdMaintenanceTimeout: 30 - # If true, the operator will create and manage MachineDisruptionBudgets to ensure OSDs are only fenced when the cluster is healthy. - # Only available on OpenShift. - manageMachineDisruptionBudgets: false - # Namespace in which to watch for the MachineDisruptionBudgets. - machineDisruptionBudgetNamespace: openshift-machine-api diff --git a/infrastructure/storage-provisioning/manifests/common.yaml b/infrastructure/storage-provisioning/manifests/common.yaml deleted file mode 100644 index 39c4660c1..000000000 --- a/infrastructure/storage-provisioning/manifests/common.yaml +++ /dev/null @@ -1,1721 +0,0 @@ -################################################################################################################### -# Create the common resources that are necessary to start the operator and the ceph cluster. -# These resources *must* be created before the operator.yaml and cluster.yaml or their variants. -# The samples all assume that a single operator will manage a single cluster crd in the same "rook-ceph" namespace. -# -# If the operator needs to manage multiple clusters (in different namespaces), see the section below -# for "cluster-specific resources". The resources below that section will need to be created for each namespace -# where the operator needs to manage the cluster. The resources above that section do not be created again. -# -# Most of the sections are prefixed with a 'OLM' keyword which is used to build our CSV for an OLM (Operator Life Cycle manager) -################################################################################################################### - -# Namespace where the operator and other rook resources are created -apiVersion: v1 -kind: Namespace -metadata: - name: rook-ceph - -# OLM: BEGIN CEPH CRD -# The CRD declarations ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephclusters.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephCluster - listKind: CephClusterList - plural: cephclusters - singular: cephcluster - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - annotations: {} - cephVersion: - properties: - allowUnsupported: - type: boolean - image: - type: string - dashboard: - properties: - enabled: - type: boolean - urlPrefix: - type: string - port: - type: integer - minimum: 0 - maximum: 65535 - ssl: - type: boolean - dataDirHostPath: - pattern: ^/(\S+) - type: string - disruptionManagement: - properties: - machineDisruptionBudgetNamespace: - type: string - managePodBudgets: - type: boolean - osdMaintenanceTimeout: - type: integer - manageMachineDisruptionBudgets: - type: boolean - skipUpgradeChecks: - type: boolean - continueUpgradeAfterChecksEvenIfNotHealthy: - type: boolean - mon: - properties: - allowMultiplePerNode: - type: boolean - count: - maximum: 9 - minimum: 0 - type: integer - volumeClaimTemplate: {} - mgr: - properties: - modules: - items: - properties: - name: - type: string - enabled: - type: boolean - network: - properties: - hostNetwork: - type: boolean - provider: - type: string - selectors: {} - storage: - properties: - disruptionManagement: - properties: - machineDisruptionBudgetNamespace: - type: string - managePodBudgets: - type: boolean - osdMaintenanceTimeout: - type: integer - manageMachineDisruptionBudgets: - type: boolean - useAllNodes: - type: boolean - nodes: - items: - properties: - name: - type: string - config: - properties: - metadataDevice: - type: string - storeType: - type: string - pattern: ^(filestore|bluestore)$ - databaseSizeMB: - type: string - walSizeMB: - type: string - journalSizeMB: - type: string - osdsPerDevice: - type: string - encryptedDevice: - type: string - pattern: ^(true|false)$ - useAllDevices: - type: boolean - deviceFilter: - type: string - devicePathFilter: - type: string - directories: - type: array - items: - properties: - path: - type: string - devices: - type: array - items: - properties: - name: - type: string - config: {} - resources: {} - type: array - useAllDevices: - type: boolean - deviceFilter: - type: string - devicePathFilter: - type: string - directories: - type: array - items: - properties: - path: - type: string - config: {} - storageClassDeviceSets: {} - monitoring: - properties: - enabled: - type: boolean - rulesNamespace: - type: string - rbdMirroring: - properties: - workers: - type: integer - removeOSDsIfOutAndSafeToRemove: - type: boolean - external: - properties: - enable: - type: boolean - placement: {} - resources: {} - additionalPrinterColumns: - - name: DataDirHostPath - type: string - description: Directory used on the K8s nodes - JSONPath: .spec.dataDirHostPath - - name: MonCount - type: string - description: Number of MONs - JSONPath: .spec.mon.count - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - - name: State - type: string - description: Current State - JSONPath: .status.state - - name: Health - type: string - description: Ceph Health - JSONPath: .status.ceph.health - -# OLM: END CEPH CRD -# OLM: BEGIN CEPH CLIENT CRD ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephclients.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephClient - listKind: CephClientList - plural: cephclients - singular: cephclient - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - caps: - type: object - -# OLM: END CEPH CLIENT CRD -# OLM: BEGIN CEPH FS CRD ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephfilesystems.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephFilesystem - listKind: CephFilesystemList - plural: cephfilesystems - singular: cephfilesystem - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - metadataServer: - properties: - activeCount: - minimum: 1 - maximum: 10 - type: integer - activeStandby: - type: boolean - annotations: {} - placement: {} - resources: {} - metadataPool: - properties: - failureDomain: - type: string - replicated: - properties: - size: - minimum: 0 - maximum: 10 - type: integer - erasureCoded: - properties: - dataChunks: - minimum: 0 - maximum: 10 - type: integer - codingChunks: - minimum: 0 - maximum: 10 - type: integer - dataPools: - type: array - items: - properties: - failureDomain: - type: string - replicated: - properties: - size: - minimum: 0 - maximum: 10 - type: integer - erasureCoded: - properties: - dataChunks: - minimum: 0 - maximum: 10 - type: integer - codingChunks: - minimum: 0 - maximum: 10 - type: integer - preservePoolsOnDelete: - type: boolean - additionalPrinterColumns: - - name: ActiveMDS - type: string - description: Number of desired active MDS daemons - JSONPath: .spec.metadataServer.activeCount - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - -# OLM: END CEPH FS CRD -# OLM: BEGIN CEPH NFS CRD ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephnfses.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephNFS - listKind: CephNFSList - plural: cephnfses - singular: cephnfs - shortNames: - - nfs - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - rados: - properties: - pool: - type: string - namespace: - type: string - server: - properties: - active: - type: integer - annotations: {} - placement: {} - resources: {} - -# OLM: END CEPH NFS CRD -# OLM: BEGIN CEPH OBJECT STORE CRD ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectstores.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectStore - listKind: CephObjectStoreList - plural: cephobjectstores - singular: cephobjectstore - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - gateway: - properties: - type: - type: string - sslCertificateRef: {} - port: - type: integer - securePort: {} - instances: - type: integer - annotations: {} - placement: {} - resources: {} - metadataPool: - properties: - failureDomain: - type: string - replicated: - properties: - size: - type: integer - erasureCoded: - properties: - dataChunks: - type: integer - codingChunks: - type: integer - dataPool: - properties: - failureDomain: - type: string - replicated: - properties: - size: - type: integer - erasureCoded: - properties: - dataChunks: - type: integer - codingChunks: - type: integer - preservePoolsOnDelete: - type: boolean - -# OLM: END CEPH OBJECT STORE CRD -# OLM: BEGIN CEPH OBJECT STORE USERS CRD ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectstoreusers.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectStoreUser - listKind: CephObjectStoreUserList - plural: cephobjectstoreusers - singular: cephobjectstoreuser - shortNames: - - rcou - - objectuser - scope: Namespaced - version: v1 - -# OLM: END CEPH OBJECT STORE USERS CRD -# OLM: BEGIN CEPH BLOCK POOL CRD ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephblockpools.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephBlockPool - listKind: CephBlockPoolList - plural: cephblockpools - singular: cephblockpool - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - failureDomain: - type: string - replicated: - properties: - size: - type: integer - minimum: 0 - maximum: 9 - targetSizeRatio: - type: number - erasureCoded: - properties: - dataChunks: - type: integer - minimum: 0 - maximum: 9 - codingChunks: - type: integer - minimum: 0 - maximum: 9 - -# OLM: END CEPH BLOCK POOL CRD -# OLM: BEGIN CEPH VOLUME POOL CRD ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: volumes.rook.io -spec: - group: rook.io - names: - kind: Volume - listKind: VolumeList - plural: volumes - singular: volume - shortNames: - - rv - scope: Namespaced - version: v1alpha2 - -# OLM: END CEPH VOLUME POOL CRD -# OLM: BEGIN OBJECTBUCKET CRD ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: objectbuckets.objectbucket.io -spec: - group: objectbucket.io - versions: - - name: v1alpha1 - served: true - storage: true - names: - kind: ObjectBucket - listKind: ObjectBucketList - plural: objectbuckets - singular: objectbucket - shortNames: - - ob - - obs - scope: Cluster - subresources: - status: {} - -# OLM: END OBJECTBUCKET CRD -# OLM: BEGIN OBJECTBUCKETCLAIM CRD ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: objectbucketclaims.objectbucket.io -spec: - versions: - - name: v1alpha1 - served: true - storage: true - group: objectbucket.io - names: - kind: ObjectBucketClaim - listKind: ObjectBucketClaimList - plural: objectbucketclaims - singular: objectbucketclaim - shortNames: - - obc - - obcs - scope: Namespaced - subresources: - status: {} - -# OLM: END OBJECTBUCKETCLAIM CRD -# OLM: BEGIN OBJECTBUCKET ROLEBINDING ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-object-bucket -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-object-bucket -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph - -# OLM: END OBJECTBUCKET ROLEBINDING -# OLM: BEGIN OPERATOR ROLE ---- -# The cluster role for managing all the cluster-specific resources in a namespace -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt - labels: - operator: rook - storage-backend: ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" -rules: [] ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" -rules: -- apiGroups: - - "" - resources: - - secrets - - pods - - pods/log - - services - - configmaps - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - apps - resources: - - deployments - - daemonsets - verbs: - - get - - list - - watch - - create - - update - - delete ---- -# The role for the operator to manage resources in its own namespace -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: rook-ceph-system - namespace: rook-ceph - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - - services - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - apps - resources: - - daemonsets - - statefulsets - - deployments - verbs: - - get - - list - - watch - - create - - update - - delete ---- -# The cluster role for managing the Rook CRDs -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" -rules: [] ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-ceph-global-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create ---- -# Aspects of ceph-mgr that require cluster-wide access -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-mgr-cluster - labels: - operator: rook - storage-backend: ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-mgr-cluster-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" -rules: -- apiGroups: - - "" - resources: - - configmaps - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - list - - get - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-object-bucket - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" -rules: -- apiGroups: - - "" - verbs: - - "*" - resources: - - secrets - - configmaps -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" - -# OLM: END OPERATOR ROLE -# OLM: BEGIN SERVICE ACCOUNT SYSTEM ---- -# The rook system service account used by the operator, agent, and discovery pods -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-system - namespace: rook-ceph - labels: - operator: rook - storage-backend: ceph -# imagePullSecrets: -# - name: my-registry-secret - -# OLM: END SERVICE ACCOUNT SYSTEM -# OLM: BEGIN OPERATOR ROLEBINDING ---- -# Grant the operator, agent, and discovery agents access to resources in the namespace -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-system - namespace: rook-ceph - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-system -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph ---- -# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-global - namespace: rook-ceph - labels: - operator: rook - storage-backend: ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-global -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph - -# OLM: END OPERATOR ROLEBINDING -################################################################################################################# -# Beginning of cluster-specific resources. The example will assume the cluster will be created in the "rook-ceph" -# namespace. If you want to create the cluster in a different namespace, you will need to modify these roles -# and bindings accordingly. -################################################################################################################# -# Service account for the Ceph OSDs. Must exist and cannot be renamed. -# OLM: BEGIN SERVICE ACCOUNT OSD ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-osd - namespace: rook-ceph -# imagePullSecrets: -# - name: my-registry-secret - -# OLM: END SERVICE ACCOUNT OSD -# OLM: BEGIN SERVICE ACCOUNT MGR ---- -# Service account for the Ceph Mgr. Must exist and cannot be renamed. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-mgr - namespace: rook-ceph -# imagePullSecrets: -# - name: my-registry-secret - -# OLM: END SERVICE ACCOUNT MGR -# OLM: BEGIN CMD REPORTER SERVICE ACCOUNT ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: rook-ceph - -# OLM: END CMD REPORTER SERVICE ACCOUNT -# OLM: BEGIN CLUSTER ROLE ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-osd - namespace: rook-ceph -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "delete"] -- apiGroups: ["ceph.rook.io"] - resources: ["cephclusters", "cephclusters/finalizers"] - verbs: ["get", "list", "create", "update", "delete"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-osd - namespace: rook-ceph -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list ---- -# Aspects of ceph-mgr that require access to the system namespace -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-mgr-system - namespace: rook-ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-mgr-system-rules - namespace: rook-ceph - labels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -# Aspects of ceph-mgr that operate within the cluster's namespace -kind: Role -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-mgr - namespace: rook-ceph -rules: -- apiGroups: - - "" - resources: - - pods - - services - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" - -# OLM: END CLUSTER ROLE -# OLM: BEGIN CMD REPORTER ROLE ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-cmd-reporter - namespace: rook-ceph -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete - -# OLM: END CMD REPORTER ROLE -# OLM: BEGIN CLUSTER ROLEBINDING ---- -# Allow the operator to create resources in this cluster's namespace -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-cluster-mgmt -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph ---- -# Allow the osd pods in this namespace to work with configmaps -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-osd - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph ---- -# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-mgr - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-mgr -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph ---- -# Allow the ceph mgr to access the rook system resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-mgr-system - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-system -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph ---- -# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-mgr-cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-cluster -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph ---- -# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-osd -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph - -# OLM: END CLUSTER ROLEBINDING -# OLM: BEGIN CMD REPORTER ROLEBINDING ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-cmd-reporter - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: rook-ceph - -# OLM: END CMD REPORTER ROLEBINDING -################################################################################################################# -# Beginning of pod security policy resources. The example will assume the cluster will be created in the -# "rook-ceph" namespace. If you want to create the cluster in a different namespace, you will need to modify -# the roles and bindings accordingly. -################################################################################################################# -# OLM: BEGIN CLUSTER POD SECURITY POLICY ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: rook-privileged -spec: - privileged: true - allowedCapabilities: - # required by CSI - - SYS_ADMIN - # fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group - fsGroup: - rule: RunAsAny - # runAsUser, supplementalGroups - Rook needs to run some pods as root - # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - # seLinux - seLinux context is unknown ahead of time; set if this is well-known - seLinux: - rule: RunAsAny - volumes: - # recommended minimum set - - configMap - - downwardAPI - - emptyDir - - persistentVolumeClaim - - secret - - projected - # required for Rook - - hostPath - - flexVolume - # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known - # directory-based OSDs make this hard to nail down - # allowedHostPaths: - # - pathPrefix: "/run/udev" # for OSD prep - # readOnly: false - # - pathPrefix: "/dev" # for OSD prep - # readOnly: false - # - pathPrefix: "/var/lib/rook" # or whatever the dataDirHostPath value is set to - # readOnly: false - # Ceph requires host IPC for setting up encrypted devices - hostIPC: true - # Ceph OSDs need to share the same PID namespace - hostPID: true - # hostNetwork can be set to 'false' if host networking isn't used - hostNetwork: true - hostPorts: - - # Ceph messenger protocol v1 - min: 6789 - max: 6790 # <- support old default port - - # Ceph messenger protocol v2 - min: 3300 - max: 3300 - - # Ceph RADOS ports for OSDs, MDSes - min: 6800 - max: 7300 - - # # Ceph dashboard port HTTP (not recommended) - # - min: 7000 - # max: 7000 - # Ceph dashboard port HTTPS - min: 8443 - max: 8443 - - # Ceph mgr Prometheus Metrics - min: 9283 - max: 9283 - -# OLM: END CLUSTER POD SECURITY POLICY -# OLM: BEGIN POD SECURITY POLICY BINDINGS ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: 'psp:rook' -rules: -- apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - rook-privileged - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-ceph-system-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-default-psp - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: default - namespace: rook-ceph ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-osd-psp - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-mgr-psp - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: rook-ceph -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: rook-ceph - -# OLM: END CLUSTER POD SECURITY POLICY BINDINGS -# OLM: BEGIN CSI CEPHFS SERVICE ACCOUNT ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-plugin-sa - namespace: rook-ceph ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph - -# OLM: END CSI CEPHFS SERVICE ACCOUNT -# OLM: BEGIN CSI CEPHFS ROLE ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: rook-ceph - name: cephfs-external-provisioner-cfg -rules: -- apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - -# OLM: END CSI CEPHFS ROLE -# OLM: BEGIN CSI CEPHFS ROLEBINDING ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role-cfg - namespace: rook-ceph -subjects: -- kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph -roleRef: - kind: Role - name: cephfs-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io - -# OLM: END CSI CEPHFS ROLEBINDING -# OLM: BEGIN CSI CEPHFS CLUSTER ROLE ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin-rules - labels: - rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" -rules: -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" -rules: -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] -- apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] -- apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - -# OLM: END CSI CEPHFS CLUSTER ROLE -# OLM: BEGIN CSI CEPHFS CLUSTER ROLEBINDING ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: -- kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: rook-ceph ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: -- kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -subjects: -- kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: rook-ceph -roleRef: - kind: ClusterRole - name: cephfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-provisioner-role -subjects: -- kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph -roleRef: - kind: ClusterRole - name: cephfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io - -# OLM: END CSI CEPHFS CLUSTER ROLEBINDING -# OLM: BEGIN CSI RBD SERVICE ACCOUNT ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-plugin-sa - namespace: rook-ceph ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph - -# OLM: END CSI RBD SERVICE ACCOUNT -# OLM: BEGIN CSI RBD ROLE ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: rook-ceph - name: rbd-external-provisioner-cfg -rules: -- apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - -# OLM: END CSI RBD ROLE -# OLM: BEGIN CSI RBD ROLEBINDING ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role-cfg - namespace: rook-ceph -subjects: -- kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph -roleRef: - kind: Role - name: rbd-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io - -# OLM: END CSI RBD ROLEBINDING -# OLM: BEGIN CSI RBD CLUSTER ROLE ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin-rules - labels: - rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" -rules: -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" -rules: -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] -- apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] -- apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] -- apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - -# OLM: END CSI RBD CLUSTER ROLE -# OLM: BEGIN CSI RBD CLUSTER ROLEBINDING ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: -- kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: rook-ceph ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: -- kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -subjects: -- kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: rook-ceph -roleRef: - kind: ClusterRole - name: rbd-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role -subjects: -- kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph -roleRef: - kind: ClusterRole - name: rbd-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io -# OLM: END CSI RBD CLUSTER ROLEBINDING diff --git a/infrastructure/storage-provisioning/manifests/operator.yaml b/infrastructure/storage-provisioning/manifests/operator.yaml deleted file mode 100644 index 34633382b..000000000 --- a/infrastructure/storage-provisioning/manifests/operator.yaml +++ /dev/null @@ -1,313 +0,0 @@ -################################################################################################################# -# The deployment for the rook operator -# Contains the common settings for most Kubernetes deployments. -# For example, to create the rook-ceph cluster: -# kubectl create -f common.yaml -# kubectl create -f operator.yaml -# kubectl create -f cluster.yaml -# -# Also see other operator sample files for variations of operator.yaml: -# - operator-openshift.yaml: Common settings for running in OpenShift -################################################################################################################# -# Rook Ceph Operator Config -# Use this ConfigMap to override operator configurations -# Precedence will be given to this config in case -# Env Var also exists for the same -# -kind: ConfigMap -apiVersion: v1 -metadata: - name: rook-ceph-operator-config - # should be in the namespace of the operator - namespace: rook-ceph -data: - -# # (Optional) Ceph Provisioner NodeAffinity. -# CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" -# # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format. -# # CSI provisioner would be best to start on the same nodes as other ceph daemons. -# CSI_PROVISIONER_TOLERATIONS: | -# - effect: NoSchedule -# key: node-role.kubernetes.io/controlplane -# operator: Exists -# - effect: NoExecute -# key: node-role.kubernetes.io/etcd -# operator: Exists -# # (Optional) Ceph CSI plugin NodeAffinity. -# CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" -# # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format. -# # CSI plugins need to be started on all the nodes where the clients need to mount the storage. -# CSI_PLUGIN_TOLERATIONS: | -# - effect: NoSchedule -# key: node-role.kubernetes.io/controlplane -# operator: Exists -# - effect: NoExecute -# key: node-role.kubernetes.io/etcd -# operator: Exists ---- -# OLM: BEGIN OPERATOR DEPLOYMENT -apiVersion: apps/v1 -kind: Deployment -metadata: - name: rook-ceph-operator - namespace: rook-ceph - labels: - operator: rook - storage-backend: ceph -spec: - selector: - matchLabels: - app: rook-ceph-operator - replicas: 1 - template: - metadata: - labels: - app: rook-ceph-operator - spec: - serviceAccountName: rook-ceph-system - containers: - - name: rook-ceph-operator - image: rook/ceph:v1.2.5 - args: ["ceph", "operator"] - volumeMounts: - - mountPath: /var/lib/rook - name: rook-config - - mountPath: /etc/ceph - name: default-config-dir - env: - - # If the operator should only watch for cluster CRDs in the same namespace, set this to "true". - # If this is not set to true, the operator will watch for cluster CRDs in all namespaces. - name: ROOK_CURRENT_NAMESPACE_ONLY - value: "false" - - # To disable RBAC, uncomment the following: - # - name: RBAC_ENABLED - # value: "false" - # Rook Agent toleration. Will tolerate all taints with all keys. - # Choose between NoSchedule, PreferNoSchedule and NoExecute: - # - name: AGENT_TOLERATION - # value: "NoSchedule" - # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate - # - name: AGENT_TOLERATION_KEY - # value: "" - # (Optional) Rook Agent tolerations list. Put here list of taints you want to tolerate in YAML format. - # - name: AGENT_TOLERATIONS - # value: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/controlplane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - # (Optional) Rook Agent priority class name to set on the pod(s) - # - name: AGENT_PRIORITY_CLASS_NAME - # value: "" - # (Optional) Rook Agent NodeAffinity. - # - name: AGENT_NODE_AFFINITY - # value: "role=storage-node; storage=rook,ceph" - # (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`. - # `Any` uses Ceph admin credentials by default/fallback. - # For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and - # set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name. - # to the namespace in which the `mountSecret` Kubernetes secret namespace. - # - name: AGENT_MOUNT_SECURITY_MODE - # value: "Any" - # Set the path where the Rook agent can find the flex volumes - # - name: FLEXVOLUME_DIR_PATH - # value: "" - # Set the path where kernel modules can be found - # - name: LIB_MODULES_DIR_PATH - # value: "" - # Mount any extra directories into the agent container - # - name: AGENT_MOUNTS - # value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2" - # Rook Discover toleration. Will tolerate all taints with all keys. - # Choose between NoSchedule, PreferNoSchedule and NoExecute: - # - name: DISCOVER_TOLERATION - # value: "NoSchedule" - # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate - # - name: DISCOVER_TOLERATION_KEY - # value: "" - # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format. - # - name: DISCOVER_TOLERATIONS - # value: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/controlplane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - # (Optional) Rook Discover priority class name to set on the pod(s) - # - name: DISCOVER_PRIORITY_CLASS_NAME - # value: "" - # (Optional) Discover Agent NodeAffinity. - # - name: DISCOVER_AGENT_NODE_AFFINITY - # value: "role=storage-node; storage=rook, ceph" - # Allow rook to create multiple file systems. Note: This is considered - # an experimental feature in Ceph as described at - # http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster - # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027 - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS - value: "false" - - # The logging level for the operator: INFO | DEBUG - name: ROOK_LOG_LEVEL - value: "INFO" - - # The interval to check the health of the ceph cluster and update the status in the custom resource. - name: ROOK_CEPH_STATUS_CHECK_INTERVAL - value: "60s" - - # The interval to check if every mon is in the quorum. - name: ROOK_MON_HEALTHCHECK_INTERVAL - value: "45s" - - # The duration to wait before trying to failover or remove/replace the - # current mon with a new mon (useful for compensating flapping network). - name: ROOK_MON_OUT_TIMEOUT - value: "600s" - - # The duration between discovering devices in the rook-discover daemonset. - name: ROOK_DISCOVER_DEVICES_INTERVAL - value: "60m" - - # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods. - # This is necessary to workaround the anyuid issues when running on OpenShift. - # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED - value: "false" - - # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins). - # Disable it here if you have similar issues. - # For more details see https://github.com/rook/rook/issues/2417 - name: ROOK_ENABLE_SELINUX_RELABELING - value: "true" - - # In large volumes it will take some time to chown all the files. Disable it here if you have performance issues. - # For more details see https://github.com/rook/rook/issues/2254 - name: ROOK_ENABLE_FSGROUP - value: "true" - - # Disable automatic orchestration when new devices are discovered - name: ROOK_DISABLE_DEVICE_HOTPLUG - value: "false" - - # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+". - # In case of more than one regex, use comma to seperate between them. - # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" - # Add regex expression after putting a comma to blacklist a disk - # If value is empty, the default regex will be used. - name: DISCOVER_DAEMON_UDEV_BLACKLIST - value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" - - # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release - # in favor of the CSI driver. - name: ROOK_ENABLE_FLEX_DRIVER - value: "false" - - # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. - # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. - name: ROOK_ENABLE_DISCOVERY_DAEMON - value: "true" - - # Enable the default version of the CSI CephFS driver. To start another version of the CSI driver, see image properties below. - name: ROOK_CSI_ENABLE_CEPHFS - value: "true" - - # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below. - name: ROOK_CSI_ENABLE_RBD - value: "true" - - name: ROOK_CSI_ENABLE_GRPC_METRICS - value: "true" - - # Enable deployment of snapshotter container in ceph-csi provisioner. - name: CSI_ENABLE_SNAPSHOTTER - value: "true" - - # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs - # If you disable the kernel client, your application may be disrupted during upgrade. - # See the upgrade guide: https://rook.io/docs/rook/v1.2/ceph-upgrade.html - name: CSI_FORCE_CEPHFS_KERNEL_CLIENT - value: "true" - - # Time to wait until the node controller will move Rook pods to other - # nodes after detecting an unreachable node. - # Pods affected by this setting are: - # mgr, rbd, mds, rgw, nfs, PVC based mons and osds, and ceph toolbox - # The value used in this variable replaces the default value of 300 secs - # added automatically by k8s as Toleration for - # - # The total amount of time to reschedule Rook pods in healthy nodes - # before detecting a condition will be the sum of: - # --> node-monitor-grace-period: 40 seconds (k8s kube-controller-manager flag) - # --> ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS: 5 seconds - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS - # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. - # Default value is RollingUpdate. - #- name: CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY - # value: "OnDelete" - # CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. - # Default value is RollingUpdate. - #- name: CSI_RBD_PLUGIN_UPDATE_STRATEGY - # value: "OnDelete" - # The default version of CSI supported by Rook will be started. To change the version - # of the CSI driver to something other than what is officially supported, change - # these images to the desired release of the CSI driver. - #- name: ROOK_CSI_CEPH_IMAGE - # value: "quay.io/cephcsi/cephcsi:v2.0.0" - #- name: ROOK_CSI_REGISTRAR_IMAGE - # value: "quay.io/k8scsi/csi-node-driver-registrar:v1.2.0" - #- name: ROOK_CSI_RESIZER_IMAGE - # value: "quay.io/k8scsi/csi-resizer:v0.4.0" - #- name: ROOK_CSI_PROVISIONER_IMAGE - # value: "quay.io/k8scsi/csi-provisioner:v1.4.0" - #- name: ROOK_CSI_SNAPSHOTTER_IMAGE - # value: "quay.io/k8scsi/csi-snapshotter:v1.2.2" - #- name: ROOK_CSI_ATTACHER_IMAGE - # value: "quay.io/k8scsi/csi-attacher:v2.1.0" - # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. - #- name: ROOK_CSI_KUBELET_DIR_PATH - # value: "/var/lib/kubelet" - # (Optional) Ceph Provisioner NodeAffinity. - # - name: CSI_PROVISIONER_NODE_AFFINITY - # value: "role=storage-node; storage=rook, ceph" - # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format. - # CSI provisioner would be best to start on the same nodes as other ceph daemons. - # - name: CSI_PROVISIONER_TOLERATIONS - # value: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/controlplane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - # (Optional) Ceph CSI plugin NodeAffinity. - # - name: CSI_PLUGIN_NODE_AFFINITY - # value: "role=storage-node; storage=rook, ceph" - # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format. - # CSI plugins need to be started on all the nodes where the clients need to mount the storage. - # - name: CSI_PLUGIN_TOLERATIONS - # value: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/controlplane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - # Configure CSI cephfs grpc and liveness metrics port - #- name: CSI_CEPHFS_GRPC_METRICS_PORT - # value: "9091" - #- name: CSI_CEPHFS_LIVENESS_METRICS_PORT - # value: "9081" - # Configure CSI rbd grpc and liveness metrics port - #- name: CSI_RBD_GRPC_METRICS_PORT - # value: "9090" - #- name: CSI_RBD_LIVENESS_METRICS_PORT - # value: "9080" - - value: "5" - - # The name of the node to pass with the downward API - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - # The pod name to pass with the downward API - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - # The pod namespace to pass with the downward API - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - # Uncomment it to run rook operator on the host network - #hostNetwork: true - volumes: - - name: rook-config - emptyDir: {} - - name: default-config-dir - emptyDir: {} -# OLM: END OPERATOR DEPLOYMENT diff --git a/infrastructure/storage-provisioning/manifests/toolbox.yaml b/infrastructure/storage-provisioning/manifests/toolbox.yaml deleted file mode 100644 index 363328db8..000000000 --- a/infrastructure/storage-provisioning/manifests/toolbox.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: rook-ceph-tools - namespace: rook-ceph - labels: - app: rook-ceph-tools -spec: - replicas: 1 - selector: - matchLabels: - app: rook-ceph-tools - template: - metadata: - labels: - app: rook-ceph-tools - spec: - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: rook-ceph-tools - image: rook/ceph:v1.2.5 - command: ["/tini"] - args: ["-g", "--", "/usr/local/bin/toolbox.sh"] - imagePullPolicy: IfNotPresent - env: - - name: ROOK_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: admin-secret - volumeMounts: - - mountPath: /etc/ceph - name: ceph-config - - name: mon-endpoint-volume - mountPath: /etc/rook - volumes: - - name: mon-endpoint-volume - configMap: - name: rook-ceph-mon-endpoints - items: - - key: data - path: mon-endpoints - - name: ceph-config - emptyDir: {} - tolerations: - - key: "node.kubernetes.io/unreachable" - operator: "Exists" - effect: "NoExecute" - tolerationSeconds: 5