From 0b86ac7029916456a4b6e0454e0fc1c15a476729 Mon Sep 17 00:00:00 2001 From: Steven Barre Date: Mon, 8 Nov 2021 09:42:13 -0800 Subject: [PATCH] Crunchy DB Samples --- README.md | 7 +- .../high-availablility/NetworkPolicy.yaml | 42 ++++++ .../high-availablility/PostgresCluster.yaml | 66 +++++++++ crunchydb/high-availablility/README.md | 74 ++++++++++ crunchydb/high-availablility/RoleBinding.yaml | 32 ++++ crunchydb/monitoring/README.md | 137 ++++++++++++++++++ crunchydb/monitoring/grafana-config.yaml | 33 +++++ crunchydb/monitoring/grafana-netpol.yaml | 51 +++++++ crunchydb/monitoring/grafana-oauth.yaml | 36 +++++ crunchydb/monitoring/grafana-route.yaml | 15 ++ crunchydb/monitoring/kustomization.yaml | 71 +++++++++ 11 files changed, 561 insertions(+), 3 deletions(-) create mode 100644 crunchydb/high-availablility/NetworkPolicy.yaml create mode 100644 crunchydb/high-availablility/PostgresCluster.yaml create mode 100644 crunchydb/high-availablility/README.md create mode 100644 crunchydb/high-availablility/RoleBinding.yaml create mode 100644 crunchydb/monitoring/README.md create mode 100644 crunchydb/monitoring/grafana-config.yaml create mode 100644 crunchydb/monitoring/grafana-netpol.yaml create mode 100644 crunchydb/monitoring/grafana-oauth.yaml create mode 100644 crunchydb/monitoring/grafana-route.yaml create mode 100644 crunchydb/monitoring/kustomization.yaml diff --git a/README.md b/README.md index 98707cb..f923c37 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ You fill find useful docs on how to do things specific to the BCGov's implementa # Index -1. [Network Policy Demo Project](./labs/netpol-demo-project); -2. [Network Policy QuickStart](./labs/netpol-quickstart). - +1. [Network Policy Demo Project](./labs/netpol-demo-project) +2. [Network Policy QuickStart](./labs/netpol-quickstart) +3. [Crunchy DB High Availability Sample](./crunchydb/high-availablility/) +4. [Crunchy DB Monitoring QuickStart](./crunchydb/monitoring/) diff --git a/crunchydb/high-availablility/NetworkPolicy.yaml b/crunchydb/high-availablility/NetworkPolicy.yaml new file mode 100644 index 0000000..04a352d --- /dev/null +++ b/crunchydb/high-availablility/NetworkPolicy.yaml @@ -0,0 +1,42 @@ +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-crunchydb-ha + labels: + postgres-operator.crunchydata.com/cluster: hippo-ha +spec: + podSelector: + matchLabels: + postgres-operator.crunchydata.com/cluster: hippo-ha + ingress: + - from: + - podSelector: + matchLabels: + postgres-operator.crunchydata.com/cluster: hippo-ha + ports: + - protocol: TCP + port: 5432 + - protocol: TCP + port: 8008 + - protocol: TCP + port: 2022 +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-crunchydb-monitoring + labels: + postgres-operator.crunchydata.com/cluster: hippo-ha +spec: + podSelector: + matchLabels: + postgres-operator.crunchydata.com/cluster: hippo-ha + ingress: + - from: + - namespaceSelector: + name: be1c6b + environment: tools + ports: + - protocol: TCP + port: 9187 diff --git a/crunchydb/high-availablility/PostgresCluster.yaml b/crunchydb/high-availablility/PostgresCluster.yaml new file mode 100644 index 0000000..3395e9c --- /dev/null +++ b/crunchydb/high-availablility/PostgresCluster.yaml @@ -0,0 +1,66 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: hippo-ha +spec: + monitoring: + pgmonitor: + exporter: + image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi8-5.0.3-0 + image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:centos8-13.4-1 + postgresVersion: 13 + instances: + - name: pgha1 + replicas: 3 + dataVolumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + storageClassName: netapp-block-standard + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/cluster: hippo-ha + postgres-operator.crunchydata.com/instance-set: pgha1 + backups: + pgbackrest: + global: + repo1-retention-full: "2" + image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:centos8-2.35-0 + repos: + - name: repo1 + schedules: + full: 0 1 * * * + incremental: 0 */4 * * * + volume: + volumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + storageClassName: netapp-file-backup + proxy: + pgBouncer: + config: + global: + client_tls_sslmode: disable + image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:centos8-1.15-3 + replicas: 2 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/cluster: hippo-ha + postgres-operator.crunchydata.com/role: pgbouncer diff --git a/crunchydb/high-availablility/README.md b/crunchydb/high-availablility/README.md new file mode 100644 index 0000000..61552db --- /dev/null +++ b/crunchydb/high-availablility/README.md @@ -0,0 +1,74 @@ +# Sample HA CrunchyDB PostgresCluster + +This is a sample `PostgresCluster` adapted from the example at . + +You can find a full tutorial and docs on CrunchyDB at . + +In this example, my license plate is `be1c6b`. + +`NetworkPolicy.yaml` has a sample NetworkPolicy for allowing the cluster pods to talk to each other, and the one needed by Monitoring for access. + +`RoleBinding` has the Role and RoleBinding needed by the Monitoring stack. + +`PostgresCluster` has the sample PostgresCluster with 3 replicas, backups, and monitoring. + +## Monitoring Sidecar + +The monitoring side car is added like this. + +```yaml +spec: + monitoring: + pgmonitor: + exporter: + image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi8-5.0.3-0 +``` + +## Storage Class + +Postgres works best on block storage. + +```yaml +spec: + instances: + - name: pgha1 + dataVolumeClaimSpec: + storageClassName: netapp-block-standard +``` + +## Backups + +Keep 2 copies of full backups. + +```yaml +spec: + backups: + pgbackrest: + global: + repo1-retention-full: "2" +``` + +Set the cron schedule for full and incremental backups. + +```yaml +spec: + backups: + pgbackrest: + repos: + - name: repo1 + schedules: + full: 0 1 * * * + incremental: 0 */4 * * * +``` + +Set the backups to go to a PVC that is [backed up](https://developer.gov.bc.ca/OCP4-Backup-and-Restore). + +```yaml +spec: + backups: + pgbackrest: + repos: + - name: repo1 + volume: + storageClassName: netapp-file-backup +``` diff --git a/crunchydb/high-availablility/RoleBinding.yaml b/crunchydb/high-availablility/RoleBinding.yaml new file mode 100644 index 0000000..7f78615 --- /dev/null +++ b/crunchydb/high-availablility/RoleBinding.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: postgres-operator-monitoring + vendor: crunchydata + name: crunchy-monitoring +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + vendor: crunchydata + name: crunchy-monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: crunchy-monitoring +subjects: +- kind: ServiceAccount + name: prometheus-sa + namespace: be1c6b-tools diff --git a/crunchydb/monitoring/README.md b/crunchydb/monitoring/README.md new file mode 100644 index 0000000..f05983a --- /dev/null +++ b/crunchydb/monitoring/README.md @@ -0,0 +1,137 @@ +# Crunchy DB PGO Monitoring + +You can install one Crunchy Monitoring stack in a tools namespace and monitor all your PGO clusters. In these directions replace `LICENSE` with the license plate of your namespaces. + +## Install monitoring stack + +Use this to install the Crunchy DB Monitoring stack into your tools namespace to monitor all your PGO instances. + +Edit `kustomization.yaml` and set your tools namespace. + +```yaml +namespace: LICENSE-tools +``` + +Edit `grafana-oauth.yaml` and set `CHANGEME` to your tools namespace. + +```yaml +- '--openshift-sar={"namespace": "LICENSE-tools", "resource": "services", "verb": "get"}' +``` + +Build the manifest. Note that `oc apply -k` seems to not work here due to using an older version of Kustomize. + +```bash +kustomize build . -o crunchy-monitoring.yaml +``` + +Edit the output manifest `crunchy-monitoring.yaml`. + +First, find the ConfigMap for `alertmanager.yml` and set a better receiver for the alerts. + +```yaml + global: + smtp_smarthost: "apps.smtp.gov.bc.ca:25" + smtp_require_tls: false + smtp_from: 'Alertmanager ' +``` + +```yaml + receivers: + - name: 'default-receiver' + email_configs: + - to: 'your-team@gov.bc.ca' + send_resolved: true +``` + +Then find the ConfigMap for `prometheus.yml` and add the namespaces you want to monitor. + +```yaml + scrape_configs: + - job_name: 'crunchy-postgres-exporter' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - LICENSE-tools + - LICENSE-dev + - LICENSE-test + - LICENSE-prod +``` + +Finally, create all the items in the manifest. + +```bash +oc -n LICENSE-tools create -f crunchy-monitoring.yaml +``` + +## Add monitoring sidecar to your PostgresCluster + +Edit your `PostgresCluster` to include the monitoring container as a sidecar + +```yaml +spec: + monitoring: + pgmonitor: + exporter: + image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi8-5.0.3-0 +``` + +Add a `Role` and `RoleBinding` to allow access from your tools namespace where you installed the monitoring. + +```yaml +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: postgres-operator-monitoring + vendor: crunchydata + name: crunchy-monitoring +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + vendor: crunchydata + name: crunchy-monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: crunchy-monitoring +subjects: +- kind: ServiceAccount + name: prometheus-sa + namespace: LICENSE-tools +``` + +Add a NetworkPolicy to allow the monitor to connect to your pods. Make sure `hippo` is changed to your `PostgresCluster` name. + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-crunchydb-monitoring + labels: + postgres-operator.crunchydata.com/cluster: hippo +spec: + podSelector: + matchLabels: + postgres-operator.crunchydata.com/cluster: hippo + ingress: + - from: + - namespaceSelector: + name: LICENSE + environment: tools + ports: + - protocol: TCP + port: 9187 +``` diff --git a/crunchydb/monitoring/grafana-config.yaml b/crunchydb/monitoring/grafana-config.yaml new file mode 100644 index 0000000..ad21e26 --- /dev/null +++ b/crunchydb/monitoring/grafana-config.yaml @@ -0,0 +1,33 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: grafana-config +data: + grafana.ini: |+ + [auth] + disable_login_form = true + disable_signout_menu = true + oauth_auto_login = true + + [auth.anonymous] + enabled = false + + [auth.basic] + enabled = true + + [auth.proxy] + auto_sign_up = true + enabled = true + header_name = X-Forwarded-User + header_property = username + + [log] + level = warn + mode = console + + [paths] + data = /var/lib/grafana + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning/ diff --git a/crunchydb/monitoring/grafana-netpol.yaml b/crunchydb/monitoring/grafana-netpol.yaml new file mode 100644 index 0000000..9153f49 --- /dev/null +++ b/crunchydb/monitoring/grafana-netpol.yaml @@ -0,0 +1,51 @@ +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-grafana-route +spec: + podSelector: + matchLabels: + name: crunchy-grafana + ingress: + - from: + - namespaceSelector: + matchLabels: + network.openshift.io/policy-group: ingress + policyTypes: + - Ingress +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-grafana-to-prometheus +spec: + podSelector: + matchLabels: + name: crunchy-prometheus + ingress: + - from: + - podSelector: + matchLabels: + name: crunchy-grafana + ports: + - protocol: TCP + port: 9090 +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-prometheus-to-alertmanager +spec: + podSelector: + matchLabels: + name: crunchy-alertmanager + ingress: + - from: + - podSelector: + matchLabels: + name: crunchy-prometheus + ports: + - protocol: TCP + port: 9093 +--- diff --git a/crunchydb/monitoring/grafana-oauth.yaml b/crunchydb/monitoring/grafana-oauth.yaml new file mode 100644 index 0000000..bb138e9 --- /dev/null +++ b/crunchydb/monitoring/grafana-oauth.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crunchy-grafana +spec: + template: + spec: + containers: + - name: grafana + volumeMounts: + - name: grafana-config + mountPath: /etc/grafana/ + - name: grafana-proxy + ports: + - name: grafana-proxy + containerPort: 9091 + protocol: TCP + imagePullPolicy: IfNotPresent + image: image-registry.openshift-image-registry.svc:5000/openshift/oauth-proxy:v4.4 + args: + - '--provider=openshift' + - '--pass-basic-auth=false' + - '--https-address=' + - '--http-address=:9091' + - '--email-domain=*' + - '--upstream=http://localhost:3000' + - '--cookie-secret=asdf' + - '--openshift-service-account=grafana' + - '--skip-auth-regex=^/metrics' + - '--openshift-sar={"namespace": "CHANGEME", "resource": "services", "verb": "get"}' + volumes: + - name: grafana-config + configMap: + name: grafana-config + defaultMode: 420 diff --git a/crunchydb/monitoring/grafana-route.yaml b/crunchydb/monitoring/grafana-route.yaml new file mode 100644 index 0000000..7f3f5de --- /dev/null +++ b/crunchydb/monitoring/grafana-route.yaml @@ -0,0 +1,15 @@ +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: crunchy-grafana +spec: + to: + kind: Service + name: crunchy-grafana + weight: 100 + port: + targetPort: grafana-proxy + tls: + termination: edge + wildcardPolicy: None diff --git a/crunchydb/monitoring/kustomization.yaml b/crunchydb/monitoring/kustomization.yaml new file mode 100644 index 0000000..6f26404 --- /dev/null +++ b/crunchydb/monitoring/kustomization.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: LICENSE-tools + +resources: +- "grafana-config.yaml" +- "grafana-route.yaml" +- "grafana-netpol.yaml" +- "github.com/CrunchyData/postgres-operator-examples/kustomize/monitoring" + +patches: +- target: + version: v1 + kind: Deployment + labelSelector: "app.kubernetes.io/name=postgres-operator-monitoring" + patch: |- + - op: remove + path: "/spec/template/spec/securityContext/fsGroup" +- target: + version: v1 + kind: PersistentVolumeClaim + name: prometheusdata + patch: |- + - op: add + path: "/spec/storageClassName" + value: netapp-block-standard +- target: + version: v1 + group: rbac.authorization.k8s.io + kind: ClusterRole + labelSelector: "vendor=crunchydata" + patch: |- + - op: replace + path: "/kind" + value: "Role" +- target: + version: v1 + group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + labelSelector: "vendor=crunchydata" + patch: |- + - op: replace + path: "/kind" + value: "RoleBinding" + - op: replace + path: "/roleRef/kind" + value: "Role" +- target: + version: v1 + kind: Service + name: crunchy-grafana + patch: |- + - op: add + path: "/spec/ports/-" + value: + name: grafana-proxy + protocol: TCP + port: 9091 + targetPort: grafana-proxy +- target: + version: v1 + kind: ServiceAccount + name: grafana + patch: |- + - op: add + path: "/metadata/annotations/serviceaccounts.openshift.io~1oauth-redirectreference.primary" + value: >- + {"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"crunchy-grafana"}} +- path: grafana-oauth.yaml