From 7e85355a8aff6b5e78dcd6c74b80fd99b16838ed Mon Sep 17 00:00:00 2001 From: Daniel Roberts Date: Thu, 29 Feb 2024 18:58:12 +0000 Subject: [PATCH] Added files for 3.1 accumulo --- helm/k8s-files/accumulo-compactor.yaml | 62 ++++++++++++ helm/k8s-files/accumulo-config.yaml | 121 +++++++++++++++++++++++ helm/k8s-files/accumulo-coordinator.yaml | 64 ++++++++++++ helm/k8s-files/accumulo-gc.yaml | 63 ++++++++++++ helm/k8s-files/accumulo-init.yaml | 61 ++++++++++++ helm/k8s-files/accumulo-manager.yaml | 63 ++++++++++++ helm/k8s-files/accumulo-monitor.yaml | 62 ++++++++++++ helm/k8s-files/accumulo-ns.yaml | 7 ++ helm/k8s-files/accumulo-secrets.yaml | 10 ++ helm/k8s-files/accumulo-sserver.yaml | 63 ++++++++++++ helm/k8s-files/accumulo-tserver.yaml | 63 ++++++++++++ helm/k8s-files/minio-dev.yaml | 44 +++++++++ helm/k8s-files/minio-svc.yaml | 18 ++++ 13 files changed, 701 insertions(+) create mode 100644 helm/k8s-files/accumulo-compactor.yaml create mode 100644 helm/k8s-files/accumulo-config.yaml create mode 100644 helm/k8s-files/accumulo-coordinator.yaml create mode 100644 helm/k8s-files/accumulo-gc.yaml create mode 100644 helm/k8s-files/accumulo-init.yaml create mode 100644 helm/k8s-files/accumulo-manager.yaml create mode 100644 helm/k8s-files/accumulo-monitor.yaml create mode 100644 helm/k8s-files/accumulo-ns.yaml create mode 100644 helm/k8s-files/accumulo-secrets.yaml create mode 100644 helm/k8s-files/accumulo-sserver.yaml create mode 100644 helm/k8s-files/accumulo-tserver.yaml create mode 100644 helm/k8s-files/minio-dev.yaml create mode 100644 helm/k8s-files/minio-svc.yaml diff --git a/helm/k8s-files/accumulo-compactor.yaml b/helm/k8s-files/accumulo-compactor.yaml new file mode 100644 index 0000000..e2ca412 --- /dev/null +++ b/helm/k8s-files/accumulo-compactor.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: accumulo-compactor + namespace: accumulo + labels: + app: accumulo-compactor +spec: + replicas: 1 + selector: + matchLabels: + app: accumulo-compactor + template: + metadata: + labels: + app: accumulo-compactor + spec: + containers: + - name: accumulo-compactor + image: accumulo-s3-fs:2.1.0 + imagePullPolicy: Never + command: ["/bin/bash", "-c"] + args: ["accumulo compactor -o compactor.queue=DCQ1 -o general.process.bind.addr=$(hostname -i)" ] + ports: + - containerPort: 9133 + resources: + requests: + cpu: 100m + memory: "2048Mi" + limits: + cpu: 200m + memory: "4096Mi" + env: + - name: ACCUMULO_JAVA_OPTS + value: "-Xmx2g" + volumeMounts: + - name: "client-config" + mountPath: "/opt/accumulo/conf/accumulo-client.properties" + subPath: "accumulo-client.properties" + - name: "config" + mountPath: "/opt/accumulo/conf/accumulo.properties" + subPath: "accumulo.properties" + - name: "logging" + mountPath: "/opt/accumulo/conf/log4j2-service.properties" + subPath: "log4j2-service.properties" + - name: "core-site" + mountPath: "/opt/accumulo/conf/core-site.xml" + subPath: "core-site.xml" + terminationGracePeriodSeconds: 300 # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal. + volumes: + - name: "client-config" + configMap: + name: "accumulo-client-properties" + - name: "config" + configMap: + name: "accumulo-properties" + - name: "logging" + configMap: + name: "accumulo-logging" + - name: "core-site" + configMap: + name: "core-site" diff --git a/helm/k8s-files/accumulo-config.yaml b/helm/k8s-files/accumulo-config.yaml new file mode 100644 index 0000000..0d5a02b --- /dev/null +++ b/helm/k8s-files/accumulo-config.yaml @@ -0,0 +1,121 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: accumulo-client-properties + namespace: accumulo +data: + accumulo-client.properties: | + auth.type=password + auth.principal=root + instance.name=s3test + instance.zookeepers=bitnami-zookeeper.zookeeper.svc.cluster.local:2181 + rpc.transport.idle.timeout=60s +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: accumulo-properties + namespace: accumulo +data: + accumulo.properties: | + general.rpc.timeout=240s + instance.secret=s3test + instance.zookeeper.host=bitnami-zookeeper.zookeeper.svc.cluster.local:2181 + manager.wal.closer.implementation=org.apache.accumulo.server.manager.recovery.NoOpLogCloser + table.durability=sync + tserver.memory.maps.native.enabled=false + tserver.walog.max.size=512M + tserver.port.search=true + ## For S3 you must define volumes for accumulo and its write ahead logs. Replace the defaule instance volumes at the top + ## with the example below. Make sure the volumes match the general.custom.volume.preferred.default, and + ## general.custom.volume.preferred.logger property values below + instance.volumes=s3a://accumulo/database,s3a://accumulo/wal + ## The default deployment will use a RandomVolumeChooser and HadoopLogCloser and neither of which work when deploying to + ## S3. Remove the comments below to use the volume chooser and log closer that are compatible with S3 + general.volume.chooser=org.apache.accumulo.core.spi.fs.PreferredVolumeChooser + ## Define the S3 objects to use for recording accumulo and write ahead log data + general.custom.volume.preferred.default=s3a://accumulo/database + general.custom.volume.preferred.logger=s3a://accumulo/wal +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: accumulo-logging + namespace: accumulo +data: + log4j2-service.properties: | + status = info + dest = err + name = AccumuloCompactorLoggingProperties + appender.console.type = Console + appender.console.name = STDOUT + appender.console.target = SYSTEM_OUT + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n + appender.console.filter.threshold.type = ThresholdFilter + appender.console.filter.threshold.level = debug + logger.hadoop.name = org.apache.hadoop + logger.zookeeper.level = warn + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = error + logger.accumulo.name = org.apache.accumulo + logger.accumulo.level = debug + rootLogger.level = debug + rootLogger.appenderRef.console.ref = STDOUT +--- +apiVersion: v1 +kind: ConfigMap +metadata: + creationTimestamp: null + name: core-site + namespace: accumulo +data: + core-site.xml: | + + + fs.defaultFS + s3a://accumulo/database + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + The implementation class of the S3A Filesystem + + + fs.s3a.path.style.access + true + + + fs.s3a.endpoint + http://minio.minio-dev.svc.cluster.local:9000 + + + fs.s3a.access.key + accumulo + + + fs.s3a.secret.key + changeme + + + fs.s3a.connection.ssl.enabled + false + + + +--- diff --git a/helm/k8s-files/accumulo-coordinator.yaml b/helm/k8s-files/accumulo-coordinator.yaml new file mode 100644 index 0000000..20694ad --- /dev/null +++ b/helm/k8s-files/accumulo-coordinator.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: accumulo-coordinator + namespace: accumulo + labels: + app: accumulo-coordinator +spec: + replicas: 1 + selector: + matchLabels: + app: accumulo-coordinator + template: + metadata: + labels: + app: accumulo-coordinator + spec: + containers: + - name: accumulo-coordinator + image: accumulo-s3-fs:2.1.0 + imagePullPolicy: Never + command: ["/bin/bash", "-c"] + args: ["accumulo compaction-coordinator -o general.process.bind.addr=$(hostname -i)"] + ports: + - containerPort: 9132 + hostPort: 9132 + protocol: TCP + resources: + requests: + cpu: 1 + memory: "2048Mi" + limits: + cpu: 2 + memory: "4096Mi" + env: + - name: ACCUMULO_JAVA_OPTS + value: "-Xmx2g" + volumeMounts: + - name: "client-config" + mountPath: "/opt/accumulo/conf/accumulo-client.properties" + subPath: "accumulo-client.properties" + - name: "config" + mountPath: "/opt/accumulo/conf/accumulo.properties" + subPath: "accumulo.properties" + - name: "logging" + mountPath: "/opt/accumulo/conf/log4j2-service.properties" + subPath: "log4j2-service.properties" + - name: "core-site" + mountPath: "/opt/accumulo/conf/core-site.xml" + subPath: "core-site.xml" + terminationGracePeriodSeconds: 300 # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal. + volumes: + - name: "client-config" + configMap: + name: "accumulo-client-properties" + - name: "config" + configMap: + name: "accumulo-properties" + - name: "logging" + configMap: + name: "accumulo-logging" + - name: "core-site" + configMap: + name: "core-site" diff --git a/helm/k8s-files/accumulo-gc.yaml b/helm/k8s-files/accumulo-gc.yaml new file mode 100644 index 0000000..0733311 --- /dev/null +++ b/helm/k8s-files/accumulo-gc.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: accumulo-gc + namespace: accumulo + labels: + app: accumulo-gc +spec: + replicas: 1 + selector: + matchLabels: + app: accumulo-gc + template: + metadata: + labels: + app: accumulo-gc + spec: + containers: + - name: accumulo-gc + image: accumulo-s3-fs:2.1.0 + imagePullPolicy: Never + command: ["/bin/bash", "-c"] + args: ["accumulo gc -o general.process.bind.addr=$(hostname -i)" ] + ports: + - containerPort: 9998 + resources: + requests: + cpu: 100m + memory: "2048Mi" + limits: + cpu: 200m + memory: "4096Mi" + env: + - name: ACCUMULO_JAVA_OPTS + value: "-Xmx2g" + volumeMounts: + - name: "client-config" + mountPath: "/opt/accumulo/conf/accumulo-client.properties" + subPath: "accumulo-client.properties" + - name: "config" + mountPath: "/opt/accumulo/conf/accumulo.properties" + subPath: "accumulo.properties" + - name: "logging" + mountPath: "/opt/accumulo/conf/log4j2-service.properties" + subPath: "log4j2-service.properties" + - name: "core-site" + mountPath: "/opt/accumulo/conf/core-site.xml" + subPath: "core-site.xml" + terminationGracePeriodSeconds: 300 # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal. + volumes: + - name: "client-config" + configMap: + name: "accumulo-client-properties" + - name: "config" + configMap: + name: "accumulo-properties" + - name: "logging" + configMap: + name: "accumulo-logging" + - name: "core-site" + configMap: + name: "core-site" + diff --git a/helm/k8s-files/accumulo-init.yaml b/helm/k8s-files/accumulo-init.yaml new file mode 100644 index 0000000..831c733 --- /dev/null +++ b/helm/k8s-files/accumulo-init.yaml @@ -0,0 +1,61 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: job + namespace: accumulo +spec: + template: + spec: + restartPolicy: Never + containers: + - name: accumulo-init + image: accumulo-s3-fs:2.1.0 + imagePullPolicy: Never + command: ["/bin/bash", "-c"] + args: ["accumulo init --clear-instance-name --instance-name ${ACCUMULO_INSTANCE} --user ${ACCUMULO_USERNAME} --password ${ACCUMULO_PASSWORD}" ] + env: + - name: ACCUMULO_USERNAME + valueFrom: + secretKeyRef: + name: accumulo-secrets + key: accumulo-username + optional: false + - name: ACCUMULO_PASSWORD + valueFrom: + secretKeyRef: + name: accumulo-secrets + key: accumulo-password + optional: false + - name: ACCUMULO_INSTANCE + valueFrom: + secretKeyRef: + name: accumulo-secrets + key: accumulo-instance-name + optional: false + volumeMounts: + - name: "client-config" + mountPath: "/opt/accumulo/conf/accumulo-client.properties" + subPath: "accumulo-client.properties" + - name: "config" + mountPath: "/opt/accumulo/conf/accumulo.properties" + subPath: "accumulo.properties" + - name: "logging" + mountPath: "/opt/accumulo/conf/log4j2-service.properties" + subPath: "log4j2-service.properties" + - name: "core-site" + mountPath: "/opt/accumulo/conf/core-site.xml" + subPath: "core-site.xml" + volumes: + - name: "client-config" + configMap: + name: "accumulo-client-properties" + - name: "config" + configMap: + name: "accumulo-properties" + - name: "logging" + configMap: + name: "accumulo-logging" + - name: "core-site" + configMap: + name: "core-site" + diff --git a/helm/k8s-files/accumulo-manager.yaml b/helm/k8s-files/accumulo-manager.yaml new file mode 100644 index 0000000..61ec244 --- /dev/null +++ b/helm/k8s-files/accumulo-manager.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: accumulo-manager + namespace: accumulo + labels: + app: accumulo-manager +spec: + replicas: 1 + selector: + matchLabels: + app: accumulo-manager + template: + metadata: + labels: + app: accumulo-manager + spec: + containers: + - name: accumulo-manager + image: accumulo-s3-fs:2.1.0 + imagePullPolicy: Never + command: ["/bin/bash", "-c"] + args: ["accumulo manager -o general.process.bind.addr=$(hostname -i)" ] + ports: + - containerPort: 9999 + resources: + requests: + cpu: 100m + memory: "2048Mi" + limits: + cpu: 200m + memory: "4096Mi" + env: + - name: ACCUMULO_JAVA_OPTS + value: "-Xmx2g" + volumeMounts: + - name: "client-config" + mountPath: "/opt/accumulo/conf/accumulo-client.properties" + subPath: "accumulo-client.properties" + - name: "config" + mountPath: "/opt/accumulo/conf/accumulo.properties" + subPath: "accumulo.properties" + - name: "logging" + mountPath: "/opt/accumulo/conf/log4j2-service.properties" + subPath: "log4j2-service.properties" + - name: "core-site" + mountPath: "/opt/accumulo/conf/core-site.xml" + subPath: "core-site.xml" + terminationGracePeriodSeconds: 300 # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal. + volumes: + - name: "client-config" + configMap: + name: "accumulo-client-properties" + - name: "config" + configMap: + name: "accumulo-properties" + - name: "logging" + configMap: + name: "accumulo-logging" + - name: "core-site" + configMap: + name: "core-site" + diff --git a/helm/k8s-files/accumulo-monitor.yaml b/helm/k8s-files/accumulo-monitor.yaml new file mode 100644 index 0000000..d0e5fbe --- /dev/null +++ b/helm/k8s-files/accumulo-monitor.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: accumulo-monitor + namespace: accumulo + labels: + app: accumulo-monitor +spec: + replicas: 1 + selector: + matchLabels: + app: accumulo-monitor + template: + metadata: + labels: + app: accumulo-monitor + spec: + containers: + - name: accumulo-monitor + image: accumulo-s3-fs:2.1.0 + imagePullPolicy: Never + command: ["/bin/bash", "-c"] + args: ["accumulo monitor -o general.process.bind.addr=0.0.0.0" ] + ports: + - containerPort: 9995 + resources: + requests: + cpu: 100m + memory: "2048Mi" + limits: + cpu: 200m + memory: "4096Mi" + env: + - name: ACCUMULO_JAVA_OPTS + value: "-Xmx2g" + volumeMounts: + - name: "client-config" + mountPath: "/opt/accumulo/conf/accumulo-client.properties" + subPath: "accumulo-client.properties" + - name: "config" + mountPath: "/opt/accumulo/conf/accumulo.properties" + subPath: "accumulo.properties" + - name: "logging" + mountPath: "/opt/accumulo/conf/log4j2-service.properties" + subPath: "log4j2-service.properties" + - name: "core-site" + mountPath: "/opt/accumulo/conf/core-site.xml" + subPath: "core-site.xml" + terminationGracePeriodSeconds: 300 # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal. + volumes: + - name: "client-config" + configMap: + name: "accumulo-client-properties" + - name: "config" + configMap: + name: "accumulo-properties" + - name: "logging" + configMap: + name: "accumulo-logging" + - name: "core-site" + configMap: + name: "core-site" diff --git a/helm/k8s-files/accumulo-ns.yaml b/helm/k8s-files/accumulo-ns.yaml new file mode 100644 index 0000000..71b7c05 --- /dev/null +++ b/helm/k8s-files/accumulo-ns.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: accumulo + labels: + name: accumulo + diff --git a/helm/k8s-files/accumulo-secrets.yaml b/helm/k8s-files/accumulo-secrets.yaml new file mode 100644 index 0000000..9e45ddb --- /dev/null +++ b/helm/k8s-files/accumulo-secrets.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: accumulo-secrets + namespace: accumulo +type: Opaque +stringData: + accumulo-username: root + accumulo-password: tercesrepus + accumulo-instance-name: s3test diff --git a/helm/k8s-files/accumulo-sserver.yaml b/helm/k8s-files/accumulo-sserver.yaml new file mode 100644 index 0000000..6600d8f --- /dev/null +++ b/helm/k8s-files/accumulo-sserver.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: accumulo-sserver + namespace: accumulo + labels: + app: accumulo-sserver +spec: + replicas: 1 + selector: + matchLabels: + app: accumulo-sserver + template: + metadata: + labels: + app: accumulo-sserver + spec: + containers: + - name: accumulo-sserver + image: accumulo-s3-fs:2.1.0 + imagePullPolicy: Never + command: ["/bin/bash", "-c"] + args: ["accumulo sserver -o general.process.bind.addr=$(hostname -i)" ] + ports: + - containerPort: 9996 + resources: + requests: + cpu: 100m + memory: "2048Mi" + limits: + cpu: 200m + memory: "4096Mi" + env: + - name: ACCUMULO_JAVA_OPTS + value: "-Xmx2g" + volumeMounts: + - name: "client-config" + mountPath: "/opt/accumulo/conf/accumulo-client.properties" + subPath: "accumulo-client.properties" + - name: "config" + mountPath: "/opt/accumulo/conf/accumulo.properties" + subPath: "accumulo.properties" + - name: "logging" + mountPath: "/opt/accumulo/conf/log4j2-service.properties" + subPath: "log4j2-service.properties" + - name: "core-site" + mountPath: "/opt/accumulo/conf/core-site.xml" + subPath: "core-site.xml" + terminationGracePeriodSeconds: 300 # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal. + volumes: + - name: "client-config" + configMap: + name: "accumulo-client-properties" + - name: "config" + configMap: + name: "accumulo-properties" + - name: "logging" + configMap: + name: "accumulo-logging" + - name: "core-site" + configMap: + name: "core-site" + diff --git a/helm/k8s-files/accumulo-tserver.yaml b/helm/k8s-files/accumulo-tserver.yaml new file mode 100644 index 0000000..3bf10a5 --- /dev/null +++ b/helm/k8s-files/accumulo-tserver.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: accumulo-tserver + namespace: accumulo + labels: + app: accumulo-tserver +spec: + replicas: 1 + selector: + matchLabels: + app: accumulo-tserver + template: + metadata: + labels: + app: accumulo-tserver + spec: + containers: + - name: accumulo-tserver + image: accumulo-s3-fs:2.1.0 + imagePullPolicy: Never + command: ["/bin/bash", "-c"] + args: ["accumulo tserver -o general.process.bind.addr=$(hostname -i)" ] + ports: + - containerPort: 9997 + resources: + requests: + cpu: 100m + memory: "2048Mi" + limits: + cpu: 200m + memory: "4096Mi" + env: + - name: ACCUMULO_JAVA_OPTS + value: "-Xmx2g" + volumeMounts: + - name: "client-config" + mountPath: "/opt/accumulo/conf/accumulo-client.properties" + subPath: "accumulo-client.properties" + - name: "config" + mountPath: "/opt/accumulo/conf/accumulo.properties" + subPath: "accumulo.properties" + - name: "logging" + mountPath: "/opt/accumulo/conf/log4j2-service.properties" + subPath: "log4j2-service.properties" + - name: "core-site" + mountPath: "/opt/accumulo/conf/core-site.xml" + subPath: "core-site.xml" + terminationGracePeriodSeconds: 300 # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal. + volumes: + - name: "client-config" + configMap: + name: "accumulo-client-properties" + - name: "config" + configMap: + name: "accumulo-properties" + - name: "logging" + configMap: + name: "accumulo-logging" + - name: "core-site" + configMap: + name: "core-site" + diff --git a/helm/k8s-files/minio-dev.yaml b/helm/k8s-files/minio-dev.yaml new file mode 100644 index 0000000..1e83283 --- /dev/null +++ b/helm/k8s-files/minio-dev.yaml @@ -0,0 +1,44 @@ +# Deploys a new Namespace for the MinIO Pod +apiVersion: v1 +kind: Namespace +metadata: + name: minio-dev # Change this value if you want a different namespace name + labels: + name: minio-dev # Change this value to match metadata.name +--- +# Deploys a new MinIO Pod into the metadata.namespace Kubernetes namespace +# +# The `spec.containers[0].args` contains the command run on the pod +# The `/data` directory corresponds to the `spec.containers[0].volumeMounts[0].mountPath` +# That mount path corresponds to a Kubernetes HostPath which binds `/data` to a local drive or volume on the worker node where the pod runs +# +apiVersion: v1 +kind: Pod +metadata: + labels: + app: minio + name: minio + namespace: minio-dev # Change this value to match the namespace metadata.name +spec: + containers: + - name: minio + image: quay.io/minio/minio:latest + command: + - /bin/bash + - -c + args: + - minio server /data --console-address :9090 + ports: + - containerPort: 9000 + - containerPort: 9090 + volumeMounts: + - mountPath: /data + name: localvolume # Corresponds to the `spec.volumes` Persistent Volume +# nodeSelector: +# kubernetes.io/hostname: # Specify a node label associated to the Worker Node on which you want to deploy the pod. + volumes: + - name: localvolume + hostPath: # MinIO generally recommends using locally-attached volumes + path: /home/rsingh2/git/accumulo-k8s/export # Specify a path to a local drive or volume on the Kubernetes worker node + + type: DirectoryOrCreate # The path to the last directory must exist diff --git a/helm/k8s-files/minio-svc.yaml b/helm/k8s-files/minio-svc.yaml new file mode 100644 index 0000000..6e3fd54 --- /dev/null +++ b/helm/k8s-files/minio-svc.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: minio-dev +spec: + type: ClusterIP + selector: + app: minio + ports: + - name: api + port: 9000 + protocol: TCP + targetPort: 9000 + - name: console + port: 9090 + protocol: TCP + targetPort: 9090