From 3924bf05ad8237119d284676cc3678e36dc5ef58 Mon Sep 17 00:00:00 2001 From: Chris Wessels Date: Tue, 20 Sep 2022 20:47:09 -0400 Subject: [PATCH] fix(erigon): node p2p port init container issues --- charts/erigon/Chart.yaml | 2 +- charts/erigon/README.md | 17 ++++--- charts/erigon/templates/rbac.yaml | 49 +++++++++++++++++++ .../templates/stateful-node/statefulset.yaml | 47 ++++++++++-------- charts/erigon/values.yaml | 38 ++++++++++++-- 5 files changed, 121 insertions(+), 32 deletions(-) create mode 100644 charts/erigon/templates/rbac.yaml diff --git a/charts/erigon/Chart.yaml b/charts/erigon/Chart.yaml index 2f534655..8fc37641 100644 --- a/charts/erigon/Chart.yaml +++ b/charts/erigon/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.0 +version: 0.3.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/erigon/README.md b/charts/erigon/README.md index 693c1c58..2401157d 100644 --- a/charts/erigon/README.md +++ b/charts/erigon/README.md @@ -2,7 +2,7 @@ Deploy and scale [Erigon](https://github.com/ledgerwatch/erigon) inside Kubernetes with ease -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![Version: 0.3.0](https://img.shields.io/badge/Version-0.3.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2022.09.03](https://img.shields.io/badge/AppVersion-v2022.09.03-informational?style=flat-square) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![Version: 0.3.1](https://img.shields.io/badge/Version-0.3.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2022.09.03](https://img.shields.io/badge/AppVersion-v2022.09.03-informational?style=flat-square) ## Features @@ -132,6 +132,9 @@ We do not recommend that you upgrade the application by overriding `image.tag`. | prometheus.serviceMonitors.labels | | object | `{}` | | prometheus.serviceMonitors.relabelings | | list | `[]` | | prometheus.serviceMonitors.scrapeTimeout | | string | `nil` | + | rbac.clusterRules | Required ClusterRole rules | list | See `values.yaml` | + | rbac.create | Specifies whether RBAC resources are to be created | bool | `true` | + | rbac.rules | Required ClusterRole rules | list | See `values.yaml` | | rpcdaemon.affinity | | object | `{}` | | rpcdaemon.affinityPresets.antiAffinityByHostname | Configure anti-affinity rules to prevent multiple Erigon instances on the same host | bool | `true` | | rpcdaemon.autoscaling.enabled | Enable auto-scaling of the rpcdaemon Deployment. Be sure to set resources.requests for rpcdaemon. | bool | `false` | @@ -156,12 +159,14 @@ We do not recommend that you upgrade the application by overriding `image.tag`. | serviceAccount.name | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | string | `""` | | statefulNode.affinity | | object | `{}` | | statefulNode.affinityPresets.antiAffinityByHostname | Configure anti-affinity rules to prevent multiple Erigon instances on the same host | bool | `true` | - | statefulNode.extraArgs | Additional CLI arguments to pass to `erigon` | list | `[]` | + | statefulNode.extraArgs | Additional CLI arguments to pass to `erigon` | list | `["--test"]` | | statefulNode.fromSnapshot.enabled | Enable initialising Erigon state from a remote Snapshot | bool | `false` | | statefulNode.fromSnapshot.snapshotUrl | URL for snapshot to download and extract to bootstrap storage | string | `nil` | - | statefulNode.jwt | JWT for clients to authenticate with the Engine API. Specify either `existingSecret` OR `fromLiteral`. | object | `{"existingSecret":{"key":"jwt","name":"some-secret-name"},"fromLiteral":"xxxx"}` | - | statefulNode.jwt.existingSecret | Load the JWT from an existing Kubernetes Secret. Takes precedence over `fromLiteral` if set. | object | `{"key":"jwt","name":"some-secret-name"}` | - | statefulNode.jwt.fromLiteral | Use this literal value for the JWT | string | `"xxxx"` | + | statefulNode.jwt | JWT for clients to authenticate with the Engine API. Specify either `existingSecret` OR `fromLiteral`. | object | `{"existingSecret":{"key":"","name":""},"fromLiteral":""}` | + | statefulNode.jwt.existingSecret | Load the JWT from an existing Kubernetes Secret. Takes precedence over `fromLiteral` if set. | object | `{"key":"","name":""}` | + | statefulNode.jwt.existingSecret.key | Data key for the JWT in the Secret | string | `""` | + | statefulNode.jwt.existingSecret.name | Name of the Secret resource in the same namespace | string | `""` | + | statefulNode.jwt.fromLiteral | Use this literal value for the JWT | string | `""` | | statefulNode.nodeSelector | | object | `{}` | | statefulNode.p2pNodePort.enabled | Expose P2P port via NodePort | bool | `false` | | statefulNode.p2pNodePort.initContainer.image.pullPolicy | Container pull policy | string | `"IfNotPresent"` | @@ -172,7 +177,7 @@ We do not recommend that you upgrade the application by overriding `image.tag`. | statefulNode.podSecurityContext | Pod-wide security context | object | `{"fsGroup":101337,"runAsGroup":101337,"runAsNonRoot":true,"runAsUser":101337}` | | statefulNode.resources | | object | `{}` | | statefulNode.service.ports.grpc-erigon | Service Port to expose Erigon GRPC interface on | int | `9090` | - | statefulNode.service.ports.http-engineapi | Service Port to expose engineAPI interface on | int | `8550` | + | statefulNode.service.ports.http-engineapi | Service Port to expose engineAPI interface on | int | `8551` | | statefulNode.service.ports.http-jsonrpc | Service Port to expose JSON-RPC interface on | int | `8545` | | statefulNode.service.ports.http-metrics | Service Port to expose Prometheus metrics on | int | `6060` | | statefulNode.service.type | | string | `"ClusterIP"` | diff --git a/charts/erigon/templates/rbac.yaml b/charts/erigon/templates/rbac.yaml new file mode 100644 index 00000000..a3911ed7 --- /dev/null +++ b/charts/erigon/templates/rbac.yaml @@ -0,0 +1,49 @@ +{{- if .Values.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "erigon.serviceAccountName" . }} + labels: + {{- include "erigon.labels" . | nindent 4 }} +rules: +{{- toYaml .Values.rbac.rules | nindent 0 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "erigon.serviceAccountName" . }} + labels: + {{- include "erigon.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "erigon.serviceAccountName" . }} +subjects: + - kind: ServiceAccount + name: {{ include "erigon.serviceAccountName" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "erigon.serviceAccountName" . }} + labels: + {{- include "erigon.labels" . | nindent 4 }} +rules: +{{- toYaml .Values.rbac.clusterRules | nindent 0 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "erigon.serviceAccountName" . }} + labels: + {{- include "erigon.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "erigon.serviceAccountName" . }} +subjects: + - kind: ServiceAccount + name: {{ include "erigon.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/erigon/templates/stateful-node/statefulset.yaml b/charts/erigon/templates/stateful-node/statefulset.yaml index 3a0fcc61..cab90a40 100644 --- a/charts/erigon/templates/stateful-node/statefulset.yaml +++ b/charts/erigon/templates/stateful-node/statefulset.yaml @@ -81,8 +81,10 @@ spec: - sh - -c - > + set -ex; export EXTERNAL_PORT=$(kubectl get services -l "pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}'); export EXTERNAL_IP=$(kubectl get nodes "${NODE_NAME}" -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}'); + export EXTERNAL_IP=${EXTERNAL_IP:-$(curl --silent v4.ident.me)}; echo "EXTERNAL_PORT=$EXTERNAL_PORT" > /env/init-nodeport; echo "EXTERNAL_IP=$EXTERNAL_IP" >> /env/init-nodeport; cat /env/init-nodeport; @@ -145,32 +147,33 @@ spec: command: - sh - -ac - - > + - | {{- if $values.p2pNodePort.enabled }} . /env/init-nodeport; {{- end }} - exec erigon - --private.api.addr=0.0.0.0:9090 - --http - --http.addr=0.0.0.0 - --authrpc.addr=0.0.0.0 - {{- if $jwtEnabled }} - --authrpc.jwtsecret=/jwt/jwt.hex - {{- end }} - --healthcheck - --metrics - --metrics.addr=0.0.0.0 - --http.vhosts=* - --datadir=/storage + set -ex; + exec erigon \ + --datadir=/storage \ {{- if $values.p2pNodePort.enabled }} - --nat=extip:${EXTERNAL_IP} - --port=${EXTERNAL_PORT} + --nat=extip:${EXTERNAL_IP} \ + --port=${EXTERNAL_PORT} \ {{- else }} - --port={{ include "erigon.p2pPort" $values }} + --port={{ include "erigon.p2pPort" $values }} \ {{- end }} - {{- with $values.extraArgs }} - {{- . | nindent 16 }} + --private.api.addr=0.0.0.0:9090 \ + --http \ + --http.addr=0.0.0.0 \ + --http.vhosts=* \ + --authrpc.addr=0.0.0.0 \ + {{- if $jwtEnabled }} + --authrpc.jwtsecret=/jwt/jwt.hex \ {{- end }} + --metrics \ + --metrics.addr=0.0.0.0 \ + {{- range $arg := $values.extraArgs }} + {{- $arg | nindent 16 }} \ + {{- end }} + --healthcheck ports: - name: grpc-erigon containerPort: 9090 @@ -182,7 +185,7 @@ spec: containerPort: 8545 protocol: TCP - name: http-engineapi - containerPort: 8550 + containerPort: 8551 protocol: TCP - name: udp-p2p containerPort: {{ include "erigon.p2pPort" $values }} @@ -201,6 +204,10 @@ spec: - name: env-nodeport mountPath: /env {{- end }} + {{- if $jwtEnabled }} + - name: jwt + mountPath: /jwt + {{- end }} - name: storage mountPath: /storage - name: tmp diff --git a/charts/erigon/values.yaml b/charts/erigon/values.yaml index 2415263c..7dc25c61 100644 --- a/charts/erigon/values.yaml +++ b/charts/erigon/values.yaml @@ -24,6 +24,32 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: "" +rbac: + # -- Specifies whether RBAC resources are to be created + create: true + # -- Required ClusterRole rules + # @default -- See `values.yaml` + clusterRules: + # Required to obtain the nodes external IP + - apiGroups: [""] + resources: + - "nodes" + verbs: + - "get" + - "list" + - "watch" + # -- Required ClusterRole rules + # @default -- See `values.yaml` + rules: + # Required to get information about the serices nodePort. + - apiGroups: [""] + resources: + - "services" + verbs: + - "get" + - "list" + - "watch" + prometheus: serviceMonitors: # -- Enable monitoring by creating `ServiceMonitor` CRDs ([prometheus-operator](https://github.com/prometheus-operator/prometheus-operator)) @@ -46,13 +72,15 @@ statefulNode: jwt: # -- Load the JWT from an existing Kubernetes Secret. Takes precedence over `fromLiteral` if set. existingSecret: - name: some-secret-name - key: jwt + # -- Name of the Secret resource in the same namespace + name: "" + # -- Data key for the JWT in the Secret + key: "" # -- Use this literal value for the JWT - fromLiteral: xxxx + fromLiteral: "" # -- Additional CLI arguments to pass to `erigon` - extraArgs: [] + extraArgs: [--test] # -- [PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#persistentvolumeclaimspec-v1-core) for Erigon storage volumeClaimSpec: @@ -95,7 +123,7 @@ statefulNode: # -- Service Port to expose JSON-RPC interface on http-jsonrpc: 8545 # -- Service Port to expose engineAPI interface on - http-engineapi: 8550 + http-engineapi: 8551 # -- Service Port to expose Prometheus metrics on http-metrics: 6060