fix(erigon): node p2p port init container issues

graphops · Sep 21, 2022 · 3924bf0 · 3924bf0
1 parent ef37082
commit 3924bf0
Show file tree

Hide file tree

Showing 5 changed files with 121 additions and 32 deletions.
diff --git a/charts/erigon/Chart.yaml b/charts/erigon/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.0
+version: 0.3.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/erigon/README.md b/charts/erigon/README.md
@@ -2,7 +2,7 @@
 
 Deploy and scale [Erigon](https://github.com/ledgerwatch/erigon) inside Kubernetes with ease
 
-[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![Version: 0.3.0](https://img.shields.io/badge/Version-0.3.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2022.09.03](https://img.shields.io/badge/AppVersion-v2022.09.03-informational?style=flat-square)
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![Version: 0.3.1](https://img.shields.io/badge/Version-0.3.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2022.09.03](https://img.shields.io/badge/AppVersion-v2022.09.03-informational?style=flat-square)
 
 ## Features
 
@@ -132,6 +132,9 @@ We do not recommend that you upgrade the application by overriding `image.tag`.
  | prometheus.serviceMonitors.labels |  | object | `{}` |
  | prometheus.serviceMonitors.relabelings |  | list | `[]` |
  | prometheus.serviceMonitors.scrapeTimeout |  | string | `nil` |
+ | rbac.clusterRules | Required ClusterRole rules | list | See `values.yaml` |
+ | rbac.create | Specifies whether RBAC resources are to be created | bool | `true` |
+ | rbac.rules | Required ClusterRole rules | list | See `values.yaml` |
  | rpcdaemon.affinity |  | object | `{}` |
  | rpcdaemon.affinityPresets.antiAffinityByHostname | Configure anti-affinity rules to prevent multiple Erigon instances on the same host | bool | `true` |
  | rpcdaemon.autoscaling.enabled | Enable auto-scaling of the rpcdaemon Deployment. Be sure to set resources.requests for rpcdaemon. | bool | `false` |
@@ -156,12 +159,14 @@ We do not recommend that you upgrade the application by overriding `image.tag`.
  | serviceAccount.name | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | string | `""` |
  | statefulNode.affinity |  | object | `{}` |
  | statefulNode.affinityPresets.antiAffinityByHostname | Configure anti-affinity rules to prevent multiple Erigon instances on the same host | bool | `true` |
- | statefulNode.extraArgs | Additional CLI arguments to pass to `erigon` | list | `[]` |
+ | statefulNode.extraArgs | Additional CLI arguments to pass to `erigon` | list | `["--test"]` |
  | statefulNode.fromSnapshot.enabled | Enable initialising Erigon state from a remote Snapshot | bool | `false` |
  | statefulNode.fromSnapshot.snapshotUrl | URL for snapshot to download and extract to bootstrap storage | string | `nil` |
- | statefulNode.jwt | JWT for clients to authenticate with the Engine API. Specify either `existingSecret` OR `fromLiteral`. | object | `{"existingSecret":{"key":"jwt","name":"some-secret-name"},"fromLiteral":"xxxx"}` |
- | statefulNode.jwt.existingSecret | Load the JWT from an existing Kubernetes Secret. Takes precedence over `fromLiteral` if set. | object | `{"key":"jwt","name":"some-secret-name"}` |
- | statefulNode.jwt.fromLiteral | Use this literal value for the JWT | string | `"xxxx"` |
+ | statefulNode.jwt | JWT for clients to authenticate with the Engine API. Specify either `existingSecret` OR `fromLiteral`. | object | `{"existingSecret":{"key":"","name":""},"fromLiteral":""}` |
+ | statefulNode.jwt.existingSecret | Load the JWT from an existing Kubernetes Secret. Takes precedence over `fromLiteral` if set. | object | `{"key":"","name":""}` |
+ | statefulNode.jwt.existingSecret.key | Data key for the JWT in the Secret | string | `""` |
+ | statefulNode.jwt.existingSecret.name | Name of the Secret resource in the same namespace | string | `""` |
+ | statefulNode.jwt.fromLiteral | Use this literal value for the JWT | string | `""` |
  | statefulNode.nodeSelector |  | object | `{}` |
  | statefulNode.p2pNodePort.enabled | Expose P2P port via NodePort | bool | `false` |
  | statefulNode.p2pNodePort.initContainer.image.pullPolicy | Container pull policy | string | `"IfNotPresent"` |
@@ -172,7 +177,7 @@ We do not recommend that you upgrade the application by overriding `image.tag`.
  | statefulNode.podSecurityContext | Pod-wide security context | object | `{"fsGroup":101337,"runAsGroup":101337,"runAsNonRoot":true,"runAsUser":101337}` |
  | statefulNode.resources |  | object | `{}` |
  | statefulNode.service.ports.grpc-erigon | Service Port to expose Erigon GRPC interface on | int | `9090` |
- | statefulNode.service.ports.http-engineapi | Service Port to expose engineAPI interface on | int | `8550` |
+ | statefulNode.service.ports.http-engineapi | Service Port to expose engineAPI interface on | int | `8551` |
  | statefulNode.service.ports.http-jsonrpc | Service Port to expose JSON-RPC interface on | int | `8545` |
  | statefulNode.service.ports.http-metrics | Service Port to expose Prometheus metrics on | int | `6060` |
  | statefulNode.service.type |  | string | `"ClusterIP"` |

diff --git a/charts/erigon/templates/rbac.yaml b/charts/erigon/templates/rbac.yaml
@@ -0,0 +1,49 @@
+{{- if .Values.rbac.create }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "erigon.serviceAccountName" . }}
+  labels:
+    {{- include "erigon.labels" . | nindent 4 }}
+rules:
+{{- toYaml .Values.rbac.rules | nindent 0 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "erigon.serviceAccountName" . }}
+  labels:
+    {{- include "erigon.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "erigon.serviceAccountName" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "erigon.serviceAccountName" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "erigon.serviceAccountName" . }}
+  labels:
+    {{- include "erigon.labels" . | nindent 4 }}
+rules:
+{{- toYaml .Values.rbac.clusterRules | nindent 0 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "erigon.serviceAccountName" . }}
+  labels:
+    {{- include "erigon.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "erigon.serviceAccountName" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "erigon.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/charts/erigon/templates/stateful-node/statefulset.yaml b/charts/erigon/templates/stateful-node/statefulset.yaml
@@ -81,8 +81,10 @@ spec:
             - sh
             - -c
             - >
+              set -ex;
               export EXTERNAL_PORT=$(kubectl get services -l "pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}');
               export EXTERNAL_IP=$(kubectl get nodes "${NODE_NAME}" -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}');
+              export EXTERNAL_IP=${EXTERNAL_IP:-$(curl --silent v4.ident.me)};
               echo "EXTERNAL_PORT=$EXTERNAL_PORT" >  /env/init-nodeport;
               echo "EXTERNAL_IP=$EXTERNAL_IP"     >> /env/init-nodeport;
               cat /env/init-nodeport;
@@ -145,32 +147,33 @@ spec:
           command:
             - sh
             - -ac
-            - >
+            - |
             {{- if $values.p2pNodePort.enabled }}
               . /env/init-nodeport;
             {{- end }}
-              exec erigon
-                --private.api.addr=0.0.0.0:9090
-                --http
-                --http.addr=0.0.0.0
-                --authrpc.addr=0.0.0.0
-            {{- if $jwtEnabled }}
-                --authrpc.jwtsecret=/jwt/jwt.hex
-            {{- end }}
-                --healthcheck
-                --metrics
-                --metrics.addr=0.0.0.0
-                --http.vhosts=*
-                --datadir=/storage
+              set -ex;
+              exec erigon \
+                --datadir=/storage \
               {{- if $values.p2pNodePort.enabled }}
-                --nat=extip:${EXTERNAL_IP}
-                --port=${EXTERNAL_PORT}
+                --nat=extip:${EXTERNAL_IP} \
+                --port=${EXTERNAL_PORT} \
               {{- else }}
-                --port={{ include "erigon.p2pPort" $values }}
+                --port={{ include "erigon.p2pPort" $values }} \
               {{- end }}
-              {{- with $values.extraArgs }}
-                {{- . | nindent 16 }}
+                --private.api.addr=0.0.0.0:9090 \
+                --http \
+                --http.addr=0.0.0.0 \
+                --http.vhosts=* \
+                --authrpc.addr=0.0.0.0 \
+              {{- if $jwtEnabled }}
+                --authrpc.jwtsecret=/jwt/jwt.hex \
               {{- end }}
+                --metrics \
+                --metrics.addr=0.0.0.0 \
+              {{- range $arg := $values.extraArgs }}
+                {{- $arg | nindent 16 }} \
+              {{- end }}
+                --healthcheck
           ports:
             - name: grpc-erigon
               containerPort: 9090
@@ -182,7 +185,7 @@ spec:
               containerPort: 8545
               protocol: TCP
             - name: http-engineapi
-              containerPort: 8550
+              containerPort: 8551
               protocol: TCP
             - name: udp-p2p
               containerPort: {{ include "erigon.p2pPort" $values }}
@@ -201,6 +204,10 @@ spec:
             - name: env-nodeport
               mountPath: /env
             {{- end }}
+            {{- if $jwtEnabled }}
+            - name: jwt
+              mountPath: /jwt
+            {{- end }}
             - name: storage
               mountPath: /storage
             - name: tmp

diff --git a/charts/erigon/values.yaml b/charts/erigon/values.yaml
@@ -24,6 +24,32 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name: ""
 
+rbac:
+  # -- Specifies whether RBAC resources are to be created
+  create: true
+  # -- Required ClusterRole rules
+  # @default -- See `values.yaml`
+  clusterRules:
+     # Required to obtain the nodes external IP
+    - apiGroups: [""]
+      resources:
+      - "nodes"
+      verbs:
+      - "get"
+      - "list"
+      - "watch"
+  # -- Required ClusterRole rules
+  # @default -- See `values.yaml`
+  rules:
+    # Required to get information about the serices nodePort.
+    - apiGroups: [""]
+      resources:
+      - "services"
+      verbs:
+      - "get"
+      - "list"
+      - "watch"
+
 prometheus:
   serviceMonitors:
     # -- Enable monitoring by creating `ServiceMonitor` CRDs ([prometheus-operator](https://github.com/prometheus-operator/prometheus-operator))
@@ -46,13 +72,15 @@ statefulNode:
   jwt:
     # -- Load the JWT from an existing Kubernetes Secret. Takes precedence over `fromLiteral` if set.
     existingSecret:
-      name: some-secret-name
-      key: jwt
+      # -- Name of the Secret resource in the same namespace
+      name: ""
+      # -- Data key for the JWT in the Secret
+      key: ""
     # -- Use this literal value for the JWT
-    fromLiteral: xxxx
+    fromLiteral: ""
 
   # -- Additional CLI arguments to pass to `erigon`
-  extraArgs: []
+  extraArgs: [--test]
 
   # -- [PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#persistentvolumeclaimspec-v1-core) for Erigon storage
   volumeClaimSpec:
@@ -95,7 +123,7 @@ statefulNode:
       # -- Service Port to expose JSON-RPC interface on
       http-jsonrpc: 8545
       # -- Service Port to expose engineAPI interface on
-      http-engineapi: 8550
+      http-engineapi: 8551
       # -- Service Port to expose Prometheus metrics on
       http-metrics: 6060