-
Notifications
You must be signed in to change notification settings - Fork 3
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Use values from secrets env #67
Merged
Merged
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
167 changes: 167 additions & 0 deletions
167
charts/neon-storage-controller/scripts/register-storage-controller.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,167 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| import os | ||
| import sys | ||
| import json | ||
| import logging | ||
| import urllib.request | ||
| import urllib.error | ||
|
|
||
| # region_id different in console/cplan with prefix aws-<region> | ||
| REGION = os.environ["REGION_ID"] | ||
| # ZONE env will be autogenerated from init container | ||
| ZONE = os.environ["ZONE"] | ||
| HOST = os.environ["HOST"] | ||
| PORT = os.getenv("PORT", 50051) | ||
|
|
||
| GLOBAL_CPLANE_JWT_TOKEN = os.environ["JWT_TOKEN"] | ||
| LOCAL_CPLANE_JWT_TOKEN = os.environ["CONTROL_PLANE_JWT_TOKEN"] | ||
| CONSOLE_API_KEY = os.environ["CONSOLE_API_KEY"] | ||
|
|
||
| # To register new pageservers | ||
| URL_PATH = "management/api/v2/pageservers" | ||
| # To get pageservers | ||
| ADMIN_URL_PATH = "api/v1/admin/pageservers" | ||
|
|
||
| GLOBAL_CPLANE_URL = f"{os.environ['GLOBAL_CPLANE_URL'].strip('/')}/{URL_PATH}" | ||
| LOCAL_CPLANE_URL = f"{os.environ['LOCAL_CPLANE_URL'].strip('/')}/{URL_PATH}" | ||
| CONSOLE_URL = f"{os.environ['CONSOLE_URL']}/{ADMIN_URL_PATH}" | ||
|
|
||
| PAYLOAD = dict( | ||
| host=HOST, | ||
| region_id=REGION, | ||
| port=6400, | ||
| disk_size=0, | ||
| instance_id=HOST, | ||
| http_host=HOST, | ||
| http_port=int(PORT), | ||
| availability_zone_id=ZONE, | ||
| instance_type="", | ||
| register_reason="Storage Controller Virtual Pageserver", | ||
| active=False, | ||
| is_storage_controller=True, | ||
| ) | ||
|
|
||
|
|
||
| def get_data(url, token, host=None): | ||
| if host is not None: | ||
| url = f"{url}/{host}" | ||
| headers = { | ||
| "Authorization": f"Bearer {token}", | ||
| "Accept": "application/json", | ||
| "Content-Type": "application/json", | ||
| } | ||
| # Check if the server is already registered | ||
| req = urllib.request.Request(url=url, headers=headers, method="GET") | ||
| try: | ||
| with urllib.request.urlopen(req) as response: | ||
| if response.getcode() == 200: | ||
| return json.loads(response.read()) | ||
| except urllib.error.URLError: | ||
| pass | ||
| return {} | ||
|
|
||
|
|
||
| def get_pageserver_id(url, token): | ||
| data = get_data(url, token, HOST) | ||
| if "node_id" in data: | ||
| return data["node_id"] | ||
|
|
||
|
|
||
| def get_pageserver_version(): | ||
| data = get_data(CONSOLE_URL, CONSOLE_API_KEY) | ||
| if "data" not in data: | ||
| return -1 | ||
| for pageserver in data["data"]: | ||
| region_id = pageserver["region_id"] | ||
| if region_id == REGION or region_id == f"{REGION}-new": | ||
| return pageserver["version"] | ||
| return -1 | ||
|
|
||
|
|
||
| def register(url, token, payload): | ||
| headers = { | ||
| "Authorization": f"Bearer {token}", | ||
| "Accept": "application/json", | ||
| "Content-Type": "application/json", | ||
| "User-Agent": "Python script 1.0", | ||
| } | ||
| data = str(json.dumps(payload)).encode() | ||
| req = urllib.request.Request( | ||
| url=url, | ||
| data=data, | ||
| headers=headers, | ||
| method="POST", | ||
| ) | ||
| with urllib.request.urlopen(req) as resp: | ||
| response = json.loads(resp.read()) | ||
| log.info(response) | ||
| if "node_id" in response: | ||
| return response["node_id"] | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| logging.basicConfig( | ||
| style="{", | ||
| format="{asctime} {levelname:8} {name}:{lineno} {message}", | ||
| level=logging.INFO, | ||
| ) | ||
|
|
||
| log = logging.getLogger() | ||
|
|
||
| log.info( | ||
| json.dumps( | ||
| dict( | ||
| GLOBAL_CPLANE_URL=GLOBAL_CPLANE_URL, | ||
| LOCAL_CPLANE_URL=LOCAL_CPLANE_URL, | ||
| CONSOLE_URL=CONSOLE_URL, | ||
| **PAYLOAD, | ||
| ), | ||
| indent=4, | ||
| ) | ||
| ) | ||
|
|
||
| log.info("get version from existing deployed pageserver") | ||
| version = get_pageserver_version() | ||
|
|
||
| if version == -1: | ||
| log.error(f"Unable to find pageserver version from {CONSOLE_URL}") | ||
| sys.exit(1) | ||
|
|
||
| log.info(f"found latest version={version} for region={REGION}") | ||
| PAYLOAD.update(dict(version=version)) | ||
|
|
||
| log.info("check if pageserver already registered or not in console") | ||
| node_id_in_console = get_pageserver_id(GLOBAL_CPLANE_URL, GLOBAL_CPLANE_JWT_TOKEN) | ||
|
|
||
| if node_id_in_console is None: | ||
| log.info("Registering storage controller in console") | ||
| node_id_in_console = register( | ||
| GLOBAL_CPLANE_URL, GLOBAL_CPLANE_JWT_TOKEN, PAYLOAD | ||
| ) | ||
| log.info( | ||
| f"Storage controller registered in console with node_id \ | ||
| {node_id_in_console}" | ||
| ) | ||
| else: | ||
| log.info( | ||
| f"Storage controller already registered in console with node_id \ | ||
| {node_id_in_console}" | ||
| ) | ||
|
|
||
| log.info("check if pageserver already registered or not in cplane") | ||
| node_id_in_cplane = get_pageserver_id(LOCAL_CPLANE_URL, LOCAL_CPLANE_JWT_TOKEN) | ||
|
|
||
| if node_id_in_cplane is None: | ||
| PAYLOAD.update(dict(node_id=str(node_id_in_console))) | ||
| log.info("Registering storage controller in cplane") | ||
| node_id_in_cplane = register(LOCAL_CPLANE_URL, LOCAL_CPLANE_JWT_TOKEN, PAYLOAD) | ||
| log.info( | ||
| f"Storage controller registered in cplane with node_id \ | ||
| {node_id_in_cplane}" | ||
| ) | ||
| else: | ||
| log.info( | ||
| f"Storage controller already registered in cplane with node_id \ | ||
| {node_id_in_cplane}" | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
charts/neon-storage-controller/templates/node-describe-rbac.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| {{- if .Values.registerControlPlane.enable -}} | ||
| apiVersion: v1 | ||
| kind: ServiceAccount | ||
| metadata: | ||
| name: {{ include "neon-storage-controller.fullname" . }}-node-describe | ||
| labels: | ||
| {{- include "neon-storage-controller.labels" . | nindent 4 }} | ||
| {{- with .Values.serviceAccount.annotations }} | ||
| annotations: | ||
| {{- toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| --- | ||
| apiVersion: rbac.authorization.k8s.io/v1 | ||
| kind: ClusterRole | ||
| metadata: | ||
| name: {{ include "neon-storage-controller.fullname" . }}-node-reader | ||
| labels: | ||
| {{- include "neon-storage-controller.labels" . | nindent 4 }} | ||
| rules: | ||
| - apiGroups: | ||
| - "" | ||
| resources: | ||
| - nodes | ||
| verbs: | ||
| - get | ||
| --- | ||
| kind: ClusterRoleBinding | ||
| apiVersion: rbac.authorization.k8s.io/v1 | ||
| metadata: | ||
| name: {{ include "neon-storage-controller.fullname" . }}-node-reader | ||
| labels: | ||
| {{- include "neon-storage-controller.labels" . | nindent 4 }} | ||
| roleRef: | ||
| apiGroup: rbac.authorization.k8s.io | ||
| kind: ClusterRole | ||
| name: {{ include "neon-storage-controller.fullname" . }}-node-reader | ||
| subjects: | ||
| - kind: ServiceAccount | ||
| name: {{ include "neon-storage-controller.fullname" . }}-node-describe | ||
| namespace: {{ .Release.Namespace }} | ||
| {{- end }} |
84 changes: 84 additions & 0 deletions
84
charts/neon-storage-controller/templates/post-install-job.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| {{- if .Values.registerControlPlane.enable -}} | ||
| {{ if (empty .Values.registerControlPlane.region_id ) }} | ||
| {{- fail (printf "Value for .Values.registerControlPlane.region_id is empty") }} | ||
| {{- end }} | ||
| {{ if (empty .Values.registerControlPlane.global_cplane_url) }} | ||
| {{- fail (printf "Value for .Values.registerControlPlane.global_cplane_url is empty") }} | ||
| {{- end }} | ||
| {{ if (empty .Values.registerControlPlane.local_cplane_url) }} | ||
| {{- fail (printf "Value for .Values.registerControlPlane.local_cplane_url is empty") }} | ||
| {{- end }} | ||
| apiVersion: batch/v1 | ||
| kind: Job | ||
| metadata: | ||
| name: {{ include "neon-storage-controller.fullname" . }}-register-job | ||
| labels: | ||
| {{- include "neon-storage-controller.labels" . | nindent 4 }} | ||
| annotations: | ||
| "helm.sh/hook": post-install | ||
| "helm.sh/hook-weight": "-5" | ||
| # Delete the previous resource before a new hook is launche | ||
| "helm.sh/hook-delete-policy": before-hook-creation | ||
| spec: | ||
| parallelism: 1 | ||
| completions: 1 | ||
| template: | ||
| spec: | ||
| serviceAccountName: "{{ include "neon-storage-controller.fullname" . }}-node-describe" | ||
| initContainers: | ||
| - name: set-envs | ||
| image: public.ecr.aws/bitnami/kubectl:1.26 | ||
| resources: | ||
| {{- toYaml .Values.registerControlPlane.resources | nindent 10 }} | ||
| command: ["sh", "-c"] | ||
| args: | ||
| - | | ||
| ZONE=$(kubectl get node "$NODE_NAME" -o=jsonpath='{.metadata.labels.topology\.kubernetes\.io/zone}') | ||
| echo export ZONE=$ZONE > /node/env.sh | ||
| volumeMounts: | ||
| - name: node-info | ||
| mountPath: /node | ||
| env: | ||
| - name: NODE_NAME | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: spec.nodeName | ||
| containers: | ||
| - name: register-storage-controller | ||
| image: public.ecr.aws/docker/library/python:3.12.2-slim-bullseye | ||
| imagePullPolicy: IfNotPresent | ||
| command: ["/bin/bash", "-c"] | ||
| args: ["source /node/env.sh && /usr/local/bin/python /opt/scripts/register-storage-controller.py"] | ||
| resources: | ||
| {{- toYaml .Values.registerControlPlane.resources | nindent 10 }} | ||
| volumeMounts: | ||
| - name: config-volume | ||
| mountPath: /opt/scripts | ||
| - name: node-info | ||
| mountPath: /node | ||
| env: | ||
| - name: HOST | ||
| value: {{ index .Values.service.annotations "external-dns.alpha.kubernetes.io/hostname" | quote }} | ||
| - name: PORT | ||
| value: {{ .Values.service.port | quote }} | ||
| - name: GLOBAL_CPLANE_URL | ||
| value: {{ .Values.registerControlPlane.global_cplane_url | quote }} | ||
| - name: LOCAL_CPLANE_URL | ||
| value: {{ .Values.registerControlPlane.local_cplane_url | quote }} | ||
| - name: CONSOLE_URL | ||
| value: {{ .Values.registerControlPlane.console_url | quote }} | ||
| - name: REGION_ID | ||
| value: {{ .Values.registerControlPlane.region_id | quote }} | ||
| envFrom: | ||
| - secretRef: | ||
| name: {{ include "neon-storage-controller.fullname" . }}-env-vars | ||
| volumes: | ||
| - name: config-volume | ||
| secret: | ||
| secretName: {{ include "neon-storage-controller.fullname" . }}-post-install-script | ||
| defaultMode: 0755 | ||
| - name: node-info | ||
| emptyDir: {} | ||
| restartPolicy: Never | ||
| terminationGracePeriodSeconds: 0 | ||
| {{- end }} |
12 changes: 12 additions & 0 deletions
12
charts/neon-storage-controller/templates/post-install-secrets.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| {{- if .Values.registerControlPlane.enable -}} | ||
| apiVersion: v1 | ||
| kind: Secret | ||
| metadata: | ||
| name: {{ include "neon-storage-controller.fullname" . }}-post-install-script | ||
| labels: | ||
| {{- include "neon-storage-controller.labels" . | nindent 4 }} | ||
| type: Opaque | ||
| data: | ||
| register-storage-controller.py: |- | ||
| {{ tpl (.Files.Get "scripts/register-storage-controller.py") . | b64enc }} | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| apiVersion: v1 | ||
| kind: Secret | ||
| metadata: | ||
| name: {{ include "neon-storage-controller.fullname" . }}-env-vars | ||
| labels: | ||
| {{- include "neon-storage-controller.labels" . | nindent 4 }} | ||
| type: Opaque | ||
| data: | ||
| DATABASE_URL: {{ .Values.settings.databaseUrl | b64enc | quote }} | ||
| JWT_TOKEN: {{ .Values.settings.jwtToken| b64enc | quote }} | ||
| CONTROL_PLANE_JWT_TOKEN: {{ .Values.settings.controlPlaneJwtToken | b64enc | quote }} | ||
| CONSOLE_API_KEY: {{ .Values.settings.apiKey | b64enc | quote }} |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Also, you removed this condition, meaning flag always set.
Which means AWS secret manager is never used, right?
Then this comment become incorrect: https://github.com/neondatabase/helm-charts/blob/main/charts/neon-storage-controller/values.yaml#L27
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
it shouldn't use secrets from AWS manager, we will be moving to GitOps with fluxcd where all secrets are in sops, also we use sops in terraform. secrets in AWS secrets can be change from UI easily which can create issue, using secrets (sops) in git is safe and single source of truth.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Update values.yaml then, please
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'll leave that to @jcsp