diff --git a/.gitignore b/.gitignore index 0a03658..f85a5bf 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ __pycache__ *kubeconfig-*.yaml **/.DS_Store .vscode +.idea diff --git a/examples/flux_operator_ca_hpa/README.md b/examples/flux_operator_ca_hpa/README.md deleted file mode 100644 index d11b60a..0000000 --- a/examples/flux_operator_ca_hpa/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Setup Kubernetes Cluster with Cluster Autoscaling - -## Deploy the cluster -This file creates/deletes/scales a EKS Cluster. The nodes are managed by both EKS Nodegroup and Cloudformation Stacks. - -``` -python3 k8s_cluster_operations.py -h -positional arguments: - cluster_name Cluster name suffix - -optional arguments: - -h, --help show this help message and exit - --experiment EXPERIMENT - Experiment name (defaults to script name) - - --node-count NODE_COUNT - starting node count of the cluster - - --max-node-count MAX_NODE_COUNT - maximum node count - - --min-node-count MIN_NODE_COUNT - minimum node count - - --machine-type MACHINE_TYPE - AWS EC2 Instance types - - --operation [{create,delete,scale}] - Define which operation you want to perform, If you want to scale, be sure to increase the NODE_COUNT. The cluster size will increase depending on the current instance size. if NODE_COUNT is less than the current, the cluster nodes will be scaled down. - - --eks-nodegroup - Include this option to use eks nodegroup for instances, otherwise, it'll use cloudformation stack. EKS Nodegroup will automatically set tags in the aws autoscaling group so that cluster autoscaler can discover them. - - --enable-cluster-autoscaler - Include this to enable cluster autoscaling. This will also create an OIDC provider for the cloud. be sure to take a note of the RoleARN that this script will print. -``` - -Example usage - -```console -basicinsect:flux_operator_ca_hpa hossen1$ python3 k8s_cluster_operations.py --operation "create" --enable-cluster-autoscaler --eks-nodegroup -📛️ Cluster name is kubernetes-flux-operator-hpa-ca-cluster -⭐️ Creating the cluster sized 1 to 5... -🥞️ Creating VPC stack and subnets... -🥣️ Creating cluster... -The status of nodegroup CREATING -Waiting for kubernetes-flux-operator-hpa-ca-cluster-worker-group nodegroup... -Setting Up the cluster OIDC Provider -The cluster autoscaler Role ARN - arn:aws:iam:::role/AmazonEKSClusterAutoscalerRole - -⏱️ Waiting for 1 nodes to be Ready... -Time for kubernetes to get nodes - 5.082208871841431 -🦊️ Writing config file to kubeconfig-aws.yaml - Usage: kubectl --kubeconfig=kubeconfig-aws.yaml get nodes -``` - -## Set UP Cluster Autoscaler - -Be sure to change two things in this file [cluster-autoscaler-autodiscover.yaml](cluster-autoscaler/cluster-autoscaler-autodiscover.yaml) - -1. RoleARN `arn:aws:iam:::role/AmazonEKSClusterAutoscalerRole` in the service account portion -2. Cluster Name - `kubernetes-flux-operator-hpa-ca-cluster` in the commnds of the cluster autoscaler. - -then apply the changes.. -```console -kubectl --kubeconfig=kubeconfig-aws.yaml apply -f cluster-autoscaler/cluster-autoscaler-autodiscover.yaml -``` - -Verify cluster autoscaler is up -```console -$ kubectl --kubeconfig=kubeconfig-aws.yaml get pods -n kube-system -NAME READY STATUS RESTARTS AGE -aws-node-2dz6x 1/1 Running 0 9h -aws-node-pzwl9 1/1 Running 0 9h -cluster-autoscaler-747689d74b-6lkfk 1/1 Running 0 8h -coredns-79df7fff65-q984f 1/1 Running 0 9h -coredns-79df7fff65-tlkwc 1/1 Running 0 9h -kube-proxy-8ch5x 1/1 Running 0 9h -kube-proxy-kq9ch 1/1 Running 0 9h -metrics-server-7db4fb59f9-qdp2c 1/1 Running 0 7h5m -``` - -This will print the logs. be sure that cluster autoscaler discovered the autoscaling group and working properly. -```console -kubectl --kubeconfig=kubeconfig-aws.yaml -n kube-system logs deploy/cluster-autoscaler -``` - -## Run application to collect metrics -Follow this [ca_hpa_readme.md](README_CA_HPA.md) to see how to run a program that will collect metrics for horizontal pod autoscaling, cluster autoscaling diff --git a/examples/flux_operator_ca_hpa/README_CA_HPA.md b/examples/flux_operator_ca_hpa/README_CA_HPA.md deleted file mode 100644 index aeb7700..0000000 --- a/examples/flux_operator_ca_hpa/README_CA_HPA.md +++ /dev/null @@ -1,38 +0,0 @@ -# Metrics Collections -The purpose of this [file](application_ca_hpa_metrics.py) is to collect application and system metrics. The assumption is that, we have a kubernetes cluster with cluster autoscaling and horizontal pod autoscaling (HPA) enabled. The metrics and logs this file capture are - - -1. How long a POD is in pending state due to resource unavailability -2. How long it takes to run the container once the pod is scheduled -3. When does HPA take action by seeing the CPU Utilization -4. When there's pending pod, how long it takes for cluster autoscaler to take action -5. when does the cluster autoscaler add new nodes? -6. when cluster autoscaler request for new nodes, how long it takes to get the nodes? -7. when the load is decreased, how long it takes for HPA to scale down pods -8. When there's no load, how long it takes for CA to remove nodes? -9. when do the nodes are actually removed? - -We can answer the above questions and many more by collecting the metrics. This file will save the results in the data directory. - -Run the file following -```console -python3 application_ca_hpa_metrics.py -h -usage: application_ca_hpa_metrics.py [-h] [--flux-namespace FLUX_NAMESPACE] [--autoscaler-namespace AUTOSCALER_NAMESPACE] [--hpa-namespace HPA_NAMESPACE] [--kubeconfig KUBECONFIG] [--outdir OUTDIR] - -Program to collect various metrics from kubernetes - -optional arguments: - -h, --help - show this help message and exit - - --flux-namespace FLUX_NAMESPACE - Namespace of the flux operator - - --autoscaler-namespace AUTOSCALER_NAMESPACE - Namespace of the cluster autoscaler - - --hpa-namespace HPA_NAMESPACE - Namespace of the horizontal pod autoscaler - - --kubeconfig KUBECONFIG - config file name, full path if the file is not in the current directory -``` diff --git a/examples/flux_operator_ca_hpa/basic-minicluster-setup/README.md b/examples/flux_operator_ca_hpa/basic-minicluster-setup/README.md deleted file mode 100644 index a158b9b..0000000 --- a/examples/flux_operator_ca_hpa/basic-minicluster-setup/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Flux Operator Mini Cluster Setup - -## Basic Minicluster setup -This setup assumes you already created kubernetes cluster with at least 1/2 Nodes by following the direction [here](../README.md) - -Create the flux-operator namespace and install the operator: - -```bash -$ kubectl create namespace flux-operator -$ kubectl apply -f operator-minicluster/basic-configs/flux-operator.yaml -``` - - -```bash -$ kubectl apply -f operator-minicluster/basic-configs/minicluster.yaml -``` - -You'll need to wait for the container to pull (status `ContainerCreating` to `Running`). -At this point, wait until the containers go from creating to running. - -```bash -$ kubectl get -n flux-operator pods -NAME READY STATUS RESTARTS AGE -flux-sample-0-4wmmp 1/1 Running 0 6m50s -flux-sample-1-mjj7b 1/1 Running 0 6m50s -``` - -## Flux Cluster for With LAMMPS Application - -For this setup, we can not use python api, because, currently, we need placement group for lammps and boto3 api lacks the support for providing `placement group` option. So, we will use `eksctl`. If you don't have `eksctl`, please install it first. - -```console -eksctl create cluster -f operator-minicluster/hpc7g-configs/eks-efa-cluster-config-hpc7g.yaml -``` - -This will create a cluster with managed nodegroup, oidc provider, and service account for cluster autoscaler. - -Now deploy an arm version of the Flux Operator. -```console -kubectl apply -f operator-minicluster/hpc7g-configs/flux-operator-arm.yaml -``` - -This will create our size 1 cluster that we will be running LAMMPS on many times: -``` -kubectl create namespace flux-operator -kubectl apply -f operator-minicluster/hpc7g-configs/minicluster-libfabric-new.yaml # 18.1.1 -``` - -More to follow... diff --git a/examples/flux_operator_ca_hpa/basic-minicluster-setup/flux-operator.yaml b/examples/flux_operator_ca_hpa/basic-minicluster-setup/flux-operator.yaml deleted file mode 100644 index 0089d54..0000000 --- a/examples/flux_operator_ca_hpa/basic-minicluster-setup/flux-operator.yaml +++ /dev/null @@ -1,1373 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: operator-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null - name: miniclusters.flux-framework.org -spec: - group: flux-framework.org - names: - kind: MiniCluster - listKind: MiniClusterList - plural: miniclusters - singular: minicluster - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: MiniCluster is the Schema for a Flux job launcher on K8s - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: MiniCluster is an HPC cluster in Kubernetes you can control - Either to submit a single job (and go away) or for a persistent single- - or multi- user cluster - properties: - archive: - description: Archive to load or save - properties: - path: - description: Save or load from this directory path - type: string - type: object - cleanup: - default: false - description: Cleanup the pods and storage when the index broker pod - is complete - type: boolean - containers: - description: Containers is one or more containers to be created in - a pod. There should only be one container to run flux with runFlux - items: - properties: - batch: - description: Indicate that the command is a batch job that will - be written to a file to submit - type: boolean - batchRaw: - description: Don't wrap batch commands in flux submit (provide - custom logic myself) - type: boolean - command: - description: Single user executable to provide to flux start - type: string - commands: - description: More specific or detailed commands for just workers/broker - properties: - brokerPre: - description: A single command for only the broker to run - type: string - init: - description: init command is run before anything - type: string - post: - description: post command is run in the entrypoint when - the broker exits / finishes - type: string - pre: - description: pre command is run after global PreCommand, - after asFlux is set (can override) - type: string - prefix: - description: Prefix to flux start / submit / broker Typically - used for a wrapper command to mount, etc. - type: string - runFluxAsRoot: - default: false - description: Run flux start as root - required for some - storage binds - type: boolean - workerPre: - description: A command only for workers to run - type: string - type: object - cores: - description: Cores the container should use - format: int32 - type: integer - diagnostics: - description: Run flux diagnostics on start instead of command - type: boolean - environment: - additionalProperties: - type: string - description: Key/value pairs for the environment - type: object - existingVolumes: - additionalProperties: - description: Mini Cluster local volumes available to mount - (these are on the host) - properties: - claimName: - description: Claim name if the existing volume is a PVC - type: string - configMapName: - description: Config map name if the existing volume is - a config map You should also define items if you are - using this - type: string - items: - additionalProperties: - type: string - description: Items (key and paths) for the config map - type: object - path: - description: Path and claim name are always required if - a secret isn't defined - type: string - readOnly: - default: false - type: boolean - secretName: - description: An existing secret - type: string - type: object - description: Existing Volumes to add to the containers - type: object - fluxUser: - description: Flux User, if created in the container - properties: - name: - default: flux - description: Flux user name - type: string - uid: - default: 1000 - description: UID for the FluxUser - type: integer - type: object - image: - default: ghcr.io/rse-ops/accounting:app-latest - description: Container image must contain flux and flux-sched - install - type: string - imagePullSecret: - description: Allow the user to pull authenticated images By - default no secret is selected. Setting this with the name - of an already existing imagePullSecret will specify that secret - in the pod spec. - type: string - launcher: - description: Indicate that the command is a launcher that will - ask for its own jobs (and provided directly to flux start) - type: boolean - lifeCycle: - description: Lifecycle can handle post start commands, etc. - properties: - postStartExec: - type: string - preStopExec: - type: string - type: object - logs: - description: Log output directory - type: string - name: - description: Container name is only required for non flux runners - type: string - ports: - description: Ports to be exposed to other containers in the - cluster We take a single list of integers and map to the same - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: atomic - preCommand: - description: Special command to run at beginning of script, - directly after asFlux is defined as sudo -u flux -E (so you - can change that if desired.) This is only valid if FluxRunner - is set (that writes a wait.sh script) This is for the indexed - job pods and the certificate generation container. - type: string - pullAlways: - default: false - description: Allow the user to dictate pulling By default we - pull if not present. Setting this to true will indicate to - pull always - type: boolean - resources: - description: Resources include limits and requests - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - type: object - runFlux: - description: Main container to run flux (only should be one) - type: boolean - securityContext: - description: Security Context https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - privileged: - description: Privileged container - type: boolean - type: object - volumes: - additionalProperties: - description: A Container volume must reference one defined - for the MiniCluster The path here is in the container - properties: - path: - type: string - readOnly: - default: false - type: boolean - required: - - path - type: object - description: Volumes that can be mounted (must be defined in - volumes) - type: object - workingDir: - description: Working directory to run command from - type: string - type: object - type: array - x-kubernetes-list-type: atomic - deadlineSeconds: - default: 31500000 - description: Should the job be limited to a particular number of seconds? - Approximately one year. This cannot be zero or job won't start - format: int64 - type: integer - flux: - description: Flux options for the broker, shared across cluster - properties: - brokerConfig: - description: Optionally provide a manually created broker config - this is intended for bursting to remote clusters - type: string - bursting: - description: Bursting - one or more external clusters to burst - to We assume a single, central MiniCluster with an ipaddress - that all connect to. - properties: - clusters: - description: External clusters to burst to. Each external - cluster must share the same listing to align ranks - items: - properties: - name: - description: The hostnames for the bursted clusters - If set, the user is responsible for ensuring uniqueness. - The operator will set to burst-N - type: string - size: - description: Size of bursted cluster. Defaults to same - size as local minicluster if not set - format: int32 - type: integer - type: object - type: array - leadBroker: - description: The lead broker ip address to join to. E.g., - if we burst to cluster 2, this is the address to connect - to cluster 1 For the first cluster, this should not be defined - properties: - address: - description: Lead broker address (ip or hostname) - type: string - name: - description: We need the name of the lead job to assemble - the hostnames - type: string - port: - default: 8050 - description: Lead broker port - should only be used for - external cluster - format: int32 - type: integer - size: - description: Lead broker size - format: int32 - type: integer - required: - - address - - name - - size - type: object - required: - - clusters - type: object - connectTimeout: - default: 5s - description: Single user executable to provide to flux start - type: string - curveCert: - description: Optionally provide an already existing curve certificate - This is not recommended in favor of providing the secret name - as curveCertSecret, below - type: string - curveCertSecret: - description: Expect a secret for a curve cert here. This is ideal - over the curveCert (as a string) above. - type: string - installRoot: - default: /usr - description: Install root location - type: string - logLevel: - default: 6 - description: Log level to use for flux logging (only in non TestMode) - format: int32 - type: integer - minimalService: - description: Only expose the broker service (to reduce load on - DNS) - type: boolean - mungeSecret: - description: Expect a secret (named according to this string) - for a munge key. This is intended for bursting. Assumed to be - at /etc/munge/munge.key This is binary data. - type: string - optionFlags: - description: Flux option flags, usually provided with -o optional - - if needed, default option flags for the server These can also - be set in the user interface to override here. This is only - valid for a FluxRunner "runFlux" true - type: string - wrap: - description: Commands for flux start --wrap - type: string - type: object - fluxRestful: - description: Customization to Flux Restful API There should only be - one container to run flux with runFlux - properties: - branch: - default: main - description: Branch to clone Flux Restful API from - type: string - port: - default: 5000 - description: Port to run Flux Restful Server On - format: int32 - type: integer - secretKey: - description: Secret key shared between server and client - type: string - token: - description: Token to use for RestFul API - type: string - username: - description: These two should not actually be set by a user, but - rather generated by tools and provided Username to use for RestFul - API - type: string - type: object - interactive: - default: false - description: Run a single-user, interactive minicluster - type: boolean - jobLabels: - additionalProperties: - type: string - description: Labels for the job - type: object - logging: - description: Logging modes determine the output you see in the job - log - properties: - debug: - default: false - description: Debug mode adds extra verbosity to Flux - type: boolean - quiet: - default: false - description: Quiet mode silences all output so the job only shows - the test running - type: boolean - strict: - default: true - description: Strict mode ensures any failure will not continue - in the job entrypoint - type: boolean - timed: - default: false - description: Timed mode adds timing to Flux commands - type: boolean - zeromq: - default: false - description: Enable Zeromq logging - type: boolean - type: object - maxSize: - description: MaxSize (maximum number of pods to allow scaling to) - format: int32 - type: integer - network: - description: A spec for exposing or defining the cluster headless - service - properties: - headlessName: - default: flux-service - description: Name for cluster headless service - type: string - type: object - pod: - description: Pod spec details - properties: - annotations: - additionalProperties: - type: string - description: Annotations for each pod - type: object - labels: - additionalProperties: - type: string - description: Labels for each pod - type: object - nodeSelector: - additionalProperties: - type: string - description: NodeSelectors for a pod - type: object - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - description: Resources include limits and requests - type: object - serviceAccountName: - description: Service account name for the pod - type: string - type: object - services: - description: Services are one or more service containers to bring - up alongside the MiniCluster. - items: - properties: - batch: - description: Indicate that the command is a batch job that will - be written to a file to submit - type: boolean - batchRaw: - description: Don't wrap batch commands in flux submit (provide - custom logic myself) - type: boolean - command: - description: Single user executable to provide to flux start - type: string - commands: - description: More specific or detailed commands for just workers/broker - properties: - brokerPre: - description: A single command for only the broker to run - type: string - init: - description: init command is run before anything - type: string - post: - description: post command is run in the entrypoint when - the broker exits / finishes - type: string - pre: - description: pre command is run after global PreCommand, - after asFlux is set (can override) - type: string - prefix: - description: Prefix to flux start / submit / broker Typically - used for a wrapper command to mount, etc. - type: string - runFluxAsRoot: - default: false - description: Run flux start as root - required for some - storage binds - type: boolean - workerPre: - description: A command only for workers to run - type: string - type: object - cores: - description: Cores the container should use - format: int32 - type: integer - diagnostics: - description: Run flux diagnostics on start instead of command - type: boolean - environment: - additionalProperties: - type: string - description: Key/value pairs for the environment - type: object - existingVolumes: - additionalProperties: - description: Mini Cluster local volumes available to mount - (these are on the host) - properties: - claimName: - description: Claim name if the existing volume is a PVC - type: string - configMapName: - description: Config map name if the existing volume is - a config map You should also define items if you are - using this - type: string - items: - additionalProperties: - type: string - description: Items (key and paths) for the config map - type: object - path: - description: Path and claim name are always required if - a secret isn't defined - type: string - readOnly: - default: false - type: boolean - secretName: - description: An existing secret - type: string - type: object - description: Existing Volumes to add to the containers - type: object - fluxUser: - description: Flux User, if created in the container - properties: - name: - default: flux - description: Flux user name - type: string - uid: - default: 1000 - description: UID for the FluxUser - type: integer - type: object - image: - default: ghcr.io/rse-ops/accounting:app-latest - description: Container image must contain flux and flux-sched - install - type: string - imagePullSecret: - description: Allow the user to pull authenticated images By - default no secret is selected. Setting this with the name - of an already existing imagePullSecret will specify that secret - in the pod spec. - type: string - launcher: - description: Indicate that the command is a launcher that will - ask for its own jobs (and provided directly to flux start) - type: boolean - lifeCycle: - description: Lifecycle can handle post start commands, etc. - properties: - postStartExec: - type: string - preStopExec: - type: string - type: object - logs: - description: Log output directory - type: string - name: - description: Container name is only required for non flux runners - type: string - ports: - description: Ports to be exposed to other containers in the - cluster We take a single list of integers and map to the same - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: atomic - preCommand: - description: Special command to run at beginning of script, - directly after asFlux is defined as sudo -u flux -E (so you - can change that if desired.) This is only valid if FluxRunner - is set (that writes a wait.sh script) This is for the indexed - job pods and the certificate generation container. - type: string - pullAlways: - default: false - description: Allow the user to dictate pulling By default we - pull if not present. Setting this to true will indicate to - pull always - type: boolean - resources: - description: Resources include limits and requests - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - type: object - runFlux: - description: Main container to run flux (only should be one) - type: boolean - securityContext: - description: Security Context https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - privileged: - description: Privileged container - type: boolean - type: object - volumes: - additionalProperties: - description: A Container volume must reference one defined - for the MiniCluster The path here is in the container - properties: - path: - type: string - readOnly: - default: false - type: boolean - required: - - path - type: object - description: Volumes that can be mounted (must be defined in - volumes) - type: object - workingDir: - description: Working directory to run command from - type: string - type: object - type: array - x-kubernetes-list-type: atomic - size: - default: 1 - description: Size (number of job pods to run, size of minicluster - in pods) This is also the minimum number required to start Flux - format: int32 - type: integer - tasks: - default: 1 - description: Total number of CPUs being run across entire cluster - format: int32 - type: integer - users: - description: Users of the MiniCluster - items: - properties: - name: - description: If a user is defined, the username is required - type: string - password: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-type: atomic - volumes: - additionalProperties: - description: Mini Cluster local volumes available to mount (these - are on the host) - properties: - annotations: - additionalProperties: - type: string - description: Annotations for the volume - type: object - attributes: - additionalProperties: - type: string - description: Optional volume attributes - type: object - capacity: - default: 5Gi - description: Capacity (string) for PVC (storage request) to - create PV - type: string - claimAnnotations: - additionalProperties: - type: string - description: Annotations for the persistent volume claim - type: object - delete: - default: true - description: Delete the persistent volume on cleanup - type: boolean - driver: - description: Storage driver, e.g., gcs.csi.ofek.dev Only needed - if not using hostpath - type: string - labels: - additionalProperties: - type: string - type: object - path: - type: string - secret: - description: Secret reference in Kubernetes with service account - role - type: string - secretNamespace: - default: default - description: Secret namespace - type: string - storageClass: - default: hostpath - type: string - volumeHandle: - description: Volume handle, falls back to storage class name - if not defined - type: string - required: - - path - type: object - description: Volumes accessible to containers from a host Not all - containers are required to use them - type: object - required: - - containers - type: object - status: - description: MiniClusterStatus defines the observed state of Flux - properties: - completed: - description: Label to indicate that job is completed, comes from Job.Completed - The user can also look at conditions -> JobFinished - type: boolean - conditions: - description: conditions hold the latest Flux Job and MiniCluster states - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-type: atomic - jobid: - description: The Jobid is set internally to associate to a miniCluster - This isn't currently in use, we only have one! - type: string - maximumSize: - description: We keep the original size of the MiniCluster request - as this is the absolute maximum - format: int32 - type: integer - selector: - type: string - size: - description: These are for the sub-resource scale functionality - format: int32 - type: integer - required: - - jobid - - maximumSize - - selector - - size - type: object - type: object - served: true - storage: true - subresources: - scale: - labelSelectorPath: .status.selector - specReplicasPath: .spec.size - statusReplicasPath: .status.size - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: operator-leader-election-role - namespace: operator-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: operator-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - watch -- apiGroups: - - "" - resources: - - events - - nodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - exec - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - create - - delete - - exec - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - "" - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - batch - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - networks - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/exec - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/log - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - clusters - - clusters/status - verbs: - - get - - list - - watch -- apiGroups: - - flux-framework.org - resources: - - machineclasses - - machinedeployments - - machinedeployments/status - - machines - - machines/status - - machinesets - - machinesets/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - miniclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - miniclusters/finalizers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - miniclusters/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: operator-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: operator-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: operator-leader-election-rolebinding - namespace: operator-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: operator-leader-election-role -subjects: -- kind: ServiceAccount - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: operator-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: operator-manager-role -subjects: -- kind: ServiceAccount - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: operator-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: operator-proxy-role -subjects: -- kind: ServiceAccount - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: v1 -data: - controller_manager_config.yaml: | - apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 - kind: ControllerManagerConfig - health: - healthProbeBindAddress: :8081 - metrics: - bindAddress: 127.0.0.1:8080 - webhook: - port: 9443 - leaderElection: - leaderElect: true - resourceName: 14dde902.flux-framework.org -kind: ConfigMap -metadata: - name: operator-manager-config - namespace: operator-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - name: operator-controller-manager-metrics-service - namespace: operator-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - control-plane: controller-manager - name: operator-controller-manager - namespace: operator-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.11.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /manager - image: ghcr.io/flux-framework/flux-operator:latest@sha256:2d5e77dd82a186d409b984c7228a39e5b738d2993faa6bf87b8a6743b68d3b89 - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - securityContext: - runAsNonRoot: true - serviceAccountName: operator-controller-manager - terminationGracePeriodSeconds: 10 diff --git a/examples/flux_operator_ca_hpa/basic-minicluster-setup/minicluster.yaml b/examples/flux_operator_ca_hpa/basic-minicluster-setup/minicluster.yaml deleted file mode 100644 index 3c2dc16..0000000 --- a/examples/flux_operator_ca_hpa/basic-minicluster-setup/minicluster.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: flux-framework.org/v1alpha1 -kind: MiniCluster -metadata: - name: flux-sample - namespace: flux-operator -spec: - size: 2 - - # If we don't set this, we won't be able to go above two! - maxSize: 10 - - # Interactive will start the broker to shell into - interactive: true - - # This is a list because a pod can support multiple containers - containers: - - image: ghcr.io/rse-ops/singularity:tag-mamba - workingDir: /data - - # You can shell in to connect to the broker and issue commands that use CPU - command: sleep infinity - - # Important! We need to have resource requests for the horizonal autoscaler - # The Flux Operator doesn't know you want to use it, so it's up to you - # to provide these if your metric is about CPU - resources: - limits: - cpu: "1" - - requests: - cpu: "1" - - fluxUser: - name: fluxuser diff --git a/examples/flux_operator_ca_hpa/cluster-autoscaler/AmazonEKSClusterAutoscalerPolicy.json b/examples/flux_operator_ca_hpa/cluster-autoscaler/AmazonEKSClusterAutoscalerPolicy.json deleted file mode 100644 index ec83711..0000000 --- a/examples/flux_operator_ca_hpa/cluster-autoscaler/AmazonEKSClusterAutoscalerPolicy.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeScalingActivities", - "autoscaling:DescribeTags", - "ec2:DescribeInstanceTypes", - "ec2:DescribeLaunchTemplateVersions" - ], - "Resource": ["*"] - }, - { - "Effect": "Allow", - "Action": [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "ec2:DescribeImages", - "ec2:GetInstanceTypesFromInstanceRequirements", - "eks:DescribeNodegroup" - ], - "Resource": ["*"] - } - ] - } diff --git a/examples/flux_operator_ca_hpa/cluster-autoscaler/README.md b/examples/flux_operator_ca_hpa/cluster-autoscaler/README.md deleted file mode 100644 index 81ac6d5..0000000 --- a/examples/flux_operator_ca_hpa/cluster-autoscaler/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Setting Up Cluster Autoscaler in EKS -Run this script to create a cluster and a nodegroup with 1 instance. The node group will have necessary tags required for cluster autoscaling. - -```console -python3 create-delete-k8s-cluster.py --min-node-count 1 --max-node-count 3 --machine-type m5.large -``` -or -```console -eksctl create cluster -f flux-with-lammps-setup/hpc7g-configs/eks-efa-cluster-config-hpc7g.yaml -``` - -Note: the eksctl yaml script will create everything needed for cluster autoscaling. So, if you creates the cluster with eksctl, skip the steps below and directly apply the cluster autoscaler. - -### Create a service account for the kubernetes autoscaler. - -First, create the IAM OIDC Provider for the cluster. Follow this link for more details - [Link](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html). Grab the cluster name from the output - -```console -$ export cluster_name="kubernetes-flux-operator-hpa-ca-cluster" - -$ oidc_id=$(aws eks describe-cluster --name $cluster_name --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5) - -$ eksctl utils associate-iam-oidc-provider --cluster $cluster_name --approve -``` - -Create a policy for the Service Account Role - -```console -$ aws iam create-policy --policy-name AmazonEKSClusterAutoscalerPolicy --policy-document file://AmazonEKSClusterAutoscalerPolicy.json -``` - -### Create a Role and attach the policy - Follow for [More](https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html#irsa-create-role) - -First, set account id and oidc_provider -```console -$ account_id=$(aws sts get-caller-identity --query "Account" --output text) - -$ oidc_provider=$(aws eks describe-cluster --name $cluster_name --region $AWS_REGION --query "cluster.identity.oidc.issuer" --output text | sed -e "s/^https:\/\///") -``` - -Now, create role with the below command. keep a note of the role arn -```console -$ aws iam create-role --role-name AmazonEKSClusterAutoscalerRole --assume-role-policy-document file://trust-relationship.json --description "AWS Iam role for cluster autoscaler" -``` - -Attach the policy we created earlier. -```console -$ aws iam attach-role-policy --role-name AmazonEKSClusterAutoscalerRole --policy-arn=arn:aws:iam::$account_id:policy/AmazonEKSClusterAutoscalerPolicy -``` - -### Create Service Account -Create a kubernetes service account with the RoleARN. autoscaler yamls provided by AWSs already have the service account portion, you can just edit and put ROLEARN. - -```yaml ---- -apiVersion: v1 -kind: ServiceAccount -metadata: -labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam:::role/AmazonEKSClusterAutoscalerRole - name: cluster-autoscaler - namespace: kube-system ---- -``` -### Deploy the autoscaler -Finally, be sure to change the name of the cluster in the command section of cluster autoscaler container. - -```console -$ kubectl apply -f cluster-autoscaler/cluster-autoscaler-autodiscover.yaml -``` - -Helpful Links -1. https://www.kubecost.com/kubernetes-autoscaling/kubernetes-cluster-autoscaler/ -2. https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/CA_with_AWS_IAM_OIDC.md -3. https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html#irsa-create-role -4. https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md diff --git a/examples/flux_operator_ca_hpa/cluster-autoscaler/autoscaler-demo.yaml b/examples/flux_operator_ca_hpa/cluster-autoscaler/autoscaler-demo.yaml deleted file mode 100644 index d0b2323..0000000 --- a/examples/flux_operator_ca_hpa/cluster-autoscaler/autoscaler-demo.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-deployment -spec: - selector: - matchLabels: - app: nginx - replicas: 3 - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 - name: "http-server" - resources: - limits: - cpu: 1000m - memory: 500Mi - requests: - cpu: 1000m - memory: 500Mi diff --git a/examples/flux_operator_ca_hpa/cluster-autoscaler/cloudformation-template-for-oidc.yaml b/examples/flux_operator_ca_hpa/cluster-autoscaler/cloudformation-template-for-oidc.yaml deleted file mode 100644 index 223f47e..0000000 --- a/examples/flux_operator_ca_hpa/cluster-autoscaler/cloudformation-template-for-oidc.yaml +++ /dev/null @@ -1,147 +0,0 @@ ---- -# SPDX-FileCopyrightText: 2020 Bamboo Loans -# SPDX-License-Identifier: MIT -AWSTemplateFormatVersion: '2010-09-09' -Description: 'An example CloudFormation template to configure an EKS cluster with an OpenID Connect provider to use IAM backed service accounts' -Metadata: - Source: https://github.com/bambooengineering/example-eks-oidc-iam-cloudformation - -Parameters: - EKSClusterName: - Type: String - Description: Name for EKS Cluster - -Resources: - ClusterOIDCURL: - Type: Custom::ClusterOIDCURL - Properties: - ServiceToken: !GetAtt ClusterOIDCURLFunction.Arn - ClusterName: !Ref EKSClusterName - - # We need to use the API to get the OpenID Connect URL from the cluster, so a Lambda does that for us here - ClusterOIDCURLFunction: - Type: AWS::Lambda::Function - Properties: - Runtime: python3.7 - Handler: index.lambda_handler - MemorySize: 128 - Role: !GetAtt ClusterOIDCLambdaExecutionRole.Arn - Timeout: 30 - Code: - ZipFile: | - import boto3 - import json - import cfnresponse - eks = boto3.client("eks") - def lambda_handler(event, context): - responseData = {} - if event['RequestType'] == 'Delete': - responseData['Reason'] = "Success" - cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, "") - else: - try: - cluster_name = event['ResourceProperties']['ClusterName'] - response = eks.describe_cluster(name=cluster_name) - cluster_oidc_url = response['cluster']['identity']['oidc']['issuer'] - # We need the url for the roles without the protocol when creating roles, so remove - # it here to make this easier to use in CF templates. - without_protocol = cluster_oidc_url.replace('https://', '') - responseData['Reason'] = "Success" - responseData['Url'] = without_protocol - cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, without_protocol) - except Exception as e: - responseData['Reason'] = str(e) - cfnresponse.send(event, context, cfnresponse.FAILED, responseData, "") - - ClusterOIDCProvider: - Type: Custom::ClusterOIDCProvider - Properties: - ServiceToken: !GetAtt ClusterOIDCProviderFunction.Arn - ClusterOIDCURL: !Ref ClusterOIDCURL - - # This defines the lambda that will run the setup (and teardown) code for the OIDC provider - ClusterOIDCProviderFunction: - Type: AWS::Lambda::Function - Properties: - Runtime: python3.7 - Handler: index.lambda_handler - MemorySize: 128 - Role: !GetAtt ClusterOIDCLambdaExecutionRole.Arn - Timeout: 30 - Code: - ZipFile: | - import boto3 - from botocore.exceptions import ClientError - import json - import cfnresponse - iam = boto3.client("iam") - def lambda_handler(event, context): - data = {} - try: - cluster_oidc_url = event['ResourceProperties']['ClusterOIDCURL'] - if event['RequestType'] == 'Create': - with_protocol = "https://" + cluster_oidc_url - # This is the ca thumbprint of AWS's issuer - issuer_thumbprint = '9e99a48a9960b14926bb7f3b02e22da2b0ab7280' - resp = iam.create_open_id_connect_provider(Url=with_protocol,ClientIDList=['sts.amazonaws.com'],ThumbprintList=[issuer_thumbprint]) - provider_arn = resp['OpenIDConnectProviderArn'] - data["Reason"] = "Provider with ARN " + provider_arn + " created" - cfnresponse.send(event, context, cfnresponse.SUCCESS, data, provider_arn) - elif event['RequestType'] == 'Delete': - provider_arn = event["PhysicalResourceId"] - if provider_arn is None: - data["Reason"] = "Provider not present" - cfnresponse.send(event, context, cfnresponse.SUCCESS, data, provider_arn) - else: - resp = iam.delete_open_id_connect_provider(OpenIDConnectProviderArn=provider_arn) - data["Reason"] = "Provider with ARN " + provider_arn + " deleted" - cfnresponse.send(event, context, cfnresponse.SUCCESS, data, provider_arn) - else: - data["Reason"] = "Unknown operation: " + event['RequestType'] - cfnresponse.send(event, context, cfnresponse.FAILED, data, "") - except Exception as e: - data["Reason"] = "Cannot " + event['RequestType'] + " Provider" + str(e) - cfnresponse.send(event, context, cfnresponse.FAILED, data, "") - - # This the role that gives the stack sufficient permissions to create the OIDC provider. It is only - # used during lifecycle operations of this stack - ClusterOIDCLambdaExecutionRole: - Type: AWS::IAM::Role - Properties: - Path: / - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Policies: - - PolicyName: root - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - eks:DescribeCluster - Resource: !Sub "arn:aws:eks:${AWS::Region}:${AWS::AccountId}:cluster/${EKSClusterName}" - - Effect: Allow - Action: - - iam:*OpenIDConnectProvider* - Resource: "*" - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: "*" - -Outputs: - ClusterOIDCURL: - Description: The OpenID Connect URL (without protocol) - Value: !Ref ClusterOIDCURL - ClusterOIDCProvider: - Description: The ARN of the OIDCProvider - Value: !Ref ClusterOIDCProvider diff --git a/examples/flux_operator_ca_hpa/cluster-autoscaler/cluster-autoscaler-autodiscover.yaml b/examples/flux_operator_ca_hpa/cluster-autoscaler/cluster-autoscaler-autodiscover.yaml deleted file mode 100644 index 1b688c7..0000000 --- a/examples/flux_operator_ca_hpa/cluster-autoscaler/cluster-autoscaler-autodiscover.yaml +++ /dev/null @@ -1,183 +0,0 @@ -# --- -# apiVersion: v1 -# kind: ServiceAccount -# metadata: -# labels: -# k8s-addon: cluster-autoscaler.addons.k8s.io -# k8s-app: cluster-autoscaler -# annotations: -# eks.amazonaws.com/role-arn: arn:aws:iam::633731392008:role/AmazonEKSClusterAutoscalerRole -# name: cluster-autoscaler -# namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cluster-autoscaler - labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler -rules: - - apiGroups: [""] - resources: ["events", "endpoints"] - verbs: ["create", "patch"] - - apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] - - apiGroups: [""] - resources: ["pods/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["endpoints"] - resourceNames: ["cluster-autoscaler"] - verbs: ["get", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["watch", "list", "get", "update"] - - apiGroups: [""] - resources: - - "namespaces" - - "pods" - - "services" - - "replicationcontrollers" - - "persistentvolumeclaims" - - "persistentvolumes" - verbs: ["watch", "list", "get"] - - apiGroups: ["extensions"] - resources: ["replicasets", "daemonsets"] - verbs: ["watch", "list", "get"] - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["watch", "list"] - - apiGroups: ["apps"] - resources: ["statefulsets", "replicasets", "daemonsets"] - verbs: ["watch", "list", "get"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csinodes", "csidrivers", "csistoragecapacities"] - verbs: ["watch", "list", "get"] - - apiGroups: ["batch", "extensions"] - resources: ["jobs"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create"] - - apiGroups: ["coordination.k8s.io"] - resourceNames: ["cluster-autoscaler"] - resources: ["leases"] - verbs: ["get", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cluster-autoscaler - namespace: kube-system - labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["create", "list", "watch"] - - apiGroups: [""] - resources: ["configmaps"] - resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] - verbs: ["delete", "get", "update", "watch"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cluster-autoscaler - labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-autoscaler -subjects: - - kind: ServiceAccount - name: cluster-autoscaler - namespace: kube-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cluster-autoscaler - namespace: kube-system - labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cluster-autoscaler -subjects: - - kind: ServiceAccount - name: cluster-autoscaler - namespace: kube-system - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cluster-autoscaler - namespace: kube-system - labels: - app: cluster-autoscaler -spec: - replicas: 1 - selector: - matchLabels: - app: cluster-autoscaler - template: - metadata: - labels: - app: cluster-autoscaler - annotations: - prometheus.io/scrape: 'true' - prometheus.io/port: '8085' - spec: - priorityClassName: system-cluster-critical - securityContext: - runAsNonRoot: true - runAsUser: 65534 - fsGroup: 65534 - seccompProfile: - type: RuntimeDefault - serviceAccountName: cluster-autoscaler - containers: - - image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.2 - name: cluster-autoscaler - resources: - limits: - cpu: 500m - memory: 600Mi - requests: - cpu: 100m - memory: 600Mi - command: - - ./cluster-autoscaler - - --v=2 - - --stderrthreshold=info - - --cloud-provider=aws - - --skip-nodes-with-local-storage=false - - --expander=least-waste - - --scale-down-unneeded-time=2m - - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,kubernetes.io/cluster/scaling-study-efa-hossen1-08142023 - volumeMounts: - - name: ssl-certs - mountPath: /etc/ssl/certs/ca-certificates.crt # /etc/ssl/certs/ca-bundle.crt for Amazon Linux Worker Nodes - readOnly: true - imagePullPolicy: "Always" - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - volumes: - - name: ssl-certs - hostPath: - path: "/etc/ssl/certs/ca-bundle.crt" diff --git a/examples/flux_operator_ca_hpa/cluster-autoscaler/cluster-autoscaler-one-asg.yaml b/examples/flux_operator_ca_hpa/cluster-autoscaler/cluster-autoscaler-one-asg.yaml deleted file mode 100644 index 4092710..0000000 --- a/examples/flux_operator_ca_hpa/cluster-autoscaler/cluster-autoscaler-one-asg.yaml +++ /dev/null @@ -1,181 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam:::role/AmazonEKSClusterAutoscalerRole - name: cluster-autoscaler - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cluster-autoscaler - labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler -rules: - - apiGroups: [""] - resources: ["events", "endpoints"] - verbs: ["create", "patch"] - - apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] - - apiGroups: [""] - resources: ["pods/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["endpoints"] - resourceNames: ["cluster-autoscaler"] - verbs: ["get", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["watch", "list", "get", "update"] - - apiGroups: [""] - resources: - - "namespaces" - - "pods" - - "services" - - "replicationcontrollers" - - "persistentvolumeclaims" - - "persistentvolumes" - verbs: ["watch", "list", "get"] - - apiGroups: ["extensions"] - resources: ["replicasets", "daemonsets"] - verbs: ["watch", "list", "get"] - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["watch", "list"] - - apiGroups: ["apps"] - resources: ["statefulsets", "replicasets", "daemonsets"] - verbs: ["watch", "list", "get"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csinodes", "csidrivers", "csistoragecapacities"] - verbs: ["watch", "list", "get"] - - apiGroups: ["batch", "extensions"] - resources: ["jobs"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create"] - - apiGroups: ["coordination.k8s.io"] - resourceNames: ["cluster-autoscaler"] - resources: ["leases"] - verbs: ["get", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cluster-autoscaler - namespace: kube-system - labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["create", "list", "watch"] - - apiGroups: [""] - resources: ["configmaps"] - resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] - verbs: ["delete", "get", "update", "watch"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cluster-autoscaler - labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-autoscaler -subjects: - - kind: ServiceAccount - name: cluster-autoscaler - namespace: kube-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cluster-autoscaler - namespace: kube-system - labels: - k8s-addon: cluster-autoscaler.addons.k8s.io - k8s-app: cluster-autoscaler -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cluster-autoscaler -subjects: - - kind: ServiceAccount - name: cluster-autoscaler - namespace: kube-system - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cluster-autoscaler - namespace: kube-system - labels: - app: cluster-autoscaler -spec: - replicas: 1 - selector: - matchLabels: - app: cluster-autoscaler - template: - metadata: - labels: - app: cluster-autoscaler - annotations: - prometheus.io/scrape: 'true' - prometheus.io/port: '8085' - spec: - priorityClassName: system-cluster-critical - securityContext: - runAsNonRoot: true - runAsUser: 65534 - fsGroup: 65534 - seccompProfile: - type: RuntimeDefault - serviceAccountName: cluster-autoscaler - containers: - - image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.2 - name: cluster-autoscaler - resources: - limits: - cpu: 100m - memory: 600Mi - requests: - cpu: 100m - memory: 600Mi - command: - - ./cluster-autoscaler - - --v=4 - - --stderrthreshold=info - - --cloud-provider=aws - - --skip-nodes-with-local-storage=false - - --nodes=1:5:test-ca-flux-cluster-hpa-ca-workers-NodeGroup-11BIA14LNOMIN - volumeMounts: - - name: ssl-certs - mountPath: /etc/ssl/certs/ca-certificates.crt # /etc/ssl/certs/ca-bundle.crt for Amazon Linux Worker Nodes - readOnly: true - imagePullPolicy: "Always" - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - volumes: - - name: ssl-certs - hostPath: - path: "/etc/ssl/certs/ca-bundle.crt" diff --git a/examples/flux_operator_ca_hpa/cluster-autoscaler/trust-relationship-example.json b/examples/flux_operator_ca_hpa/cluster-autoscaler/trust-relationship-example.json deleted file mode 100644 index 6483b85..0000000 --- a/examples/flux_operator_ca_hpa/cluster-autoscaler/trust-relationship-example.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam::$account_id:oidc-provider/$oidc_provider" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "$oidc_provider:aud": "sts.amazonaws.com", - "$oidc_provider:sub": "system:serviceaccount:kube-system:cluster-autoscaler" - } - } - } - ] -} diff --git a/examples/flux_operator_ca_hpa/cluster-autoscaler/trust-relationship.json b/examples/flux_operator_ca_hpa/cluster-autoscaler/trust-relationship.json deleted file mode 100644 index 353e65f..0000000 --- a/examples/flux_operator_ca_hpa/cluster-autoscaler/trust-relationship.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam:::oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/8E36223D740C8483550F50501AAA97EE" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks.us-east-1.amazonaws.com/id/8E36223D740C8483550F50501AAA97EE:aud": "sts.amazonaws.com", - "oidc.eks.us-east-1.amazonaws.com/id/8E36223D740C8483550F50501AAA97EE:sub": "system:serviceaccount:kube-system:cluster-autoscaler" - } - } - } - ] -} diff --git a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/README.md b/examples/flux_operator_ca_hpa/flux-with-lammps-setup/README.md deleted file mode 100644 index 10b39b0..0000000 --- a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Flux cluster setup with an application - -We will need to setup a kubernetes cluster in aws that will have necessary supports to run an application specifically LAMMPS. -For example, LAMMPS requires EFA enabled networking and a specific placement group. - -Note: For some reason, efa plugin failed with eksctl latest version. So, the current tested and working version is eksctl v0.109.0. - -Deploy the cluster using the below command -```console -$ eksctl create cluster -f flux-with-lammps-setup/hpc7g-configs/eks-efa-cluster-config-hpc7g.yaml - -$ kubectl apply -f flux-with-lammps-setup/hpc7g-configs/flux-operator-arm.yaml -``` - -```console -$ kubectl create namespace flux-operator -$ kubectl apply -f flux-with-lammps-setup/hpc7g-configs/minicluster-libfabric-new.yaml -``` - -```console -$ POD=$(kubectl get pods -n flux-operator -o json | jq -r .items[0].metadata.name) - -# This will copy configs / create directories for it -$ kubectl cp -n flux-operator scripts/run-experiments.py ${POD}:/home/flux/examples/reaxff/HNS/run-experiments.py -c flux-sample -``` - -```console -$ kubectl exec -it -n flux-operator ${POD} bash -``` - -```console -source /etc/profile.d/z10_spack_environment.sh -asFlux="sudo -u flux -E PYTHONPATH=$PYTHONPATH -E PATH=$PATH -E FI_PROVIDER=efa -E OMPI_MCA_btl=self,ofi -E RDMAV_FORK_SAFE=1 -E FI_EFA_USE_DEVICE_RDMA=1" -. /etc/profile.d/z10_spack_environment.sh -cd /opt/spack-environment -. /opt/spack-environment/spack/share/spack/setup-env.sh -spack env activate . -cd /home/flux/examples/reaxff/HNS -``` - -```console -$ sudo -u flux -E PYTHONPATH=$PYTHONPATH -E PATH=$PATH -E HOME=/home/flux -E FI_PROVIDER=efa -E OMPI_MCA_btl=self,ofi -E RDMAV_FORK_SAFE=1 -E FI_EFA_USE_DEVICE_RDMA=1 flux proxy local:///run/flux/local bash -``` - -```console -$ flux resource list -``` - -```console -export FI_PROVIDER=efa -export OMPI_MCA_btl=self,ofi -export RDMAV_FORK_SAFE=1 -export FI_EFA_USE_DEVICE_RDMA=1 -``` - -```console -flux mini submit -N 8 -n 512 --quiet -ompi=openmpi@5 -c 1 -o cpu-affinity=per-task --watch -vvv lmp -v x 1 -v y 1 -v z 1 -in in.reaxc.hns -nocite - -python3 run-experiments.py --outdir /home/flux --workdir /opt/lammps/examples/reaxff/HNS --times 20 -N 8 --tasks 512 lmp -v x 64 -v y 16 -v z 16 -in in.reaxc.hns -nocite -``` - -Follow this [link](https://github.com/converged-computing/operator-experiments/tree/main/aws/lammps/hpc7g/run2) for more information. @ diff --git a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/efa-device-plugin.yaml b/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/efa-device-plugin.yaml deleted file mode 100644 index fe606fc..0000000 --- a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/efa-device-plugin.yaml +++ /dev/null @@ -1,175 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: aws-efa-k8s-device-plugin-daemonset - namespace: kube-system -spec: - selector: - matchLabels: - name: aws-efa-k8s-device-plugin - updateStrategy: - type: RollingUpdate - template: - metadata: - # This annotation is deprecated. Kept here for backward compatibility - # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - name: aws-efa-k8s-device-plugin - spec: - serviceAccount: default - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: aws.amazon.com/efa - operator: Exists - effect: NoSchedule - # Mark this pod as a critical add-on; when enabled, the critical add-on - # scheduler reserves resources for critical add-on pods so that they can - # be rescheduled after a failure. - # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - priorityClassName: "system-node-critical" - affinity: - nodeAffinity: - # EFA supported instances: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html#efa-instance-types - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "node.kubernetes.io/instance-type" - operator: In - values: - - c5n.18xlarge - - c5n.9xlarge - - c5n.metal - - c6a.32xlarge - - c6a.48xlarge - - c6a.metal - - c6gn.16xlarge - - c6i.32xlarge - - c6i.metal - - c6id.32xlarge - - c6id.metal - - dl1.24xlarge - - g4dn.12xlarge - - g4dn.8xlarge - - g4dn.metal - - g5.48xlarge - - hpc6a.48xlarge - - hpc7g.16xlarge - - hpc7g.8xlarge - - hpc7g.4xlarge - - i3en.12xlarge - - i3en.24xlarge - - i3en.metal - - i4i.32xlarge - - i4i.metal - - im4gn.16xlarge - - inf1.24xlarge - - m5dn.24xlarge - - m5dn.metal - - m5n.24xlarge - - m5n.metal - - m5zn.12xlarge - - m5zn.metal - - m6a.32xlarge - - m6a.48xlarge - - m6a.metal - - m6i.32xlarge - - m6i.metal - - m6id.32xlarge - - m6id.metal - - p3dn.24xlarge - - p4d.24xlarge - - p4de.24xlarge - - r5dn.24xlarge - - r5dn.metal - - r5n.24xlarge - - r5n.metal - - r6i.32xlarge - - r6i.metal - - vt1.24xlarge - - x2idn.32xlarge - - x2idn.metal - - x2iedn.32xlarge - - x2iedn.metal - - x2iezn.12xlarge - - x2iezn.metal - - matchExpressions: - - key: "node.kubernetes.io/instance-type" - operator: In - values: - - c5n.18xlarge - - c5n.9xlarge - - c5n.metal - - c6a.32xlarge - - c6a.48xlarge - - c6a.metal - - c6gn.16xlarge - - c6i.32xlarge - - c6i.metal - - c6id.32xlarge - - c6id.metal - - dl1.24xlarge - - g4dn.12xlarge - - g4dn.8xlarge - - g4dn.metal - - g5.48xlarge - - hpc6a.48xlarge - - hpc7g.16xlarge - - hpc7g.8xlarge - - hpc7g.4xlarge - - i3en.12xlarge - - i3en.24xlarge - - i3en.metal - - i4i.32xlarge - - i4i.metal - - im4gn.16xlarge - - inf1.24xlarge - - m5dn.24xlarge - - m5dn.metal - - m5n.24xlarge - - m5n.metal - - m5zn.12xlarge - - m5zn.metal - - m6a.32xlarge - - m6a.48xlarge - - m6a.metal - - m6i.32xlarge - - m6i.metal - - m6id.32xlarge - - m6id.metal - - p3dn.24xlarge - - p4d.24xlarge - - p4de.24xlarge - - r5dn.24xlarge - - r5dn.metal - - r5n.24xlarge - - r5n.metal - - r6i.32xlarge - - r6i.metal - - vt1.24xlarge - - x2idn.32xlarge - - x2idn.metal - - x2iedn.32xlarge - - x2iedn.metal - - x2iezn.12xlarge - - x2iezn.metal - hostNetwork: true - containers: - - image: 602401143452.dkr.ecr.us-east-1.amazonaws.com/eks/aws-efa-k8s-device-plugin:v0.3.3 - # This would be populated by eksctl, but doing this manually you can populate it! - # - image: "%s.dkr.ecr.%s.%s/eks/aws-efa-k8s-device-plugin:v0.3.3" - name: aws-efa-k8s-device-plugin - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - volumes: - - name: device-plugin - hostPath: - path: /var/lib/kubelet/device-plugins diff --git a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/eks-efa-cluster-config-hpc6a.yaml b/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/eks-efa-cluster-config-hpc6a.yaml deleted file mode 100644 index 8fc876a..0000000 --- a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/eks-efa-cluster-config-hpc6a.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig - -metadata: - name: scaling-study-efa - region: us-east-2 - version: "1.23" - -availabilityZones: ["us-east-2b", "us-east-2c"] - -iam: - withOIDC: true - serviceAccounts: - - metadata: - name: cluster-autoscaler - namespace: kube-system - attachPolicy: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - "autoscaling:DescribeAutoScalingGroups" - - "autoscaling:DescribeAutoScalingInstances" - - "autoscaling:DescribeLaunchConfigurations" - - "autoscaling:DescribeTags" - - "autoscaling:SetDesiredCapacity" - - "autoscaling:TerminateInstanceInAutoScalingGroup" - - "ec2:DescribeLaunchTemplateVersions" - Resource: '*' - -managedNodeGroups: - - name: workers - availabilityZones: ["us-east-2b"] - instanceType: hpc6a.48xlarge - minSize: 1 - maxSize: 10 - desiredCapacity: 2 - efaEnabled: true - placement: - groupName: eks-efa-testing - labels: { "flux-operator": "true" } - tags: - k8s.io/cluster-autoscaler/enabled: "true" - k8s.io/cluster-autoscaler/scaling-study-efa: "owned" - ssh: - allow: true - publicKeyPath: ~/.ssh/id_rsa.pub diff --git a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/flux-operator.yaml b/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/flux-operator.yaml deleted file mode 100644 index f6d50b1..0000000 --- a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/flux-operator.yaml +++ /dev/null @@ -1,1412 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: operator-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null - name: miniclusters.flux-framework.org -spec: - group: flux-framework.org - names: - kind: MiniCluster - listKind: MiniClusterList - plural: miniclusters - singular: minicluster - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: MiniCluster is the Schema for a Flux job launcher on K8s - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: MiniCluster is an HPC cluster in Kubernetes you can control - Either to submit a single job (and go away) or for a persistent single- - or multi- user cluster - properties: - archive: - description: Archive to load or save - properties: - path: - description: Save or load from this directory path - type: string - type: object - cleanup: - default: false - description: Cleanup the pods and storage when the index broker pod - is complete - type: boolean - containers: - description: Containers is one or more containers to be created in - a pod. There should only be one container to run flux with runFlux - items: - properties: - batch: - description: Indicate that the command is a batch job that will - be written to a file to submit - type: boolean - batchRaw: - description: Don't wrap batch commands in flux submit (provide - custom logic myself) - type: boolean - command: - description: Single user executable to provide to flux start - type: string - commands: - description: More specific or detailed commands for just workers/broker - properties: - brokerPre: - description: A single command for only the broker to run - type: string - init: - description: init command is run before anything - type: string - post: - description: post command is run in the entrypoint when - the broker exits / finishes - type: string - pre: - description: pre command is run after global PreCommand, - after asFlux is set (can override) - type: string - prefix: - description: Prefix to flux start / submit / broker Typically - used for a wrapper command to mount, etc. - type: string - runFluxAsRoot: - default: false - description: Run flux start as root - required for some - storage binds - type: boolean - workerPre: - description: A command only for workers to run - type: string - type: object - cores: - description: Cores the container should use - format: int32 - type: integer - diagnostics: - description: Run flux diagnostics on start instead of command - type: boolean - environment: - additionalProperties: - type: string - description: Key/value pairs for the environment - type: object - existingVolumes: - additionalProperties: - description: Mini Cluster local volumes available to mount - (these are on the host) - properties: - claimName: - description: Claim name if the existing volume is a PVC - type: string - configMapName: - description: Config map name if the existing volume is - a config map You should also define items if you are - using this - type: string - items: - additionalProperties: - type: string - description: Items (key and paths) for the config map - type: object - path: - description: Path and claim name are always required if - a secret isn't defined - type: string - readOnly: - default: false - type: boolean - secretName: - description: An existing secret - type: string - type: object - description: Existing Volumes to add to the containers - type: object - fluxUser: - description: Flux User, if created in the container - properties: - name: - default: flux - description: Flux user name - type: string - uid: - default: 1000 - description: UID for the FluxUser - type: integer - type: object - image: - default: ghcr.io/rse-ops/accounting:app-latest - description: Container image must contain flux and flux-sched - install - type: string - imagePullSecret: - description: Allow the user to pull authenticated images By - default no secret is selected. Setting this with the name - of an already existing imagePullSecret will specify that secret - in the pod spec. - type: string - launcher: - description: Indicate that the command is a launcher that will - ask for its own jobs (and provided directly to flux start) - type: boolean - lifeCycle: - description: Lifecycle can handle post start commands, etc. - properties: - postStartExec: - type: string - preStopExec: - type: string - type: object - logs: - description: Log output directory - type: string - name: - description: Container name is only required for non flux runners - type: string - ports: - description: Ports to be exposed to other containers in the - cluster We take a single list of integers and map to the same - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: atomic - preCommand: - description: Special command to run at beginning of script, - directly after asFlux is defined as sudo -u flux -E (so you - can change that if desired.) This is only valid if FluxRunner - is set (that writes a wait.sh script) This is for the indexed - job pods and the certificate generation container. - type: string - pullAlways: - default: false - description: Allow the user to dictate pulling By default we - pull if not present. Setting this to true will indicate to - pull always - type: boolean - resources: - description: Resources include limits and requests - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - type: object - runFlux: - description: Main container to run flux (only should be one) - type: boolean - secrets: - additionalProperties: - description: Secret describes a secret from the environment. - The envar name should be the key of the top level map. - properties: - key: - description: Key under secretKeyRef->Key - type: string - name: - description: Name under secretKeyRef->Name - type: string - required: - - key - - name - type: object - description: Secrets that will be added to the environment The - user is expected to create their own secrets for the operator - to find - type: object - securityContext: - description: Security Context https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - privileged: - description: Privileged container - type: boolean - type: object - volumes: - additionalProperties: - description: A Container volume must reference one defined - for the MiniCluster The path here is in the container - properties: - path: - type: string - readOnly: - default: false - type: boolean - required: - - path - type: object - description: Volumes that can be mounted (must be defined in - volumes) - type: object - workingDir: - description: Working directory to run command from - type: string - type: object - type: array - x-kubernetes-list-type: atomic - deadlineSeconds: - default: 31500000 - description: Should the job be limited to a particular number of seconds? - Approximately one year. This cannot be zero or job won't start - format: int64 - type: integer - flux: - description: Flux options for the broker, shared across cluster - properties: - brokerConfig: - description: Optionally provide a manually created broker config - this is intended for bursting to remote clusters - type: string - bursting: - description: Bursting - one or more external clusters to burst - to We assume a single, central MiniCluster with an ipaddress - that all connect to. - properties: - clusters: - description: External clusters to burst to. Each external - cluster must share the same listing to align ranks - items: - properties: - name: - description: The hostnames for the bursted clusters - If set, the user is responsible for ensuring uniqueness. - The operator will set to burst-N - type: string - size: - description: Size of bursted cluster. Defaults to same - size as local minicluster if not set - format: int32 - type: integer - type: object - type: array - hostlist: - description: Hostlist is a custom hostlist for the broker.toml - that includes the local plus bursted cluster. This is typically - used for bursting to another resource type, where we can - predict the hostnames but they don't follow the same convention - as the Flux Operator - type: string - leadBroker: - description: The lead broker ip address to join to. E.g., - if we burst to cluster 2, this is the address to connect - to cluster 1 For the first cluster, this should not be defined - properties: - address: - description: Lead broker address (ip or hostname) - type: string - name: - description: We need the name of the lead job to assemble - the hostnames - type: string - port: - default: 8050 - description: Lead broker port - should only be used for - external cluster - format: int32 - type: integer - size: - description: Lead broker size - format: int32 - type: integer - required: - - address - - name - - size - type: object - type: object - connectTimeout: - default: 5s - description: Single user executable to provide to flux start - type: string - curveCert: - description: Optionally provide an already existing curve certificate - This is not recommended in favor of providing the secret name - as curveCertSecret, below - type: string - curveCertSecret: - description: Expect a secret for a curve cert here. This is ideal - over the curveCert (as a string) above. - type: string - installRoot: - default: /usr - description: Install root location - type: string - logLevel: - default: 6 - description: Log level to use for flux logging (only in non TestMode) - format: int32 - type: integer - minimalService: - description: Only expose the broker service (to reduce load on - DNS) - type: boolean - mungeSecret: - description: Expect a secret (named according to this string) - for a munge key. This is intended for bursting. Assumed to be - at /etc/munge/munge.key This is binary data. - type: string - optionFlags: - description: Flux option flags, usually provided with -o optional - - if needed, default option flags for the server These can also - be set in the user interface to override here. This is only - valid for a FluxRunner "runFlux" true - type: string - wrap: - description: Commands for flux start --wrap - type: string - type: object - fluxRestful: - description: Customization to Flux Restful API There should only be - one container to run flux with runFlux - properties: - branch: - default: main - description: Branch to clone Flux Restful API from - type: string - port: - default: 5000 - description: Port to run Flux Restful Server On - format: int32 - type: integer - secretKey: - description: Secret key shared between server and client - type: string - token: - description: Token to use for RestFul API - type: string - username: - description: These two should not actually be set by a user, but - rather generated by tools and provided Username to use for RestFul - API - type: string - type: object - interactive: - default: false - description: Run a single-user, interactive minicluster - type: boolean - jobLabels: - additionalProperties: - type: string - description: Labels for the job - type: object - logging: - description: Logging modes determine the output you see in the job - log - properties: - debug: - default: false - description: Debug mode adds extra verbosity to Flux - type: boolean - quiet: - default: false - description: Quiet mode silences all output so the job only shows - the test running - type: boolean - strict: - default: true - description: Strict mode ensures any failure will not continue - in the job entrypoint - type: boolean - timed: - default: false - description: Timed mode adds timing to Flux commands - type: boolean - zeromq: - default: false - description: Enable Zeromq logging - type: boolean - type: object - maxSize: - description: MaxSize (maximum number of pods to allow scaling to) - format: int32 - type: integer - network: - description: A spec for exposing or defining the cluster headless - service - properties: - headlessName: - default: flux-service - description: Name for cluster headless service - type: string - type: object - pod: - description: Pod spec details - properties: - annotations: - additionalProperties: - type: string - description: Annotations for each pod - type: object - labels: - additionalProperties: - type: string - description: Labels for each pod - type: object - nodeSelector: - additionalProperties: - type: string - description: NodeSelectors for a pod - type: object - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - description: Resources include limits and requests - type: object - serviceAccountName: - description: Service account name for the pod - type: string - type: object - services: - description: Services are one or more service containers to bring - up alongside the MiniCluster. - items: - properties: - batch: - description: Indicate that the command is a batch job that will - be written to a file to submit - type: boolean - batchRaw: - description: Don't wrap batch commands in flux submit (provide - custom logic myself) - type: boolean - command: - description: Single user executable to provide to flux start - type: string - commands: - description: More specific or detailed commands for just workers/broker - properties: - brokerPre: - description: A single command for only the broker to run - type: string - init: - description: init command is run before anything - type: string - post: - description: post command is run in the entrypoint when - the broker exits / finishes - type: string - pre: - description: pre command is run after global PreCommand, - after asFlux is set (can override) - type: string - prefix: - description: Prefix to flux start / submit / broker Typically - used for a wrapper command to mount, etc. - type: string - runFluxAsRoot: - default: false - description: Run flux start as root - required for some - storage binds - type: boolean - workerPre: - description: A command only for workers to run - type: string - type: object - cores: - description: Cores the container should use - format: int32 - type: integer - diagnostics: - description: Run flux diagnostics on start instead of command - type: boolean - environment: - additionalProperties: - type: string - description: Key/value pairs for the environment - type: object - existingVolumes: - additionalProperties: - description: Mini Cluster local volumes available to mount - (these are on the host) - properties: - claimName: - description: Claim name if the existing volume is a PVC - type: string - configMapName: - description: Config map name if the existing volume is - a config map You should also define items if you are - using this - type: string - items: - additionalProperties: - type: string - description: Items (key and paths) for the config map - type: object - path: - description: Path and claim name are always required if - a secret isn't defined - type: string - readOnly: - default: false - type: boolean - secretName: - description: An existing secret - type: string - type: object - description: Existing Volumes to add to the containers - type: object - fluxUser: - description: Flux User, if created in the container - properties: - name: - default: flux - description: Flux user name - type: string - uid: - default: 1000 - description: UID for the FluxUser - type: integer - type: object - image: - default: ghcr.io/rse-ops/accounting:app-latest - description: Container image must contain flux and flux-sched - install - type: string - imagePullSecret: - description: Allow the user to pull authenticated images By - default no secret is selected. Setting this with the name - of an already existing imagePullSecret will specify that secret - in the pod spec. - type: string - launcher: - description: Indicate that the command is a launcher that will - ask for its own jobs (and provided directly to flux start) - type: boolean - lifeCycle: - description: Lifecycle can handle post start commands, etc. - properties: - postStartExec: - type: string - preStopExec: - type: string - type: object - logs: - description: Log output directory - type: string - name: - description: Container name is only required for non flux runners - type: string - ports: - description: Ports to be exposed to other containers in the - cluster We take a single list of integers and map to the same - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: atomic - preCommand: - description: Special command to run at beginning of script, - directly after asFlux is defined as sudo -u flux -E (so you - can change that if desired.) This is only valid if FluxRunner - is set (that writes a wait.sh script) This is for the indexed - job pods and the certificate generation container. - type: string - pullAlways: - default: false - description: Allow the user to dictate pulling By default we - pull if not present. Setting this to true will indicate to - pull always - type: boolean - resources: - description: Resources include limits and requests - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - type: object - runFlux: - description: Main container to run flux (only should be one) - type: boolean - secrets: - additionalProperties: - description: Secret describes a secret from the environment. - The envar name should be the key of the top level map. - properties: - key: - description: Key under secretKeyRef->Key - type: string - name: - description: Name under secretKeyRef->Name - type: string - required: - - key - - name - type: object - description: Secrets that will be added to the environment The - user is expected to create their own secrets for the operator - to find - type: object - securityContext: - description: Security Context https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - privileged: - description: Privileged container - type: boolean - type: object - volumes: - additionalProperties: - description: A Container volume must reference one defined - for the MiniCluster The path here is in the container - properties: - path: - type: string - readOnly: - default: false - type: boolean - required: - - path - type: object - description: Volumes that can be mounted (must be defined in - volumes) - type: object - workingDir: - description: Working directory to run command from - type: string - type: object - type: array - x-kubernetes-list-type: atomic - size: - default: 1 - description: Size (number of job pods to run, size of minicluster - in pods) This is also the minimum number required to start Flux - format: int32 - type: integer - tasks: - default: 1 - description: Total number of CPUs being run across entire cluster - format: int32 - type: integer - users: - description: Users of the MiniCluster - items: - properties: - name: - description: If a user is defined, the username is required - type: string - password: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-type: atomic - volumes: - additionalProperties: - description: Mini Cluster local volumes available to mount (these - are on the host) - properties: - annotations: - additionalProperties: - type: string - description: Annotations for the volume - type: object - attributes: - additionalProperties: - type: string - description: Optional volume attributes - type: object - capacity: - default: 5Gi - description: Capacity (string) for PVC (storage request) to - create PV - type: string - claimAnnotations: - additionalProperties: - type: string - description: Annotations for the persistent volume claim - type: object - delete: - default: true - description: Delete the persistent volume on cleanup - type: boolean - driver: - description: Storage driver, e.g., gcs.csi.ofek.dev Only needed - if not using hostpath - type: string - labels: - additionalProperties: - type: string - type: object - path: - type: string - secret: - description: Secret reference in Kubernetes with service account - role - type: string - secretNamespace: - default: default - description: Secret namespace - type: string - storageClass: - default: hostpath - type: string - volumeHandle: - description: Volume handle, falls back to storage class name - if not defined - type: string - required: - - path - type: object - description: Volumes accessible to containers from a host Not all - containers are required to use them - type: object - required: - - containers - type: object - status: - description: MiniClusterStatus defines the observed state of Flux - properties: - conditions: - description: conditions hold the latest Flux Job and MiniCluster states - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-type: atomic - jobid: - description: The Jobid is set internally to associate to a miniCluster - This isn't currently in use, we only have one! - type: string - maximumSize: - description: We keep the original size of the MiniCluster request - as this is the absolute maximum - format: int32 - type: integer - selector: - type: string - size: - description: These are for the sub-resource scale functionality - format: int32 - type: integer - required: - - jobid - - maximumSize - - selector - - size - type: object - type: object - served: true - storage: true - subresources: - scale: - labelSelectorPath: .status.selector - specReplicasPath: .spec.size - statusReplicasPath: .status.size - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: operator-leader-election-role - namespace: operator-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: operator-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - watch -- apiGroups: - - "" - resources: - - events - - nodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - exec - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - create - - delete - - exec - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - "" - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - batch - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - networks - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/exec - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/log - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - clusters - - clusters/status - verbs: - - get - - list - - watch -- apiGroups: - - flux-framework.org - resources: - - machineclasses - - machinedeployments - - machinedeployments/status - - machines - - machines/status - - machinesets - - machinesets/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - miniclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - miniclusters/finalizers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - miniclusters/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: operator-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: operator-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: operator-leader-election-rolebinding - namespace: operator-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: operator-leader-election-role -subjects: -- kind: ServiceAccount - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: operator-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: operator-manager-role -subjects: -- kind: ServiceAccount - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: operator-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: operator-proxy-role -subjects: -- kind: ServiceAccount - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: v1 -data: - controller_manager_config.yaml: | - apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 - kind: ControllerManagerConfig - health: - healthProbeBindAddress: :8081 - metrics: - bindAddress: 127.0.0.1:8080 - webhook: - port: 9443 - leaderElection: - leaderElect: true - resourceName: 14dde902.flux-framework.org -kind: ConfigMap -metadata: - name: operator-manager-config - namespace: operator-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - name: operator-controller-manager-metrics-service - namespace: operator-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - control-plane: controller-manager - name: operator-controller-manager - namespace: operator-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.11.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /manager - image: ghcr.io/flux-framework/flux-operator:latest - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - securityContext: - runAsNonRoot: true - serviceAccountName: operator-controller-manager - terminationGracePeriodSeconds: 10 diff --git a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/minicluster.yaml b/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/minicluster.yaml deleted file mode 100644 index 24f4ce0..0000000 --- a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc6a-configs/minicluster.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: flux-framework.org/v1alpha1 -kind: MiniCluster -metadata: - name: flux-sample - namespace: flux-operator -spec: - - # Number of pods to create for MiniCluster - # 12 * 64 - size: 2 - maxSize: 10 - - # This doesn't matter so much in interactive mode, tasks specified with the job - tasks: 188 - - # This starts the flux broker without a command (interactive) - interactive: true - logging: - quiet: false - strict: false - zeromq: true - debug: true - - # Spack installs to /opt/view, without flux-security so no imp - flux: - installRoot: /opt/view - optionFlags: "-ompi=openmpi@5 -c 1 -o cpu-affinity=per-task" - - # This is a list because a pod can support multiple containers - containers: - - image: ghcr.io/rse-ops/lammps-efa:ubuntu-20.04 - workingDir: /home/flux/examples/reaxff/HNS - - # This shouldn't be needed since we are launching manually - # cores: 64 - - # This would be the problem size we used for 8 - command: lmp -v x 64 -v y 16 -v z 16 -in in.reaxc.hns -nocite - - # Resource limits to ensure 1 pod assigned per node - # These are purposefully lower - the actual value didn't work, but - # this should still assign 1:1. - resources: - limits: - vpc.amazonaws.com/efa: 1 - memory: "340G" - cpu: "94" - - requests: - vpc.amazonaws.com/efa: 1 - memory: "340G" # This is actually 128 - cpu: "94" - - # We still need to setup spack so flux can start the broker! - commands: - # The workers need to come up after the broker and network - we are hitting this issue - # when we get to this larger scale (and it is compounded by the new networking issue with the service) - # that appeared after we removed the certificate generation pod - workerPre: sleep 60 - pre: | - source /etc/profile.d/z10_spack_environment.sh - asFlux="sudo -u flux -E PYTHONPATH=$PYTHONPATH -E PATH=$PATH -E FI_EFA_USE_DEVICE_RDMA=1 -E RDMAV_FORK_SAFE=1" - #asFlux="sudo -u flux -E PYTHONPATH=$PYTHONPATH -E PATH=$PATH" - . /etc/profile.d/z10_spack_environment.sh - cd /opt/spack-environment - . /opt/spack-environment/spack/share/spack/setup-env.sh - spack env activate . - cd /home/flux/examples/reaxff/HNS diff --git a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/eks-efa-cluster-config-hpc7g.yaml b/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/eks-efa-cluster-config-hpc7g.yaml deleted file mode 100644 index a25fc16..0000000 --- a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/eks-efa-cluster-config-hpc7g.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig - -metadata: - name: scaling-study-efa-hossen1-08142023 - region: us-east-1 - version: "1.27" - -iam: - withOIDC: true - serviceAccounts: - - metadata: - name: cluster-autoscaler - namespace: kube-system - attachPolicy: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - "autoscaling:DescribeAutoScalingGroups" - - "autoscaling:DescribeAutoScalingInstances" - - "autoscaling:DescribeLaunchConfigurations" - - "autoscaling:DescribeTags" - - "autoscaling:SetDesiredCapacity" - - "autoscaling:TerminateInstanceInAutoScalingGroup" - - "ec2:DescribeLaunchTemplateVersions" - Resource: '*' - -availabilityZones: ["us-east-1a", "us-east-1b"] -managedNodeGroups: - - name: scaling-study-efa-hossen1-08142023 - availabilityZones: ["us-east-1a"] - instanceType: hpc7g.16xlarge - minSize: 8 - maxSize: 50 - desiredCapacity: 8 - efaEnabled: true - placement: - groupName: eks-efa-testing - labels: { "flux-operator": "true" } - tags: - k8s.io/cluster-autoscaler/enabled: "true" - k8s.io/cluster-autoscaler/scaling-study-efa: "owned" - Name: 'scaling-study-efa-hossen1-08142023' - propagateASGTags: true - ssh: - allow: true - publicKeyPath: ~/.ssh/id_rsa.pub diff --git a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/flux-operator-arm.yaml b/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/flux-operator-arm.yaml deleted file mode 100644 index 5cd3acf..0000000 --- a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/flux-operator-arm.yaml +++ /dev/null @@ -1,1419 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: operator-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null - name: miniclusters.flux-framework.org -spec: - group: flux-framework.org - names: - kind: MiniCluster - listKind: MiniClusterList - plural: miniclusters - singular: minicluster - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: MiniCluster is the Schema for a Flux job launcher on K8s - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: MiniCluster is an HPC cluster in Kubernetes you can control - Either to submit a single job (and go away) or for a persistent single- - or multi- user cluster - properties: - archive: - description: Archive to load or save - properties: - path: - description: Save or load from this directory path - type: string - type: object - cleanup: - default: false - description: Cleanup the pods and storage when the index broker pod - is complete - type: boolean - containers: - description: Containers is one or more containers to be created in - a pod. There should only be one container to run flux with runFlux - items: - properties: - batch: - description: Indicate that the command is a batch job that will - be written to a file to submit - type: boolean - batchRaw: - description: Don't wrap batch commands in flux submit (provide - custom logic myself) - type: boolean - command: - description: Single user executable to provide to flux start - type: string - commands: - description: More specific or detailed commands for just workers/broker - properties: - brokerPre: - description: A single command for only the broker to run - type: string - init: - description: init command is run before anything - type: string - post: - description: post command is run in the entrypoint when - the broker exits / finishes - type: string - pre: - description: pre command is run after global PreCommand, - after asFlux is set (can override) - type: string - prefix: - description: Prefix to flux start / submit / broker Typically - used for a wrapper command to mount, etc. - type: string - runFluxAsRoot: - default: false - description: Run flux start as root - required for some - storage binds - type: boolean - workerPre: - description: A command only for workers to run - type: string - type: object - cores: - description: Cores the container should use - format: int32 - type: integer - diagnostics: - description: Run flux diagnostics on start instead of command - type: boolean - environment: - additionalProperties: - type: string - description: Key/value pairs for the environment - type: object - existingVolumes: - additionalProperties: - description: Mini Cluster local volumes available to mount - (these are on the host) - properties: - claimName: - description: Claim name if the existing volume is a PVC - type: string - configMapName: - description: Config map name if the existing volume is - a config map You should also define items if you are - using this - type: string - items: - additionalProperties: - type: string - description: Items (key and paths) for the config map - type: object - path: - description: Path and claim name are always required if - a secret isn't defined - type: string - readOnly: - default: false - type: boolean - secretName: - description: An existing secret - type: string - type: object - description: Existing Volumes to add to the containers - type: object - fluxUser: - description: Flux User, if created in the container - properties: - name: - default: flux - description: Flux user name - type: string - uid: - default: 1000 - description: UID for the FluxUser - type: integer - type: object - image: - default: ghcr.io/rse-ops/accounting:app-latest - description: Container image must contain flux and flux-sched - install - type: string - imagePullSecret: - description: Allow the user to pull authenticated images By - default no secret is selected. Setting this with the name - of an already existing imagePullSecret will specify that secret - in the pod spec. - type: string - launcher: - description: Indicate that the command is a launcher that will - ask for its own jobs (and provided directly to flux start) - type: boolean - lifeCycle: - description: Lifecycle can handle post start commands, etc. - properties: - postStartExec: - type: string - preStopExec: - type: string - type: object - logs: - description: Log output directory - type: string - name: - description: Container name is only required for non flux runners - type: string - ports: - description: Ports to be exposed to other containers in the - cluster We take a single list of integers and map to the same - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: atomic - pullAlways: - default: false - description: Allow the user to dictate pulling By default we - pull if not present. Setting this to true will indicate to - pull always - type: boolean - resources: - description: Resources include limits and requests - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - type: object - runFlux: - description: Main container to run flux (only should be one) - type: boolean - secrets: - additionalProperties: - description: Secret describes a secret from the environment. - The envar name should be the key of the top level map. - properties: - key: - description: Key under secretKeyRef->Key - type: string - name: - description: Name under secretKeyRef->Name - type: string - required: - - key - - name - type: object - description: Secrets that will be added to the environment The - user is expected to create their own secrets for the operator - to find - type: object - securityContext: - description: Security Context https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - addCapabilities: - description: Capabilities to add - items: - type: string - type: array - privileged: - description: Privileged container - type: boolean - type: object - volumes: - additionalProperties: - description: A Container volume must reference one defined - for the MiniCluster The path here is in the container - properties: - path: - type: string - readOnly: - default: false - type: boolean - required: - - path - type: object - description: Volumes that can be mounted (must be defined in - volumes) - type: object - workingDir: - description: Working directory to run command from - type: string - type: object - type: array - x-kubernetes-list-type: atomic - deadlineSeconds: - default: 31500000 - description: Should the job be limited to a particular number of seconds? - Approximately one year. This cannot be zero or job won't start - format: int64 - type: integer - flux: - description: Flux options for the broker, shared across cluster - properties: - brokerConfig: - description: Optionally provide a manually created broker config - this is intended for bursting to remote clusters - type: string - bursting: - description: Bursting - one or more external clusters to burst - to We assume a single, central MiniCluster with an ipaddress - that all connect to. - properties: - clusters: - description: External clusters to burst to. Each external - cluster must share the same listing to align ranks - items: - properties: - name: - description: The hostnames for the bursted clusters - If set, the user is responsible for ensuring uniqueness. - The operator will set to burst-N - type: string - size: - description: Size of bursted cluster. Defaults to same - size as local minicluster if not set - format: int32 - type: integer - type: object - type: array - hostlist: - description: Hostlist is a custom hostlist for the broker.toml - that includes the local plus bursted cluster. This is typically - used for bursting to another resource type, where we can - predict the hostnames but they don't follow the same convention - as the Flux Operator - type: string - leadBroker: - description: The lead broker ip address to join to. E.g., - if we burst to cluster 2, this is the address to connect - to cluster 1 For the first cluster, this should not be defined - properties: - address: - description: Lead broker address (ip or hostname) - type: string - name: - description: We need the name of the lead job to assemble - the hostnames - type: string - port: - default: 8050 - description: Lead broker port - should only be used for - external cluster - format: int32 - type: integer - size: - description: Lead broker size - format: int32 - type: integer - required: - - address - - name - - size - type: object - type: object - connectTimeout: - default: 5s - description: Single user executable to provide to flux start - type: string - curveCert: - description: Optionally provide an already existing curve certificate - This is not recommended in favor of providing the secret name - as curveCertSecret, below - type: string - curveCertSecret: - description: Expect a secret for a curve cert here. This is ideal - over the curveCert (as a string) above. - type: string - installRoot: - default: /usr - description: Install root location - type: string - logLevel: - default: 6 - description: Log level to use for flux logging (only in non TestMode) - format: int32 - type: integer - minimalService: - description: Only expose the broker service (to reduce load on - DNS) - type: boolean - mungeSecret: - description: Expect a secret (named according to this string) - for a munge key. This is intended for bursting. Assumed to be - at /etc/munge/munge.key This is binary data. - type: string - optionFlags: - description: Flux option flags, usually provided with -o optional - - if needed, default option flags for the server These can also - be set in the user interface to override here. This is only - valid for a FluxRunner "runFlux" true - type: string - scheduler: - description: Custom attributes for the fluxion scheduler - properties: - queuePolicy: - description: Scheduler queue policy, defaults to "fcfs" can - also be "easy" - type: string - type: object - wrap: - description: Commands for flux start --wrap - type: string - type: object - fluxRestful: - description: Customization to Flux Restful API There should only be - one container to run flux with runFlux - properties: - branch: - default: main - description: Branch to clone Flux Restful API from - type: string - port: - default: 5000 - description: Port to run Flux Restful Server On - format: int32 - type: integer - secretKey: - description: Secret key shared between server and client - type: string - token: - description: Token to use for RestFul API - type: string - username: - description: These two should not actually be set by a user, but - rather generated by tools and provided Username to use for RestFul - API - type: string - type: object - interactive: - default: false - description: Run a single-user, interactive minicluster - type: boolean - jobLabels: - additionalProperties: - type: string - description: Labels for the job - type: object - logging: - description: Logging modes determine the output you see in the job - log - properties: - debug: - default: false - description: Debug mode adds extra verbosity to Flux - type: boolean - quiet: - default: false - description: Quiet mode silences all output so the job only shows - the test running - type: boolean - strict: - default: true - description: Strict mode ensures any failure will not continue - in the job entrypoint - type: boolean - timed: - default: false - description: Timed mode adds timing to Flux commands - type: boolean - zeromq: - default: false - description: Enable Zeromq logging - type: boolean - type: object - maxSize: - description: MaxSize (maximum number of pods to allow scaling to) - format: int32 - type: integer - network: - description: A spec for exposing or defining the cluster headless - service - properties: - headlessName: - default: flux-service - description: Name for cluster headless service - type: string - type: object - pod: - description: Pod spec details - properties: - annotations: - additionalProperties: - type: string - description: Annotations for each pod - type: object - labels: - additionalProperties: - type: string - description: Labels for each pod - type: object - nodeSelector: - additionalProperties: - type: string - description: NodeSelectors for a pod - type: object - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - description: Resources include limits and requests - type: object - serviceAccountName: - description: Service account name for the pod - type: string - type: object - services: - description: Services are one or more service containers to bring - up alongside the MiniCluster. - items: - properties: - batch: - description: Indicate that the command is a batch job that will - be written to a file to submit - type: boolean - batchRaw: - description: Don't wrap batch commands in flux submit (provide - custom logic myself) - type: boolean - command: - description: Single user executable to provide to flux start - type: string - commands: - description: More specific or detailed commands for just workers/broker - properties: - brokerPre: - description: A single command for only the broker to run - type: string - init: - description: init command is run before anything - type: string - post: - description: post command is run in the entrypoint when - the broker exits / finishes - type: string - pre: - description: pre command is run after global PreCommand, - after asFlux is set (can override) - type: string - prefix: - description: Prefix to flux start / submit / broker Typically - used for a wrapper command to mount, etc. - type: string - runFluxAsRoot: - default: false - description: Run flux start as root - required for some - storage binds - type: boolean - workerPre: - description: A command only for workers to run - type: string - type: object - cores: - description: Cores the container should use - format: int32 - type: integer - diagnostics: - description: Run flux diagnostics on start instead of command - type: boolean - environment: - additionalProperties: - type: string - description: Key/value pairs for the environment - type: object - existingVolumes: - additionalProperties: - description: Mini Cluster local volumes available to mount - (these are on the host) - properties: - claimName: - description: Claim name if the existing volume is a PVC - type: string - configMapName: - description: Config map name if the existing volume is - a config map You should also define items if you are - using this - type: string - items: - additionalProperties: - type: string - description: Items (key and paths) for the config map - type: object - path: - description: Path and claim name are always required if - a secret isn't defined - type: string - readOnly: - default: false - type: boolean - secretName: - description: An existing secret - type: string - type: object - description: Existing Volumes to add to the containers - type: object - fluxUser: - description: Flux User, if created in the container - properties: - name: - default: flux - description: Flux user name - type: string - uid: - default: 1000 - description: UID for the FluxUser - type: integer - type: object - image: - default: ghcr.io/rse-ops/accounting:app-latest - description: Container image must contain flux and flux-sched - install - type: string - imagePullSecret: - description: Allow the user to pull authenticated images By - default no secret is selected. Setting this with the name - of an already existing imagePullSecret will specify that secret - in the pod spec. - type: string - launcher: - description: Indicate that the command is a launcher that will - ask for its own jobs (and provided directly to flux start) - type: boolean - lifeCycle: - description: Lifecycle can handle post start commands, etc. - properties: - postStartExec: - type: string - preStopExec: - type: string - type: object - logs: - description: Log output directory - type: string - name: - description: Container name is only required for non flux runners - type: string - ports: - description: Ports to be exposed to other containers in the - cluster We take a single list of integers and map to the same - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: atomic - pullAlways: - default: false - description: Allow the user to dictate pulling By default we - pull if not present. Setting this to true will indicate to - pull always - type: boolean - resources: - description: Resources include limits and requests - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object - type: object - runFlux: - description: Main container to run flux (only should be one) - type: boolean - secrets: - additionalProperties: - description: Secret describes a secret from the environment. - The envar name should be the key of the top level map. - properties: - key: - description: Key under secretKeyRef->Key - type: string - name: - description: Name under secretKeyRef->Name - type: string - required: - - key - - name - type: object - description: Secrets that will be added to the environment The - user is expected to create their own secrets for the operator - to find - type: object - securityContext: - description: Security Context https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - addCapabilities: - description: Capabilities to add - items: - type: string - type: array - privileged: - description: Privileged container - type: boolean - type: object - volumes: - additionalProperties: - description: A Container volume must reference one defined - for the MiniCluster The path here is in the container - properties: - path: - type: string - readOnly: - default: false - type: boolean - required: - - path - type: object - description: Volumes that can be mounted (must be defined in - volumes) - type: object - workingDir: - description: Working directory to run command from - type: string - type: object - type: array - x-kubernetes-list-type: atomic - shareProcessNamespace: - description: Share process namespace? - type: boolean - size: - default: 1 - description: Size (number of job pods to run, size of minicluster - in pods) This is also the minimum number required to start Flux - format: int32 - type: integer - tasks: - default: 1 - description: Total number of CPUs being run across entire cluster - format: int32 - type: integer - users: - description: Users of the MiniCluster - items: - properties: - name: - description: If a user is defined, the username is required - type: string - password: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-type: atomic - volumes: - additionalProperties: - description: Mini Cluster local volumes available to mount (these - are on the host) - properties: - annotations: - additionalProperties: - type: string - description: Annotations for the volume - type: object - attributes: - additionalProperties: - type: string - description: Optional volume attributes - type: object - capacity: - default: 5Gi - description: Capacity (string) for PVC (storage request) to - create PV - type: string - claimAnnotations: - additionalProperties: - type: string - description: Annotations for the persistent volume claim - type: object - delete: - default: true - description: Delete the persistent volume on cleanup - type: boolean - driver: - description: Storage driver, e.g., gcs.csi.ofek.dev Only needed - if not using hostpath - type: string - labels: - additionalProperties: - type: string - type: object - path: - type: string - secret: - description: Secret reference in Kubernetes with service account - role - type: string - secretNamespace: - default: default - description: Secret namespace - type: string - storageClass: - default: hostpath - type: string - volumeHandle: - description: Volume handle, falls back to storage class name - if not defined - type: string - required: - - path - type: object - description: Volumes accessible to containers from a host Not all - containers are required to use them - type: object - required: - - containers - type: object - status: - description: MiniClusterStatus defines the observed state of Flux - properties: - conditions: - description: conditions hold the latest Flux Job and MiniCluster states - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-type: atomic - jobid: - description: The Jobid is set internally to associate to a miniCluster - This isn't currently in use, we only have one! - type: string - maximumSize: - description: We keep the original size of the MiniCluster request - as this is the absolute maximum - format: int32 - type: integer - selector: - type: string - size: - description: These are for the sub-resource scale functionality - format: int32 - type: integer - required: - - jobid - - maximumSize - - selector - - size - type: object - type: object - served: true - storage: true - subresources: - scale: - labelSelectorPath: .status.selector - specReplicasPath: .spec.size - statusReplicasPath: .status.size - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: operator-leader-election-role - namespace: operator-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: operator-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - watch -- apiGroups: - - "" - resources: - - events - - nodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - exec - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - create - - delete - - exec - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - "" - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - batch - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - networks - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/exec - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/log - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - clusters - - clusters/status - verbs: - - get - - list - - watch -- apiGroups: - - flux-framework.org - resources: - - machineclasses - - machinedeployments - - machinedeployments/status - - machines - - machines/status - - machinesets - - machinesets/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - miniclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - miniclusters/finalizers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - flux-framework.org - resources: - - miniclusters/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: operator-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: operator-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: operator-leader-election-rolebinding - namespace: operator-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: operator-leader-election-role -subjects: -- kind: ServiceAccount - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: operator-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: operator-manager-role -subjects: -- kind: ServiceAccount - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: operator-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: operator-proxy-role -subjects: -- kind: ServiceAccount - name: operator-controller-manager - namespace: operator-system ---- -apiVersion: v1 -data: - controller_manager_config.yaml: | - apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 - kind: ControllerManagerConfig - health: - healthProbeBindAddress: :8081 - metrics: - bindAddress: 127.0.0.1:8080 - webhook: - port: 9443 - leaderElection: - leaderElect: true - resourceName: 14dde902.flux-framework.org -kind: ConfigMap -metadata: - name: operator-manager-config - namespace: operator-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - name: operator-controller-manager-metrics-service - namespace: operator-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - control-plane: controller-manager - name: operator-controller-manager - namespace: operator-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.11.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /manager - image: ghcr.io/flux-framework/flux-operator:arm - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - securityContext: - runAsNonRoot: true - serviceAccountName: operator-controller-manager - terminationGracePeriodSeconds: 10 diff --git a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/minicluster-libfabric-new.yaml b/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/minicluster-libfabric-new.yaml deleted file mode 100644 index e4d6e63..0000000 --- a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/minicluster-libfabric-new.yaml +++ /dev/null @@ -1,70 +0,0 @@ -apiVersion: flux-framework.org/v1alpha1 -kind: MiniCluster -metadata: - name: flux-sample - namespace: flux-operator -spec: - - # Number of pods to create for MiniCluster - size: 8 - maxSize: 50 - - # This doesn't matter so much in interactive mode, tasks specified with the job - # 12 * 64 - # tasks: 128 - - # This starts the flux broker without a command (interactive) - interactive: true - logging: - quiet: false - strict: false - zeromq: true - debug: true - - # Spack installs to /opt/view, without flux-security so no imp - flux: - installRoot: /opt/view - optionFlags: "-ompi=openmpi@5 -c 1 -o cpu-affinity=per-task" - scheduler: - queuePolicy: easy - - containers: - - # image: ghcr.io/rse-ops/flux-arm-lammps:june-2023-arm64 - # spack 0.19 with a custom package.py that has updated libfabric - image: ghcr.io/rse-ops/flux-arm-lammps:libfabric-1.18.1 - workingDir: /home/flux/examples/reaxff/HNS - - # This shouldn't be needed since we are launching manually - # cores: 64 - - # This would be the problem size we used for 8 - # command: lmp -v x 64 -v y 16 -v z 16 -in in.reaxc.hns -nocite - - # Resource limits to ensure 1 pod assigned per node - # These are purposefully lower - the actual value didn't work, but - # this should still assign 1:1. - resources: - limits: - vpc.amazonaws.com/efa: 1 - memory: "110G" - cpu: "62" - - requests: - vpc.amazonaws.com/efa: 1 - memory: "110G" # This is actually 128 - cpu: "62" - - # We still need to setup spack so flux can start the broker! - commands: - # The workers need to come up after the broker and network - we are hitting this issue - # when we get to this larger scale (and it is compounded by the new networking issue with the service) - # that appeared after we removed the certificate generation pod - workerPre: sleep 60 - pre: | - source /etc/profile.d/z10_spack_environment.sh - asFlux="sudo -u flux -E PYTHONPATH=$PYTHONPATH -E PATH=$PATH -E FI_PROVIDER=efa -E OMPI_MCA_btl=self,ofi -E RDMAV_FORK_SAFE=1 -E FI_EFA_USE_DEVICE_RDMA=1" - . /etc/profile.d/z10_spack_environment.sh - cd /opt/spack-environment - . /opt/spack-environment/spack/share/spack/setup-env.sh - spack env activate . - cd /home/flux/examples/reaxff/HNS diff --git a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/minicluster-no-efa.yaml b/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/minicluster-no-efa.yaml deleted file mode 100644 index ed4858d..0000000 --- a/examples/flux_operator_ca_hpa/flux-with-lammps-setup/hpc7g-configs/minicluster-no-efa.yaml +++ /dev/null @@ -1,65 +0,0 @@ -apiVersion: flux-framework.org/v1alpha1 -kind: MiniCluster -metadata: - name: flux-sample - namespace: flux-operator -spec: - - # Number of pods to create for MiniCluster - size: 8 - - # This doesn't matter so much in interactive mode, tasks specified with the job - # 12 * 64 - # tasks: 768 - - # This starts the flux broker without a command (interactive) - interactive: true - logging: - quiet: false - strict: false - zeromq: true - debug: true - - # Spack installs to /opt/view, without flux-security so no imp - flux: - installRoot: /opt/view - optionFlags: "-ompi=openmpi@5 -c 1 -o cpu-affinity=per-task" - - containers: - - # image: ghcr.io/rse-ops/flux-arm-lammps:june-2023-arm64 - # spack 0.19 with a custom package.py that has updated libfabric - image: ghcr.io/rse-ops/flux-arm-lammps:libfabric-1.18.1 - workingDir: /home/flux/examples/reaxff/HNS - - # This shouldn't be needed since we are launching manually - # cores: 64 - - # This would be the problem size we used for 8 - # command: lmp -v x 64 -v y 16 -v z 16 -in in.reaxc.hns -nocite - - # Resource limits to ensure 1 pod assigned per node - # These are purposefully lower - the actual value didn't work, but - # this should still assign 1:1. - resources: - limits: - memory: "110G" - cpu: "62" - - requests: - memory: "110G" # This is actually 128 - cpu: "62" - - # We still need to setup spack so flux can start the broker! - commands: - # The workers need to come up after the broker and network - we are hitting this issue - # when we get to this larger scale (and it is compounded by the new networking issue with the service) - # that appeared after we removed the certificate generation pod - workerPre: sleep 60 - pre: | - source /etc/profile.d/z10_spack_environment.sh - asFlux="sudo -u flux -E PYTHONPATH=$PYTHONPATH -E PATH=$PATH -E FI_PROVIDER=efa -E OMPI_MCA_btl=self,ofi -E RDMAV_FORK_SAFE=1 -E FI_EFA_USE_DEVICE_RDMA=1" - . /etc/profile.d/z10_spack_environment.sh - cd /opt/spack-environment - . /opt/spack-environment/spack/share/spack/setup-env.sh - spack env activate . - cd /home/flux/examples/reaxff/HNS diff --git a/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/README.md b/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/README.md deleted file mode 100644 index 9e395f0..0000000 --- a/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/README.md +++ /dev/null @@ -1,120 +0,0 @@ -# Horizontal Pod Autoscaling in Flux Operator Mini Cluster. - -Follow this link - [Flux Operator Elasticity](https://github.com/flux-framework/flux-operator/blob/24d54d7378d35d7a28e46bcf19fc74f796536f13/docs/tutorials/elasticity.md) for details setup and latest releases. This scripts is derived from the above link. Courtesy of [@vsoch](https://github.com/vsoch) - -Quick Access Link -1. [Flux Operator Elasticity](https://github.com/flux-framework/flux-operator/blob/24d54d7378d35d7a28e46bcf19fc74f796536f13/docs/tutorials/elasticity.md) - -### Horizontal Autoscaler (v2) Example - -The version 2 API is more flexible than version 1 in allowing custom metrics. This means we can use a [prometheus-flux](https://github.com/converged-computing/prometheus-flux) -exporter running inside of an instance to interact with it. This small set of tutorials will show setting a basic autoscaling example -based on CPU, and then one based on custom metrics. - - **[Tutorial File](https://github.com/flux-framework/flux-operator/blob/main/examples/elasticity/horizontal-autoscaler/v2-cpu/minicluster.yaml)** - -Look at the scale endpoint of the MiniCluster with `kubectl` directly! Remember that we haven't installed a horizontal auto-scaler yet: - -```console -$ kubectl get --raw /apis/flux-framework.org/v1alpha1/namespaces/flux-operator/miniclusters/flux-sample/scale | jq -``` - -```console -{ - "kind": "Scale", - "apiVersion": "autoscaling/v1", - "metadata": { - "name": "flux-sample", - "namespace": "flux-operator", - "uid": "581c708a-0eb2-48da-84b1-3da7679d349d", - "resourceVersion": "3579", - "creationTimestamp": "2023-05-20T05:11:28Z" - }, - "spec": { - "replicas": 2 - }, - "status": { - "replicas": 0, - "selector": "hpa-selector=flux-sample" - } -} -``` - -The above knows the selector to use to get pods (and look at current resource usage). -The output above is also telling us the `autoscaler/v1` is being used, which I used to think -means I could not use autoscaler/v2, but they seem to work OK (note that autoscaler/v2 is -installed to my current cluster with Kubernetes 1.27). - -### HPA Setup -Before we deploy any autoscaler, we need a metrics server! This doesn't come out of the box with kind so -we install it: -Note: You can look at aws documentation [here](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) - -```console -$ kubectl apply -f horizontal-pod-autoscaling/metrics-server.yaml -``` - -I found this suggestion [here](https://gist.github.com/sanketsudake/a089e691286bf2189bfedf295222bd43). Ensure -it's running: - -```bash -$ kubectl get deploy,svc -n kube-system | egrep metrics-server -``` - -#### Autoscaler with CPU - -This first autoscaler will work based on CPU. We can create it as follows: - - -```console -$ kubectl apply -f horizontal-pod-autoscaling/hpa-cpu.yaml -horizontalpodautoscaler.autoscaling/flux-sample-hpa created -``` - -Remember that when you first created your cluster, your size was two, and we had two? - -```bash -$ kubectl get -n flux-operator pods -NAME READY STATUS RESTARTS AGE -flux-sample-0-4wmmp 1/1 Running 0 6m50s -flux-sample-1-mjj7b 1/1 Running 0 6m50s -``` - -If you watch your pods (and your autoscaler and your endpoint) you'll -see first that the resource usage changes (just by way of Flux starting): -And to get it to change more, try shelling into your broker leader pod, connecting -to the broker, and issuing commands: - -```bash -$ kubectl exec -it -n flux-operator flux-sample-0-p85cj bash -$ sudo -u fluxuser -E $(env) -E HOME=/home/fluxuser flux proxy local:///run/flux/local bash -$ openssl speed -multi 4 -``` - -You'll see it change (with updates between 15 seconds and 1.5 minutes!): - -``` -$ kubectl get -n flux-operator hpa -w -NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE -flux-sample-hpa MiniCluster/flux-sample 0%/2% 2 4 2 33m -flux-sample-hpa MiniCluster/flux-sample 0%/2% 2 4 2 34m -flux-sample-hpa MiniCluster/flux-sample 3%/2% 2 4 2 34m -flux-sample-hpa MiniCluster/flux-sample 0%/2% 2 4 3 34m -``` - -With the openssl command above, I got it to hit a much higher load: - -```bash -flux-sample-hpa MiniCluster/flux-sample 0%/2% 2 4 4 7m30s -flux-sample-hpa MiniCluster/flux-sample 21%/2% 2 4 4 8m30s -flux-sample-hpa MiniCluster/flux-sample 25%/2% 2 4 4 8m46s -flux-sample-hpa MiniCluster/flux-sample 25%/2% 2 4 4 9m1s -``` - -See the [autoscaler/v1](#creating-the-v1-autoscaler) example for more detail about outputs. They have -a slightly different design, but result in the same output to the terminal. -When you are done demo-ing the CPU autoscaler, you can clean it up: - -```bash -$ kubectl delete -f hpa-cpu.yaml -``` diff --git a/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/hpa-cpu.yaml b/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/hpa-cpu.yaml deleted file mode 100644 index a3e0eea..0000000 --- a/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/hpa-cpu.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: flux-sample-hpa - namespace: flux-operator -spec: - scaleTargetRef: - apiVersion: flux-framework.org/v1alpha1 - kind: MiniCluster - name: flux-sample - minReplicas: 8 - maxReplicas: 16 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 diff --git a/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/hpa-simulations.py b/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/hpa-simulations.py deleted file mode 100644 index 29a9c22..0000000 --- a/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/hpa-simulations.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import csv -import math -import random - - -def get_parser(): - parser = argparse.ArgumentParser( - description="Horizontal Pod Autoscaling Desired Replica Counts simulation", - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "--current_replicas", - default=8, - type=int, - help="The current replica count of the applicatino", - ) - parser.add_argument( - "--current_metric_value_low", - default=95, - type=int, - help="Lowest anticipated average utilization of the PODS", - ) - parser.add_argument( - "--current_metric_value_high", - default=100, - type=int, - help="Highest anticipated average utilization of the PODS", - ) - parser.add_argument( - "--desired_metric_starting", - default=50, - type=int, - help="starting of desired metrics value, it will go up to 100", - ) - parser.add_argument( - "--repeat_simulation", - default=20, - type=int, - help="Number of times to repeat the simulation", - ) - parser.add_argument( - "--outdir", default=None, help="Filename to write the simulation results" - ) - return parser - - -def main(): - """ - Simulates Horizontal Pod Autoscaling Behavior for calculating the desired replicas with known CPU utilization - """ - parser = get_parser() - - # If an error occurs while parsing the arguments, the interpreter will exit with value 2 - args, _ = parser.parse_known_args() - - current_replicas = args.current_replicas - current_metric_value_low = args.current_metric_value_low - current_metric_value_high = args.current_metric_value_high - desired_metric_value_starting = args.desired_metric_starting - metric_upper_limit = 100 - - outfile = f"hpa-simulation-cr-{current_replicas}-dm-{desired_metric_value_starting}-repeat-{args.repeat_simulation}.csv" - - with open(outfile, "w", newline="") as csv_file: - writer = csv.writer(csv_file) - writer.writerow( - ["desired_metric_value", "current_metric_value", "desired_replicas"] - ) - for i in range(args.repeat_simulation): - for desired_metric_value in range( - desired_metric_value_starting, metric_upper_limit + 1 - ): - current_metric_value = random.uniform( - current_metric_value_low, current_metric_value_high - ) - desiredReplicas = math.ceil( - current_replicas * (current_metric_value / desired_metric_value) - ) - writer.writerow( - [desired_metric_value, current_metric_value, desiredReplicas] - ) - - -if __name__ == "__main__": - main() diff --git a/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/metrics-server.yaml b/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/metrics-server.yaml deleted file mode 100644 index 385f5d3..0000000 --- a/examples/flux_operator_ca_hpa/horizontal-pod-autoscaling/metrics-server.yaml +++ /dev/null @@ -1,197 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: system:aggregated-metrics-reader -rules: -- apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -rules: -- apiGroups: - - "" - resources: - - nodes/metrics - verbs: - - get -- apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server:system:auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - k8s-app: metrics-server ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: metrics-server - strategy: - rollingUpdate: - maxUnavailable: 0 - template: - metadata: - labels: - k8s-app: metrics-server - spec: - containers: - - args: - - --cert-dir=/tmp - - --secure-port=4443 - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - - --kubelet-insecure-tls - image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /livez - port: https - scheme: HTTPS - periodSeconds: 10 - name: metrics-server - ports: - - containerPort: 4443 - name: https - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 200Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - mountPath: /tmp - name: tmp-dir - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: metrics-server - volumes: - - emptyDir: {} - name: tmp-dir ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - labels: - k8s-app: metrics-server - name: v1beta1.metrics.k8s.io -spec: - group: metrics.k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: metrics-server - namespace: kube-system - version: v1beta1 - versionPriority: 100 diff --git a/examples/flux_operator_ca_hpa/k8s_cluster_operations.py b/examples/flux_operator_ca_hpa/k8s_cluster_operations.py deleted file mode 100644 index 4044ab3..0000000 --- a/examples/flux_operator_ca_hpa/k8s_cluster_operations.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import sys -import time - -from kubescaler.scaler.aws import EKSCluster - - -def get_parser(): - parser = argparse.ArgumentParser( - description="K8s Cluster Creator / Destroyer!", - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "cluster_name", - default="kubernetes-flux-operator", - help="Cluster name suffix", - nargs="?", - ) - parser.add_argument( - "--experiment", - default="hpa-ca-cluster", - help="Experiment name (defaults to script name)", - ) - parser.add_argument("--node-count", default=1, type=int, help="starting node count") - parser.add_argument( - "--max-node-count", - default=5, - type=int, - help="maximum node count", - ) - parser.add_argument( - "--min-node-count", default=1, type=int, help="minimum node count" - ) - parser.add_argument("--machine-type", default="m5.large", help="AWS machine type") - parser.add_argument( - "--operation", - default="create", - const="create", - nargs="?", - choices=["create", "delete", "scale"], - help="create or delete Cluster", - ) - parser.add_argument( - "--eks-nodegroup", - default=False, - action="store_true", - help="set this to use eks nodegroup for instances, otherwise, it'll use cloudformation stack", - ) - parser.add_argument( - "--enable-cluster-autoscaler", - default=False, - action="store_true", - help="set this to enable cluster autoscaling", - ) - return parser - - -def main(): - """ - Demonstrate creating and deleting a cluster. If the cluster exists, - we should be able to retrieve it and not create a second one. - """ - parser = get_parser() - - # If an error occurs while parsing the arguments, the interpreter will exit with value 2 - args, _ = parser.parse_known_args() - - # Pull cluster name out of argument - cluster_name = args.cluster_name - - # Derive the experiment name, either named or from script - experiment_name = args.experiment - if not experiment_name: - experiment_name = sys.argv[0].replace(".py", "") - time.sleep(2) - - # Update cluster name to include experiment name - cluster_name = f"{cluster_name}-{experiment_name}" - print(f"📛️ Cluster name is {cluster_name}") - - cli = EKSCluster( - name=cluster_name, - node_count=args.node_count, - max_nodes=args.max_node_count, - min_nodes=args.min_node_count, - machine_type=args.machine_type, - eks_nodegroup=args.eks_nodegroup, - enable_cluster_autoscaler=args.enable_cluster_autoscaler, - ) - - if args.operation == "create": - print( - f"⭐️ Creating the cluster sized {args.min_node_count} to {args.max_node_count}..." - ) - cluster_details = cli.create_cluster() - elif args.operation == "delete": - print("⭐️ Deleting the cluster...") - cli.delete_cluster() - elif args.operation == "scale": - print(f"Adding/Removing {args.node_count} from the cluster - {cluster_name}") - cluster_details = cli.load_cluster_info() - - if not cluster_details: - exit() - cli.scale(args.node_count) - else: - raise argparse.ArgumentError( - args.operation, "Please specify a valid operations the cluster" - ) - - -if __name__ == "__main__": - main() diff --git a/examples/flux_operator_ca_hpa/scripts/application_ca_hpa_metrics.py b/examples/flux_operator_ca_hpa/scripts/application_ca_hpa_metrics.py deleted file mode 100644 index 124bfed..0000000 --- a/examples/flux_operator_ca_hpa/scripts/application_ca_hpa_metrics.py +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import threading -from datetime import datetime, timezone - -from kubernetes import client as k8s -from kubernetes import config, watch - -import kubescaler.utils as utils - -# Save data here -here = os.path.dirname(os.path.abspath(__file__)) - -# Create data output directory -data = os.path.join(here, "data") - - -def datetime_utcnow_str(): - return datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def str_to_datetime(datetime_str_obj): - return datetime.strptime(datetime_str_obj, "%Y-%m-%dT%H:%M:%SZ").replace( - tzinfo=timezone.utc - ) - - -def watch_for_pod_events( - k8s_client, output_dir, namespace="flux-operator", outdir=None -): - index = 0 - pods_for_flux = {} - watcher = watch.Watch() - for event in watcher.stream( - func=k8s_client.list_namespaced_pod, namespace=namespace - ): - event_type = event["type"] - # object = event["object"] # object is one of type return_type - raw_object = event["raw_object"] # raw_object is a dict - - name = raw_object["metadata"]["name"] - status = raw_object["status"] - - # temporary save all objects for inspection. - utils.write_json( - raw_object, os.path.join(outdir, f"{name}-{event_type}-{str(index)}.json") - ) - - if event_type == "ADDED": - pods_for_flux[name] = {} - pods_for_flux[name]["created"] = raw_object["metadata"]["creationTimestamp"] - if raw_object["status"]["phase"] == "Running": - pods_for_flux[name]["waiting_for_scheduled"] = { - "current_status": True, - "start": None, - "end": None, - "duration": None, - } - pods_for_flux[name]["waiting_for_container"] = { - "current_status": True, - "start": None, - "end": None, - "duration": None, - } - else: - pods_for_flux[name]["waiting_for_scheduled"] = { - "current_status": False, - "start": datetime_utcnow_str(), - "end": None, - "duration": None, - } - pods_for_flux[name]["waiting_for_container"] = { - "current_status": False, - "start": datetime_utcnow_str(), - "end": None, - "duration": None, - } - - elif event_type == "MODIFIED": - for condition in status["conditions"]: - if ( - condition["type"] == "PodScheduled" - and condition["status"] == "True" - and ( - not pods_for_flux[name]["waiting_for_scheduled"][ - "current_status" - ] - ) - ): - pods_for_flux[name]["waiting_for_scheduled"][ - "current_status" - ] = True - pods_for_flux[name]["waiting_for_scheduled"][ - "end" - ] = datetime_utcnow_str() - pods_for_flux[name]["waiting_for_scheduled"]["duration"] = ( - datetime.now(tz=timezone.utc) - - str_to_datetime( - pods_for_flux[name]["waiting_for_scheduled"]["start"] - ) - ).total_seconds() - elif condition["type"] == "Initialized": - pass - elif condition["type"] == "Ready": - pass - elif condition["type"] == "ContainersReady": - pass - - if "containerStatuses" in status.keys(): - for container_status in status["containerStatuses"]: - if ( - container_status["name"] == "flux-sample" - and container_status["ready"] is True - and ( - not pods_for_flux[name]["waiting_for_container"][ - "current_status" - ] - ) - ): - pods_for_flux[name]["waiting_for_container"][ - "current_status" - ] = True - pods_for_flux[name]["waiting_for_container"][ - "end" - ] = datetime_utcnow_str() - pods_for_flux[name]["waiting_for_container"]["duration"] = ( - datetime.now(tz=timezone.utc) - - str_to_datetime( - pods_for_flux[name]["waiting_for_container"]["start"] - ) - ).total_seconds() - - elif event_type == "DELETED": - for container_status in status["containerStatuses"]: - pods_for_flux[name]["pod_started"] = container_status["state"][ - "terminated" - ]["startedAt"] - pods_for_flux[name]["pod_finished"] = container_status["state"][ - "terminated" - ]["finishedAt"] - pods_for_flux[name]["pod_container_duration"] = ( - str_to_datetime(pods_for_flux[name]["pod_finished"]) - - str_to_datetime(pods_for_flux[name]["pod_started"]) - ).total_seconds() - index += 1 - utils.write_json(pods_for_flux, output_dir) - print("🫛 POD Event") - - -def watch_cluster_autoscaler(k8s_client, output_dir, namespace="kube-system"): - cluster_autoscaler_pod_name = None - response = k8s_client.list_namespaced_pod(namespace=namespace) - - for pod in response.items: - if "cluster-autoscaler" in pod.metadata.name: - cluster_autoscaler_pod_name = pod.metadata.name - - print("🛺 Scaler Cluster event streaming started...") - watcher = watch.Watch() - for log_line in watcher.stream( - func=k8s_client.read_namespaced_pod_log, - name=cluster_autoscaler_pod_name, - namespace=namespace, - follow=True, - timestamps=True, - since_seconds=10, - ): - log_line += "\n" - utils.write_file(log_line, output_dir, mode="a") - - -def watch_hpa_events(autoscaling_v2, output_dir, namespace="flux-operator"): - watcher = watch.Watch() - event_data = [] - for event in watcher.stream( - func=autoscaling_v2.list_namespaced_horizontal_pod_autoscaler, - namespace=namespace, - ): - event_type = event["type"] - # object = event["object"] # object is one of type return_type - raw_object = event["raw_object"] # raw_object is a dict - - event_dict = {} - event_dict["event_type"] = event_type - event_dict["event_receive_time"] = datetime_utcnow_str() - event_dict["spec"] = raw_object["spec"] - event_dict["status"] = raw_object["status"] - - event_data.append(event_dict) - utils.write_json(event_data, output_dir) - print("↔️ ↔️ HPA Autoscaler events..") - - -def get_parser(): - parser = argparse.ArgumentParser( - description="K8s Scaling Experiment Runner", - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "--flux-namespace", - help="Namespace of the flux operator", - default="flux-operator", - ) - parser.add_argument( - "--autoscaler-namespace", - help="Namespace of the cluster autoscaler", - default="kube-system", - ) - parser.add_argument( - "--hpa-namespace", - help="Namespace of the horizontal pod autoscaler", - default="flux-operator", - ) - parser.add_argument( - "--kubeconfig", - help="kubernetes config file name, full path if the file is not in the current directory", - default="kubeconfig-aws.yaml", - ) - parser.add_argument( - "--outdir", help="Path for the experimental results", default=data - ) - parser.add_argument( - "--filedir", help="Directory name for the experiment", default=None - ) - return parser - - -def main(): - parser = get_parser() - - # If an error occurs while parsing the arguments, the interpreter will exit with value 2 - args, _ = parser.parse_known_args() - - # loading kubernetes config file and initializing client - # config.load_kube_config(config_file=args.kubeconfig) - config.load_kube_config() - coreV1 = k8s.CoreV1Api() - autoscalingV2 = k8s.AutoscalingV2Api() - - experiment_name = "kubernetes-pods-ca-hpa" - if args.filedir: - outdir = os.path.join(args.outdir, experiment_name, args.filedir) - else: - outdir = os.path.join(args.outdir, experiment_name, datetime_utcnow_str()) - if not os.path.exists(outdir): - print(f"📁️ Creating output directory {outdir}") - os.makedirs(outdir) - - pods_event_file = os.path.join(outdir, f"{args.flux_namespace}-pods-events.json") - ca_events_logs = os.path.join(outdir, f"{args.autoscaler_namespace}-ca-events.logs") - hpa_events_file = os.path.join( - outdir, f"{args.hpa_namespace}-hpa-status-events.json" - ) - - print("🍥 Starting the threads for collecting data") - - pod_events_thread = threading.Thread( - target=watch_for_pod_events, - args=(coreV1, pods_event_file, args.flux_namespace, outdir), - ) - ca_events_thread = threading.Thread( - target=watch_cluster_autoscaler, - args=(coreV1, ca_events_logs, args.autoscaler_namespace), - ) - hpa_events_thread = threading.Thread( - target=watch_hpa_events, - args=(autoscalingV2, hpa_events_file, args.hpa_namespace), - ) - - pod_events_thread.start() - ca_events_thread.start() - hpa_events_thread.start() - - pod_events_thread.join() - ca_events_thread.join() - hpa_events_thread.join() - - -if __name__ == "__main__": - main() diff --git a/examples/flux_operator_ca_hpa/scripts/flux-metrics.py b/examples/flux_operator_ca_hpa/scripts/flux-metrics.py deleted file mode 100644 index dff8d02..0000000 --- a/examples/flux_operator_ca_hpa/scripts/flux-metrics.py +++ /dev/null @@ -1,71 +0,0 @@ -try: - import flux - import flux.job - import flux.resource -except ImportError: - exit() - -# Keep a global handle so we make it just once -handle = flux.Flux() - - -def get_queue_metrics(): - """ - Update metrics for counts of jobs in the queue - - See https://github.com/flux-framework/flux-core/blob/master/src/common/libjob/job.h#L45-L53 - for identifiers. - """ - jobs = flux.job.job_list(handle) - listing = jobs.get() - # print(listing) - pending_jobs = 0 - for jobs in listing["jobs"]: - payload = {"id": jobs["id"], "attrs": ["all"]} - rpc = flux.job.list.JobListIdRPC(handle, "job-list.list-id", payload) - try: - jobinfo = rpc.get() - # The job does not exist, assume completed - except FileNotFoundError: - return "INACTIVE" - # print(jobinfo) - # User friendly string from integer - jobinfo = jobinfo["job"] - state = jobinfo["state"] - if state == 8: - pending_jobs += 1 - - # print(jobs['id'], flux.job.info.statetostr(state)) - return pending_jobs - - -def main(): - job_runtime = 120 # 120 seconds on average - estimated_node_uptime = 250 # in seconds from aws to kubernetes - max_allowable_nodes = 50 - - jobs_in_queue = get_queue_metrics() - - print(jobs_in_queue) - - estimated_total_job_runtime = job_runtime * jobs_in_queue - - current_nodes = 1 - while True: - if ( - (estimated_total_job_runtime - estimated_node_uptime) / current_nodes - ) > estimated_node_uptime: - current_nodes += 1 - else: - break - - if current_nodes * 8 > max_allowable_nodes: - print(int(max_allowable_nodes / current_nodes)) - else: - print(current_nodes) - - # perform kubernetes scaling operation on the minicluster - - -if __name__ == "__main__": - main() diff --git a/examples/flux_operator_ca_hpa/scripts/run-experiments-semi-auto.py b/examples/flux_operator_ca_hpa/scripts/run-experiments-semi-auto.py deleted file mode 100644 index b1ccbe2..0000000 --- a/examples/flux_operator_ca_hpa/scripts/run-experiments-semi-auto.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json -import os -import subprocess -import sys -import time - -here = os.path.abspath(os.path.dirname(__file__)) - -# Basic submit and monitor / saving of logs for multiple jobs -# Submit 10 times, each on 2 nodes -# cd /opt/lammps/examples/reaxff/HNS -# python run-experiments.py --workdir /opt/lammps/examples/reaxff/HNS --tasks 2 --times 10 -N 2 lmp -v x 1 -v y 1 -v z 1 -in in.reaxc.hns -nocite --outdir /home/scohat1/etc --identifier lammps - -# Make sure we can connect to the handle -try: - import flux - import flux.job - - handle = flux.Flux() - -except ImportError: - sys.exit("Cannot import flux, is the broker running?") - - -def get_parser(): - parser = argparse.ArgumentParser( - description="Flux Basic Experiment Runner", - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "-id", "--identifier", help="Identifier for the run", default="lammps" - ) - parser.add_argument("--workdir", help="working directory") - parser.add_argument( - "--outdir", - help="output directory for logs, etc.", - default=os.path.join(here, "data"), - ) - parser.add_argument("-N", help="number of nodes", type=int, default=1) - parser.add_argument( - "--sleep", help="sleep seconds waiting for jobs", type=int, default=10 - ) - parser.add_argument("--tasks", help="number of tasks", type=int, default=1) - parser.add_argument( - "--times", help="Number of times to run experiment command", type=int - ) - parser.add_argument("--index", help="Providing job number", type=int, default=0) - parser.add_argument( - "--dry-run", - action="store_true", - default=False, - help="Print example command and exit", - ) - return parser - - -def get_info(jobid): - """ - Get details for a job - """ - jobid = flux.job.JobID(jobid) - payload = {"id": jobid, "attrs": ["all"]} - rpc = flux.job.list.JobListIdRPC(handle, "job-list.list-id", payload) - try: - jobinfo = rpc.get() - - # The job does not exist! - except FileNotFoundError: - return None - - jobinfo = jobinfo["job"] - - # User friendly string from integer - state = jobinfo["state"] - jobinfo["state"] = flux.job.info.statetostr(state) - - # Get job info to add to result - info = rpc.get_jobinfo() - jobinfo["nnodes"] = info._nnodes - jobinfo["result"] = info.result - jobinfo["returncode"] = info.returncode - jobinfo["runtime"] = info.runtime - jobinfo["priority"] = info._priority - jobinfo["waitstatus"] = info._waitstatus - jobinfo["nodelist"] = info._nodelist - jobinfo["nodelist"] = info._nodelist - jobinfo["exception"] = info._exception.__dict__ - - # Only appears after finished? - if "duration" not in jobinfo: - jobinfo["duration"] = "" - return jobinfo - - -# Keep a global handle so we make it just once -handle = flux.Flux() - - -def get_queue_metrics(): - jobs = flux.job.job_list(handle) - listing = jobs.get() - # print(listing) - pending_jobs = 0 - for jobs in listing["jobs"]: - payload = {"id": jobs["id"], "attrs": ["all"]} - rpc = flux.job.list.JobListIdRPC(handle, "job-list.list-id", payload) - try: - jobinfo = rpc.get() - # The job does not exist, assume completed - except FileNotFoundError: - return "INACTIVE" - - jobinfo = jobinfo["job"] - state = jobinfo["state"] - if state == 8: - pending_jobs += 1 - - return pending_jobs - - -def calculate_node_count(): - job_runtime = 120 # 120 seconds on average - estimated_node_uptime = 250 # in seconds from aws to kubernetes - max_allowable_nodes = 50 - - jobs_in_queue = get_queue_metrics() - - print(f"Total jobs in the queue - {jobs_in_queue}") - - estimated_total_job_runtime = job_runtime * jobs_in_queue - - current_nodes = 1 - while True: - if ( - (estimated_total_job_runtime - estimated_node_uptime) / current_nodes - ) > estimated_node_uptime: - current_nodes += 1 - else: - break - - if (current_nodes * 8) > max_allowable_nodes: - print(f"Calculated nodes are - {int(max_allowable_nodes/8)}") - else: - print(f"Calculated nodes are - {current_nodes}") - - -def main(): - parser = get_parser() - - # If an error occurs while parsing the arguments, the interpreter will exit with value 2 - args, command = parser.parse_known_args() - - # Show args to the user - print(" N: %s" % args.N) - print(" times: %s" % args.times) - print(" Job Index: %s" % args.index) - print(" sleep: %s" % args.sleep) - print(" outdir: %s" % args.outdir) - print(" tasks: %s" % args.tasks) - print(" command: %s" % " ".join(command)) - print(" workdir: %s" % args.workdir) - print(" dry-run: %s" % args.dry_run) - print("identifier: %s" % args.identifier) - - # Ensure output directory exists - if not os.path.exists(args.outdir): - os.makedirs(args.outdir) - - def get_job_prefix(i): - identifier = f"{args.identifier}-{i}" - return os.path.join(args.outdir, identifier) - - # Hard code options for all setups - flux_options = [ - "-ompi=openmpi@5", - "-c", - "1", - "-o", - "cpu-affinity=per-task", - "-vvv", - ] - - # Submit all jobs - jobs = [] - for i in range(args.index, args.times + args.index): - prefix = get_job_prefix(i) - outfile = f"{prefix}.log" - - flux_command = ( - [ - "flux", - # The flux in the container is 0.44.0 - "mini", - "submit", - "-N", - str(args.N), - "-n", - str(args.tasks), - "--output", - outfile, - "--error", - outfile, - ] - + flux_options - + command - ) - - # If doing a dry run, stop here - print(" ".join(flux_command)) - if args.dry_run: - continue - - job = subprocess.Popen(flux_command, stdout=subprocess.PIPE) - jobid = job.communicate()[0] - # jobid = subprocess.check_output(flux_command) - jobid = jobid.decode("utf-8").strip() - count = i + 1 - print(f"Submit {jobid}: {count} of {args.times}") - jobs.append(jobid) - - # At this point all jobs are submit, and each should use all resources - # so we *should* be running one at a time. Now we can wait for each to save output, etc. - # Wait for futures - print("\n⭐️ Waiting for jobs to finish...") - print(f"Job ID's are - {jobs} \n") - for i, jobid in enumerate(jobs): - # calculate node count for semi autoscaling - calculate_node_count() - state = "RUN" - while state == "RUN": - info = get_info(jobid) - if info and info["state"] == "INACTIVE": - state = info["state"] - print( - f"No longer waiting on job {jobid}, FINISHED {info['returncode']}!" - ) - break - else: - print( - f"Still waiting for job {jobid} on {info['nodelist']}, has state {info['state']}" - ) - time.sleep(args.sleep) - - # When we get here, save all the metadata - prefix = get_job_prefix(i + args.index) - outfile = f"{prefix}-info.json" - with open(outfile, "w") as fd: - fd.write(json.dumps(info, indent=4)) - print("Jobs are complete, goodbye! 👋️") - - -if __name__ == "__main__": - main() diff --git a/examples/flux_operator_ca_hpa/scripts/run-experiments.py b/examples/flux_operator_ca_hpa/scripts/run-experiments.py deleted file mode 100644 index ae705d9..0000000 --- a/examples/flux_operator_ca_hpa/scripts/run-experiments.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json -import os -import subprocess -import sys -import time - -here = os.path.abspath(os.path.dirname(__file__)) - -# Basic submit and monitor / saving of logs for multiple jobs -# Submit 10 times, each on 2 nodes -# cd /opt/lammps/examples/reaxff/HNS -# python run-experiments.py --workdir /opt/lammps/examples/reaxff/HNS --tasks 2 --times 10 -N 2 lmp -v x 1 -v y 1 -v z 1 -in in.reaxc.hns -nocite --outdir /home/scohat1/etc --identifier lammps - -# Make sure we can connect to the handle -try: - import flux - import flux.job - - handle = flux.Flux() - -except ImportError: - sys.exit("Cannot import flux, is the broker running?") - - -def get_parser(): - parser = argparse.ArgumentParser( - description="Flux Basic Experiment Runner", - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "-id", "--identifier", help="Identifier for the run", default="lammps" - ) - parser.add_argument("--workdir", help="working directory") - parser.add_argument( - "--outdir", - help="output directory for logs, etc.", - default=os.path.join(here, "data"), - ) - parser.add_argument("-N", help="number of nodes", type=int, default=1) - parser.add_argument( - "--sleep", help="sleep seconds waiting for jobs", type=int, default=10 - ) - parser.add_argument("--tasks", help="number of tasks", type=int, default=1) - parser.add_argument( - "--times", help="Number of times to run experiment command", type=int - ) - parser.add_argument("--index", help="Providing job number", type=int, default=0) - parser.add_argument( - "--dry-run", - action="store_true", - default=False, - help="Print example command and exit", - ) - return parser - - -def get_info(jobid): - """ - Get details for a job - """ - jobid = flux.job.JobID(jobid) - payload = {"id": jobid, "attrs": ["all"]} - rpc = flux.job.list.JobListIdRPC(handle, "job-list.list-id", payload) - try: - jobinfo = rpc.get() - - # The job does not exist! - except FileNotFoundError: - return None - - jobinfo = jobinfo["job"] - - # User friendly string from integer - state = jobinfo["state"] - jobinfo["state"] = flux.job.info.statetostr(state) - - # Get job info to add to result - info = rpc.get_jobinfo() - jobinfo["nnodes"] = info._nnodes - jobinfo["result"] = info.result - jobinfo["returncode"] = info.returncode - jobinfo["runtime"] = info.runtime - jobinfo["priority"] = info._priority - jobinfo["waitstatus"] = info._waitstatus - jobinfo["nodelist"] = info._nodelist - jobinfo["nodelist"] = info._nodelist - jobinfo["exception"] = info._exception.__dict__ - - # Only appears after finished? - if "duration" not in jobinfo: - jobinfo["duration"] = "" - return jobinfo - - -def main(): - parser = get_parser() - - # If an error occurs while parsing the arguments, the interpreter will exit with value 2 - args, command = parser.parse_known_args() - - # Show args to the user - print(" N: %s" % args.N) - print(" times: %s" % args.times) - print(" Job Index: %s" % args.index) - print(" sleep: %s" % args.sleep) - print(" outdir: %s" % args.outdir) - print(" tasks: %s" % args.tasks) - print(" command: %s" % " ".join(command)) - print(" workdir: %s" % args.workdir) - print(" dry-run: %s" % args.dry_run) - print("identifier: %s" % args.identifier) - - # Ensure output directory exists - if not os.path.exists(args.outdir): - os.makedirs(args.outdir) - - def get_job_prefix(i): - identifier = f"{args.identifier}-{i}" - return os.path.join(args.outdir, identifier) - - # Hard code options for all setups - flux_options = [ - "-ompi=openmpi@5", - "-c", - "1", - "-o", - "cpu-affinity=per-task", - "-vvv", - ] - - # Submit all jobs - jobs = [] - for i in range(args.index, args.times + args.index): - prefix = get_job_prefix(i) - outfile = f"{prefix}.log" - - flux_command = ( - [ - "flux", - # The flux in the container is 0.44.0 - "mini", - "submit", - "-N", - str(args.N), - "-n", - str(args.tasks), - "--output", - outfile, - "--error", - outfile, - ] - + flux_options - + command - ) - - # If doing a dry run, stop here - print(" ".join(flux_command)) - if args.dry_run: - continue - - job = subprocess.Popen(flux_command, stdout=subprocess.PIPE) - jobid = job.communicate()[0] - # jobid = subprocess.check_output(flux_command) - jobid = jobid.decode("utf-8").strip() - count = i + 1 - print(f"Submit {jobid}: {count} of {args.times}") - jobs.append(jobid) - - # This didn't work! Keeping here if we want to try again.. - # TODO check the parameters here against the operator, I always forget the difference here... - # jobspec = flux.job.JobspecV1.from_command(command, num_nodes=args.N, num_tasks=args.cores) - # if args.workdir is not None: - # jobspec.cwd = args.workdir - - # Set output and error to the log file - # jobspec.stdout = outfile - # jobspec.stderr = outfile - # jobspec.environment = dict(os.environ) - - # Add mpi option flag - # jobspec.setattr_shell_option("mpi", "openmpi@5") - - # future = flux.job.submit(handle, jobspec) - # futures.append(future) - - # At this point all jobs are submit, and each should use all resources - # so we *should* be running one at a time. Now we can wait for each to save output, etc. - # Wait for futures - print("\n⭐️ Waiting for jobs to finish...") - print(f"Job ID's are - {jobs} \n") - for i, jobid in enumerate(jobs): - state = "RUN" - while state == "RUN": - info = get_info(jobid) - if info and info["state"] == "INACTIVE": - state = info["state"] - print( - f"No longer waiting on job {jobid}, FINISHED {info['returncode']}!" - ) - break - else: - print( - f"Still waiting for job {jobid} on {info['nodelist']}, has state {info['state']}" - ) - time.sleep(args.sleep) - - # When we get here, save all the metadata - prefix = get_job_prefix(i + args.index) - outfile = f"{prefix}-info.json" - with open(outfile, "w") as fd: - fd.write(json.dumps(info, indent=4)) - print("Jobs are complete, goodbye! 👋️") - - -if __name__ == "__main__": - main() diff --git a/examples/flux_operator_ca_hpa/semi-autoscaling/api-service.yaml b/examples/flux_operator_ca_hpa/semi-autoscaling/api-service.yaml deleted file mode 100644 index 7aeca5e..0000000 --- a/examples/flux_operator_ca_hpa/semi-autoscaling/api-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - name: v1beta2.custom.metrics.k8s.io -spec: - # You'll want to not do this in production - insecureSkipTLSVerify: true - service: - name: custom-metrics-apiserver - namespace: flux-operator - group: custom.metrics.k8s.io - version: v1beta2 - groupPriorityMinimum: 1000 - versionPriority: 5 - # caBundle: ${CA_BUNDLE} diff --git a/examples/flux_operator_ca_hpa/semi-autoscaling/certs.sh b/examples/flux_operator_ca_hpa/semi-autoscaling/certs.sh deleted file mode 100644 index a66df02..0000000 --- a/examples/flux_operator_ca_hpa/semi-autoscaling/certs.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Create CA Key -openssl genrsa -out ca.key 2048 - -# Create CA Cert -openssl req -new -key ca.key -x509 -out ca.crt -days 3650 -subj "/CN=ca" - -# Create Server Key and Signing Request -openssl req -new -nodes -newkey rsa:2048 -keyout server.key -out server.req -batch -subj "/CN=custom-metrics-apiserver.custom-metrics.svc" - -# Create Signed Server Cert -openssl x509 -req -in server.req -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 3650 -sha256 - -rm server.req -rm ca.srl -rm ca.key diff --git a/examples/flux_operator_ca_hpa/semi-autoscaling/minicluster-libfabric-new-custom-metrics.yaml b/examples/flux_operator_ca_hpa/semi-autoscaling/minicluster-libfabric-new-custom-metrics.yaml deleted file mode 100644 index 9807e96..0000000 --- a/examples/flux_operator_ca_hpa/semi-autoscaling/minicluster-libfabric-new-custom-metrics.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: flux-framework.org/v1alpha1 -kind: MiniCluster -metadata: - name: flux-sample - namespace: flux-operator -spec: - - # Number of pods to create for MiniCluster - size: 2 - maxSize: 16 - - # This doesn't matter so much in interactive mode, tasks specified with the job - # 12 * 64 - # tasks: 128 - - # This starts the flux broker without a command (interactive) - interactive: true - logging: - quiet: false - strict: false - zeromq: true - debug: true - - # Spack installs to /opt/view, without flux-security so no imp - flux: - installRoot: /opt/view - optionFlags: "-ompi=openmpi@5 -c 1 -o cpu-affinity=per-task" - scheduler: - queuePolicy: easy - - containers: - - # image: ghcr.io/rse-ops/flux-arm-lammps:june-2023-arm64 - # spack 0.19 with a custom package.py that has updated libfabric - image: ghcr.io/rse-ops/flux-arm-lammps:libfabric-1.18.1 - workingDir: /home/flux/examples/reaxff/HNS - - # This shouldn't be needed since we are launching manually - # cores: 64 - - # This would be the problem size we used for 8 - # command: lmp -v x 64 -v y 16 -v z 16 -in in.reaxc.hns -nocite - - # Resource limits to ensure 1 pod assigned per node - # These are purposefully lower - the actual value didn't work, but - # this should still assign 1:1. - resources: - limits: - vpc.amazonaws.com/efa: 1 - memory: "110G" - cpu: "62" - - requests: - vpc.amazonaws.com/efa: 1 - memory: "110G" # This is actually 128 - cpu: "62" - - existingVolumes: - certs: - path: /etc/certs - secretName: certs - # We still need to setup spack so flux can start the broker! - commands: - # The workers need to come up after the broker and network - we are hitting this issue - # when we get to this larger scale (and it is compounded by the new networking issue with the service) - # that appeared after we removed the certificate generation pod - workerPre: sleep 60 - pre: | - source /etc/profile.d/z10_spack_environment.sh - asFlux="sudo -u flux -E PYTHONPATH=$PYTHONPATH -E PATH=$PATH -E FI_PROVIDER=efa -E OMPI_MCA_btl=self,ofi -E RDMAV_FORK_SAFE=1 -E FI_EFA_USE_DEVICE_RDMA=1" - . /etc/profile.d/z10_spack_environment.sh - cd /opt/spack-environment - . /opt/spack-environment/spack/share/spack/setup-env.sh - spack env activate . - cd /home/flux/examples/reaxff/HNS - pip3 install flux-metrics-api diff --git a/examples/flux_operator_ca_hpa/semi-autoscaling/service.yaml b/examples/flux_operator_ca_hpa/semi-autoscaling/service.yaml deleted file mode 100644 index da41131..0000000 --- a/examples/flux_operator_ca_hpa/semi-autoscaling/service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: custom-metrics-apiserver - namespace: flux-operator -spec: - # This says "the service is on the pod with this selector" - selector: - api-server: custom-metrics - ports: - - protocol: TCP - port: 443 - targetPort: 8443