From 44e2fd80891d555915aa2f65a2faf5164b8752f7 Mon Sep 17 00:00:00 2001 From: luoyuanze Date: Wed, 25 Dec 2024 11:15:16 +0800 Subject: [PATCH 1/2] Add NodeLocalDNS configuration to the kubenest workflow. Signed-off-by: luoyuanze --- .../kosmos.io_kubenestconfigurations.yaml | 6 ++++ deploy/crds/kosmos.io_virtualclusters.yaml | 5 ++- hack/k8s-in-k8s/kubelet_node_helper.sh | 10 +++++- .../v1alpha1/kubenestconfiguration_types.go | 3 ++ pkg/generated/openapi/zz_generated.openapi.go | 6 ++++ pkg/kubenest/constants/constant.go | 6 ++++ .../workflow/task/task.go | 7 ++-- pkg/kubenest/tasks/manifests_components.go | 34 +++++++++++++++++++ 8 files changed, 73 insertions(+), 4 deletions(-) diff --git a/deploy/crds/kosmos.io_kubenestconfigurations.yaml b/deploy/crds/kosmos.io_kubenestconfigurations.yaml index d1f1c02d3..4d5387ec2 100644 --- a/deploy/crds/kosmos.io_kubenestconfigurations.yaml +++ b/deploy/crds/kosmos.io_kubenestconfigurations.yaml @@ -57,6 +57,9 @@ spec: type: string etcdUnitSize: type: string + externalPort: + format: int32 + type: integer forceDestroy: description: todo Group according to the parameters of apiserver, etcd, coredns, etc. ForceDestroy indicates whether to force destroy @@ -77,6 +80,9 @@ spec: type: string type: array type: object + useNodeLocalDNS: + default: false + type: boolean useTenantDNS: default: false type: boolean diff --git a/deploy/crds/kosmos.io_virtualclusters.yaml b/deploy/crds/kosmos.io_virtualclusters.yaml index 5e2a2c617..cb72f1aa2 100644 --- a/deploy/crds/kosmos.io_virtualclusters.yaml +++ b/deploy/crds/kosmos.io_virtualclusters.yaml @@ -75,8 +75,8 @@ spec: etcdUnitSize: type: string externalPort: - type: integer format: int32 + type: integer forceDestroy: description: todo Group according to the parameters of apiserver, etcd, coredns, etc. ForceDestroy indicates whether to force @@ -98,6 +98,9 @@ spec: type: string type: array type: object + useNodeLocalDNS: + default: false + type: boolean useTenantDNS: default: false type: boolean diff --git a/hack/k8s-in-k8s/kubelet_node_helper.sh b/hack/k8s-in-k8s/kubelet_node_helper.sh index d02e75012..c780ca762 100755 --- a/hack/k8s-in-k8s/kubelet_node_helper.sh +++ b/hack/k8s-in-k8s/kubelet_node_helper.sh @@ -8,6 +8,7 @@ LOG_NAME=${2:-kubelet} JOIN_HOST=$2 JOIN_TOKEN=$3 JOIN_CA_HASH=$4 +NODE_LOCAL_DNS_ADDRESS=$3 function unjoin() { # before unjoin, you need delete node by kubectl @@ -248,7 +249,14 @@ function join() { exit 1 fi echo "exec(4/8): set core dns address...." - sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME" + if [ -n "$NODE_LOCAL_DNS_ADDRESS" ]; then + sed -e "/__DNS_ADDRESS__/i - ${NODE_LOCAL_DNS_ADDRESS}" \ + -e "s|__DNS_ADDRESS__|${DNS_ADDRESS}|g" \ + "$PATH_FILE_TMP/$KUBELET_CONFIG_NAME" \ + > "${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}" + else + sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME" + fi if [ $? -ne 0 ]; then exit 1 fi diff --git a/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go b/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go index 5fd4cd1f5..2122b99d6 100644 --- a/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go +++ b/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go @@ -91,6 +91,9 @@ type KubeInKubeConfig struct { UseTenantDNS bool `yaml:"useTenantDNS" json:"useTenantDNS,omitempty"` // +optional ExternalPort int32 `json:"externalPort,omitempty"` + // +kubebuilder:default=false + // +optional + UseNodeLocalDNS bool `yaml:"useNodeLocalDNS" json:"useNodeLocalDNS,omitempty"` } // TenantEntrypoint contains the configuration for the tenant entrypoint. diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index a9a579096..4d2befe7f 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -1995,6 +1995,12 @@ func schema_pkg_apis_kosmos_v1alpha1_KubeInKubeConfig(ref common.ReferenceCallba Format: "int32", }, }, + "useNodeLocalDNS": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, }, }, }, diff --git a/pkg/kubenest/constants/constant.go b/pkg/kubenest/constants/constant.go index f15d9440b..3d24c5111 100644 --- a/pkg/kubenest/constants/constant.go +++ b/pkg/kubenest/constants/constant.go @@ -137,6 +137,12 @@ const ( //in virtual cluster APIServerExternalService = "api-server-external-service" + + //nodelocaldns + NodeLocalDNSComponentName = "virtual-node-local-dns" + NodeLocalDNSIp = "169.254.20.10" + NodeLocalDNSClusterDomain = "cluster.local" + NodeLocalDNSService = "__PILLAR__DNS__SERVER__" ) type Action string diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go index 118551e5d..ff549b0e1 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go @@ -254,9 +254,12 @@ func NewRemoteNodeJoinTask() Task { Retry: true, Run: func(ctx context.Context, to TaskOpt, _ interface{}) (interface{}, error) { exectHelper := exector.NewExectorHelper(to.NodeInfo.Spec.NodeIP, "") - + baseCmd := fmt.Sprintf("bash %s join %s", env.GetExectorShellName(), to.KubeDNSAddress) + if to.VirtualCluster.Spec.KubeInKubeConfig != nil && to.VirtualCluster.Spec.KubeInKubeConfig.UseNodeLocalDNS { + baseCmd = fmt.Sprintf("bash %s join %s %s", env.GetExectorShellName(), to.KubeDNSAddress, constants.NodeLocalDNSIp) + } joinCmd := &exector.CMDExector{ - Cmd: fmt.Sprintf("bash %s join %s", env.GetExectorShellName(), to.KubeDNSAddress), + Cmd: baseCmd, } to.Loger().Infof("join node %s with cmd: %s", to.NodeInfo.Name, joinCmd.Cmd) ret := exectHelper.DoExector(ctx.Done(), joinCmd) diff --git a/pkg/kubenest/tasks/manifests_components.go b/pkg/kubenest/tasks/manifests_components.go index 6b59d9866..4e8b3f7fc 100644 --- a/pkg/kubenest/tasks/manifests_components.go +++ b/pkg/kubenest/tasks/manifests_components.go @@ -2,6 +2,7 @@ package tasks import ( "context" + "fmt" "os" "path/filepath" @@ -15,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" @@ -95,6 +97,11 @@ func applyComponentsManifests(r workflow.RunData) error { templatedMapping["KUBE_PROXY_KUBECONFIG"] = string(secret.Data[constants.KubeConfig]) imageRepository, _ := util.GetImageMessage() templatedMapping["ImageRepository"] = imageRepository + + templatedMapping["PillarLocalDNS"] = constants.NodeLocalDNSIp + templatedMapping["PillarDNSDomain"] = constants.NodeLocalDNSClusterDomain + templatedMapping["PillarDNSServer"] = "" + for k, v := range data.PluginOptions() { templatedMapping[k] = v } @@ -111,6 +118,7 @@ func applyComponentsManifests(r workflow.RunData) error { } UseTenantDNS := data.VirtualCluster().Spec.KubeInKubeConfig != nil && data.VirtualCluster().Spec.KubeInKubeConfig.UseTenantDNS + UseNodeLocalDNS := data.VirtualCluster().Spec.KubeInKubeConfig != nil && data.VirtualCluster().Spec.KubeInKubeConfig.UseNodeLocalDNS skipComponents := getSkipComponentsForVirtualCluster([]*SkipComponentCondition{ { @@ -122,6 +130,11 @@ func applyComponentsManifests(r workflow.RunData) error { Condition: !keepalivedEnable, ComponentName: constants.VipKeepalivedComponentName, }, + { + // skip nodelocaldns component if nodelocaldns is not enabled + Condition: !UseNodeLocalDNS, + ComponentName: constants.NodeLocalDNSComponentName, + }, }) for _, component := range components { @@ -130,6 +143,14 @@ func applyComponentsManifests(r workflow.RunData) error { klog.V(2).Infof("Deploy component %s skipped", component.Name) continue } + if component.Name == constants.NodeLocalDNSComponentName { + kubeDNSIP, err := getKubeDNSClusterIP(config) + if err != nil { + return errors.Wrap(err, "Failed to get kube-dns ClusterIP") + } + klog.Infof("kube-dns CLUSTER-IP: %s", kubeDNSIP) + templatedMapping["PillarClusterDNS"] = kubeDNSIP + } err = applyTemplatedManifests(component.Name, dynamicClient, component.Path, templatedMapping) if err != nil { return err @@ -215,3 +236,16 @@ func applyTemplatedManifests(component string, dynamicClient dynamic.Interface, } return nil } +func getKubeDNSClusterIP(config *rest.Config) (string, error) { + client, err := clientset.NewForConfig(config) + if err != nil { + return "", fmt.Errorf("failed to create kubernetes client: %v", err) + } + + svc, err := client.CoreV1().Services("kube-system").Get(context.TODO(), "kube-dns", metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("failed to get kube-dns service: %v", err) + } + + return svc.Spec.ClusterIP, nil +} From 838026b75cab1db810b8760b1be236c1a093537e Mon Sep 17 00:00:00 2001 From: baoyinghai_yewu Date: Mon, 30 Dec 2024 14:59:45 +0800 Subject: [PATCH 2/2] feat: support lvscare for kubenest Signed-off-by: baoyinghai_yewu --- hack/k8s-in-k8s/generate_env.sh | 32 +++- hack/k8s-in-k8s/kubelet_node_helper.sh | 156 ++++++++++++++++-- .../workflow/task/logger.go | 11 ++ .../workflow/workflow.go | 6 +- 4 files changed, 186 insertions(+), 19 deletions(-) diff --git a/hack/k8s-in-k8s/generate_env.sh b/hack/k8s-in-k8s/generate_env.sh index 5f8b16040..b55a9969e 100644 --- a/hack/k8s-in-k8s/generate_env.sh +++ b/hack/k8s-in-k8s/generate_env.sh @@ -106,6 +106,7 @@ PATH_KUBERNETES=$(GetDirectory $PATH_KUBERNETES_PKI) HOST_CORE_DNS=$(GetKubeDnsClusterIP) DOCKER_IMAGE_NGINX="registry.paas/cmss/nginx:1.21.4" +DOCKER_IMAGE_LVSCARE="registry.paas/cmss/lvscare:1.0.0" master_lables=("master", "control-plane") @@ -157,10 +158,35 @@ KUBELET_CONF_TIMEOUT=30 # load balance DOCKER_IMAGE_NGINX=$DOCKER_IMAGE_NGINX +DOCKER_IMAGE_LVSCARE=$DOCKER_IMAGE_LVSCARE SERVERS=($SERVERS) -LOCAL_PORT="6443" -LOCAL_IP="127.0.0.1" # [::1] -USE_NGINX=true + +# Proxy Configuration Options +# Specify the proxy server to be used for traffic management or load balancing. +# Available options for USE_PROXY: +# - "NGINX" : Use NGINX as the proxy server. +# - "LVSCARE" : Use LVSCARE for load balancing (based on IPVS). +# - "NONE" : No proxy server will be used. +# Note: When USE_PROXY is set to "NONE", no proxy service will be configured. +USE_PROXY="LVSCARE" # Current proxy setting: LVSCARE for load balancing. + +# Proxy Service Port Configuration +# LOCAL_PORT specifies the port on which the proxy service listens. +# Example: +# - For Kubernetes setups, this is typically the API server port. +LOCAL_PORT="6443" # Proxy service listening port (default: 6443 for Kubernetes API). + +# Proxy Address Configuration +# LOCAL_IP specifies the address of the proxy service. +# - When USE_PROXY is set to "NGINX": +# - Use LOCAL_IP="127.0.0.1" (IPv4) or LOCAL_IP="[::1]" (IPv6 loopback). +# - When USE_PROXY is set to "LVSCARE": +# - Use LOCAL_IP as the VIP (e.g., "192.0.0.2") for LVSCARE load balancing. +# - Ensure this address is added to the "excludeCIDRs" list in the kube-proxy configuration file +# to avoid routing conflicts. +LOCAL_IP="192.0.0.2" # LVSCARE setup: Proxy address and VIP for load balancing. + + CRI_SOCKET=$CRI_SOCKET function GenerateKubeadmConfig() { diff --git a/hack/k8s-in-k8s/kubelet_node_helper.sh b/hack/k8s-in-k8s/kubelet_node_helper.sh index 235938f1f..161053fa5 100755 --- a/hack/k8s-in-k8s/kubelet_node_helper.sh +++ b/hack/k8s-in-k8s/kubelet_node_helper.sh @@ -202,8 +202,8 @@ function get_ca_certificate() { # verify the kubeconfig data is not empty if [ -z "$kubeconfig_data" ]; then - echo "Failed to extract certificate-authority-data." - return 1 + echo "Failed to extract certificate-authority-data." + return 1 fi # Base64 decoded and written to a file @@ -284,6 +284,9 @@ function revert() { echo "NONONO use kubeadm to join node to host" get_ca_certificate $JOIN_HOST + if [ $? -ne 0 ]; then + exit 1 + fi create_kubelet_bootstrap_config $JOIN_HOST $JOIN_TOKEN if [ -f "${PATH_FILE_TMP}/kubeadm-flags.env.origin" ]; then cp "${PATH_FILE_TMP}/kubeadm-flags.env.origin" "${PATH_KUBELET_LIB}" && \ @@ -457,16 +460,31 @@ function is_ipv6() { fi } -function install_lb() { - if [ -z "$USE_NGINX" ]; then - export USE_NGINX=false - fi +function wait_api_server_proxy_ready() { + local retries=0 + local max_retries=10 + local sleep_duration=6 - if [ "$USE_NGINX" = false ]; then - exit 0 - fi + while true; do + response=$(curl -k --connect-timeout 5 --max-time 10 https://${LOCAL_IP}:${LOCAL_PORT}/healthz) + + if [ "$response" == "ok" ]; then + echo "apiserver proxy is ready!" + return 0 + else + retries=$((retries + 1)) + echo "apiserver proxy is not ready. Retrying(${retries}/${max_retries})..." + if [ "$retries" -ge "$max_retries" ]; then + echo "Max retries reached. apiserver proxy did not become ready." + return 1 + fi + sleep $sleep_duration + fi + done +} - echo "exec(1/6): get port of apiserver...." +function install_nginx_lb() { + echo "exec(1/7): get port of apiserver...." PORT=$(grep 'server:' "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" | awk -F '[:/]' '{print $NF}') @@ -483,7 +501,7 @@ function install_lb() { fi # Start generating nginx.conf - echo "exec(2/6): generate nginx.conf...." + echo "exec(2/7): generate nginx.conf...." cat < "$PATH_FILE_TMP/nginx.conf" error_log stderr notice; worker_processes 1; @@ -520,23 +538,135 @@ EOL } EOL - echo "exec(3/6): create static pod" + echo "exec(3/7): create static pod" GenerateStaticNginxProxy true - echo "exec(4/6): restart static pod" + echo "exec(4/7): restart static pod" mv "${PATH_KUBERNETES}/manifests/nginx-proxy.yaml" "${PATH_KUBERNETES}/nginx-proxy.yaml" sleep 2 mv "${PATH_KUBERNETES}/nginx-proxy.yaml" "${PATH_KUBERNETES}/manifests/nginx-proxy.yaml" + echo "exec(5/7): wati nginx ready" + if wait_api_server_proxy_ready; then + echo "nginx is ready" + else + echo "nginx is not ready" + exit 1 + fi + + echo "exec(6/7): update kubelet.conf" + cp "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}.bak" + sed -i "s|server: .*|server: https://${LOCAL_IP}:${LOCAL_PORT}|" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" + + echo "exec(7/7): restart kubelet" + systemctl restart kubelet +} + +function install_lvscare_lb() { + echo "exec(1/6): get port of apiserver...." + + PORT=$(grep 'server:' "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" | awk -F '[:/]' '{print $NF}') + + if [ -z "$PORT" ]; then + echo "can not get port" + exit 1 + else + echo "port is $PORT" + fi + + # Start generating kube-lvscare.yaml + echo "exec(2/6): generate kube-lvscare.yaml...." + + cat < $PATH_KUBERNETES/manifests/kube-lvscare.yaml +apiVersion: v1 +kind: Pod +metadata: + labels: + app: kube-lvscare + name: kube-lvscare + namespace: kube-system +spec: + containers: + - args: + - care + - --vs + - ${LOCAL_IP}:${LOCAL_PORT} + - --health-path + - /healthz + - --health-schem + - https +EOL + + # Loop through the array and append each server to the kube-lvscare.yaml file + for SERVER in "${SERVERS[@]}"; do + if is_ipv6 "$SERVER"; then + echo " - --rs" >> "$PATH_KUBERNETES/manifests/kube-lvscare.yaml" + echo " - [$SERVER]:$PORT" >> "$PATH_KUBERNETES/manifests/kube-lvscare.yaml" + else + echo " - --rs" >> "$PATH_KUBERNETES/manifests/kube-lvscare.yaml" + echo " - $SERVER:$PORT" >> "$PATH_KUBERNETES/manifests/kube-lvscare.yaml" + fi + done + + # Continue writing the rest of the kube-lvscare.yaml file + cat <> "$PATH_KUBERNETES/manifests/kube-lvscare.yaml" + command: + - /usr/bin/lvscare + image: $DOCKER_IMAGE_LVSCARE + imagePullPolicy: Always + name: kube-lvscare + resources: {} + securityContext: + privileged: true + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + hostNetwork: true + volumes: + - hostPath: + path: /lib/modules + name: lib-modules +status: {} +EOL + + echo "exec(3/6): restart static pod" + mv "${PATH_KUBERNETES}/manifests/kube-lvscare.yaml" "${PATH_KUBERNETES}/kube-lvscare.yaml" + sleep 2 + mv "${PATH_KUBERNETES}/kube-lvscare.yaml" "${PATH_KUBERNETES}/manifests/kube-lvscare.yaml" + + echo "exec(4/6): wait lvscare ready" + if wait_api_server_proxy_ready; then + echo "lvscare is ready" + else + echo "lvscare is not ready" + exit 1 + fi + echo "exec(5/6): update kubelet.conf" cp "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}.bak" sed -i "s|server: .*|server: https://${LOCAL_IP}:${LOCAL_PORT}|" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" + sed -i 's|certificate-authority-data: .*|insecure-skip-tls-verify: true|' "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" echo "exec(6/6): restart kubelet" systemctl restart kubelet } +function install_lb() { + if [ -z "$USE_PROXY" ]; then + export USE_PROXY="LVSCARE" + fi + + if [ "$USE_PROXY" = "NGINX" ]; then + install_nginx_lb + elif [ "$USE_PROXY" = "LVSCARE" ]; then + install_lvscare_lb + else + exit 0 + fi +} + # See how we were called. case "$1" in unjoin) diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/logger.go b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/logger.go index 740312bdc..d48113de6 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/logger.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/logger.go @@ -27,6 +27,17 @@ func (p *PrefixedLogger) Infof(format string, args ...interface{}) { } } +func (p *PrefixedLogger) Warn(args ...interface{}) { + if p.level.Enabled() { + klog.WarningDepth(1, append([]interface{}{p.prefix}, args...)...) + } +} +func (p *PrefixedLogger) Warnf(format string, args ...interface{}) { + if p.level.Enabled() { + klog.WarningDepth(1, fmt.Sprintf(p.prefix+format, args...)) + } +} + func (p *PrefixedLogger) Error(args ...interface{}) { klog.ErrorDepth(1, append([]interface{}{p.prefix}, args...)...) } diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go index a3f99b2ea..e8eba31b5 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go @@ -28,7 +28,7 @@ func RunWithRetry(ctx context.Context, task task.Task, opt task.TaskOpt, preArgs break } waitTime := 3 * (i + 1) - opt.Loger().Infof("work flow retry %d after %ds, task name: %s, err: %s", i, waitTime, task.Name, err) + opt.Loger().Warnf("work flow retry %d after %ds, task name: %s, err: %s", i, waitTime, task.Name, err) time.Sleep(time.Duration(waitTime) * time.Second) } else { break @@ -36,10 +36,10 @@ func RunWithRetry(ctx context.Context, task task.Task, opt task.TaskOpt, preArgs } if err != nil { if task.ErrorIgnore { - opt.Loger().Infof("work flow ignore err, task name: %s, err: %s", task.Name, err) + opt.Loger().Warnf("work flow ignore err, task name: %s, err: %s", task.Name, err) return nil, nil } - opt.Loger().Infof("work flow interrupt, task name: %s, err: %s", task.Name, err) + opt.Loger().Warnf("work flow interrupt, task name: %s, err: %s", task.Name, err) return nil, err } return args, nil