From 9ee2d7a7933bd67936e96dc0e4d359e949eba40d Mon Sep 17 00:00:00 2001 From: Bipul Adhikari Date: Mon, 11 Nov 2024 18:51:07 +0545 Subject: [PATCH] Adds support for TLS and token reviews Signed-off-by: Bipul Adhikari --- Makefile | 16 +-- api/csiaddons/v1alpha1/csiaddonsnode_types.go | 4 + cmd/manager/main.go | 13 +- config/manager/kustomization.yaml | 4 +- deploy/controller/setup-controller.yaml | 2 +- internal/connection/connection.go | 30 ++++- .../csiaddons/csiaddonsnode_controller.go | 12 +- internal/kubernetes/token/grpc.go | 123 ++++++++++++++++++ sidecar/internal/server/server.go | 34 ++++- sidecar/main.go | 3 +- 10 files changed, 212 insertions(+), 29 deletions(-) create mode 100644 internal/kubernetes/token/grpc.go diff --git a/Makefile b/Makefile index dbb8de941..078bcf45b 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,12 @@ # Image URL to use all building/pushing image targets -CONTROLLER_IMG ?= quay.io/csiaddons/k8s-controller -SIDECAR_IMG ?= quay.io/csiaddons/k8s-sidecar -BUNDLE_IMG ?= quay.io/csiaddons/k8s-bundle -TOOLS_IMG ?= quay.io/csiaddons/tools +CONTROLLER_IMG ?= quay.io/badhikar/k8s-controller +SIDECAR_IMG ?= quay.io/badhikar/k8s-sidecar +BUNDLE_IMG ?= quay.io/badhikar/k8s-bundle +TOOLS_IMG ?= quay.io/badhikar/tools # set TAG to a release for consumption in the bundle -TAG ?= latest +TAG ?= v4 # In case the *_IMG variables can contain a full qualified container-image # resource (includes a ":"), the container-images should not use the TAG @@ -175,8 +175,8 @@ run: manifests generate fmt vet ## Run a controller from your host. go run ./cmd/manager/main.go .PHONY: docker-build -docker-build: container-cmd test ## Build docker image with the manager. - $(CONTAINER_CMD) build -t ${CONTROLLER_IMG} . +docker-build: container-cmd ## Build docker image with the manager. + $(CONTAINER_CMD) build -t ${CONTROLLER_IMG} . --platform=linux/amd64 .PHONY: docker-push docker-push: container-cmd ## Push docker image with the manager. @@ -184,7 +184,7 @@ docker-push: container-cmd ## Push docker image with the manager. .PHONY: docker-build-sidecar docker-build-sidecar: container-cmd - $(CONTAINER_CMD) build -f ./build/Containerfile.sidecar -t ${SIDECAR_IMG} . + $(CONTAINER_CMD) build -f ./build/Containerfile.sidecar -t ${SIDECAR_IMG} . --platform=linux/amd64 .PHONY: docker-push-sidecar docker-push-sidecar: container-cmd diff --git a/api/csiaddons/v1alpha1/csiaddonsnode_types.go b/api/csiaddons/v1alpha1/csiaddonsnode_types.go index 36e7a5b30..e7a0c5d2c 100644 --- a/api/csiaddons/v1alpha1/csiaddonsnode_types.go +++ b/api/csiaddons/v1alpha1/csiaddonsnode_types.go @@ -44,6 +44,10 @@ type CSIAddonsNodeDriver struct { // side-car listens to. EndPoint string `json:"endpoint"` + // The name of the service that has the sidecar + // side-car listens to. + SidecarService string `json:"sidecarService"` + // NodeID is the ID of the node to identify on which node the side-car // is running. // +kubebuilder:validation:Required diff --git a/cmd/manager/main.go b/cmd/manager/main.go index df218544b..ed9da6084 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -77,6 +77,8 @@ func main() { enableAdmissionWebhooks bool ctx = context.Background() cfg = util.NewConfig() + enableTLS bool + skipInsecureVerify bool ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -92,6 +94,8 @@ func main() { flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", false, "[DEPRECATED] Enable the admission webhooks") flag.BoolVar(&showVersion, "version", false, "Print Version details") flag.StringVar(&cfg.SchedulePrecedence, "schedule-precedence", "", "The order of precedence in which schedule of reclaimspace and keyrotation is considered. Possible values are sc-only") + flag.BoolVar(&enableTLS, "enable-tls", false, "Enable TLS(disabled by default)") + flag.BoolVar(&skipInsecureVerify, "insecure-skip-tls-verify", false, "skip server certificate verification") opts := zap.Options{ Development: true, TimeEncoder: zapcore.ISO8601TimeEncoder, @@ -145,16 +149,17 @@ func main() { setupLog.Error(err, "unable to start manager") os.Exit(1) } - connPool := connection.NewConnectionPool() ctrlOptions := controller.Options{ MaxConcurrentReconciles: cfg.MaxConcurrentReconciles, } if err = (&controllers.CSIAddonsNodeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ConnPool: connPool, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ConnPool: connPool, + EnableTLS: enableTLS, + SkipInsecureVerify: skipInsecureVerify, }).SetupWithManager(mgr, ctrlOptions); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CSIAddonsNode") os.Exit(1) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 4cbda177d..01ed169c4 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -26,8 +26,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: quay.io/csiaddons/k8s-controller - newTag: latest + newName: quay.io/badhikar/k8s-controller + newTag: v2 - name: rbac-proxy newName: quay.io/brancz/kube-rbac-proxy newTag: v0.18.0 diff --git a/deploy/controller/setup-controller.yaml b/deploy/controller/setup-controller.yaml index d602b6024..7bc46adb3 100644 --- a/deploy/controller/setup-controller.yaml +++ b/deploy/controller/setup-controller.yaml @@ -80,7 +80,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: quay.io/csiaddons/k8s-controller:latest + image: quay.io/badhikar/k8s-controller:v2 livenessProbe: httpGet: path: /healthz diff --git a/internal/connection/connection.go b/internal/connection/connection.go index 0b24f0210..bf1c77f9c 100644 --- a/internal/connection/connection.go +++ b/internal/connection/connection.go @@ -18,11 +18,17 @@ package connection import ( "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" "time" + "github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token" + "github.com/csi-addons/spec/lib/go/identity" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials" ) // Connection struct consists of to NodeID, DriverName, Capabilities for the controller @@ -39,11 +45,29 @@ type Connection struct { // NewConnection establishes connection with sidecar, fetches capability and returns Connection object // filled with required information. -func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string) (*Connection, error) { +func NewConnection(ctx context.Context, endpoint, nodeID, driverName, namespace, podName string, enableTLS, skipInsecureVerify bool) (*Connection, error) { opts := []grpc.DialOption{ - grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithIdleTimeout(time.Duration(0)), } + + if enableTLS { + opts = append(opts, token.WithServiceAccountToken()) + + caFile, caError := token.GetCACert() + if caError != nil { + return nil, fmt.Errorf("failed to get server cert %w", caError) + } + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM([]byte(caFile)) { + return nil, errors.New("failed to append CA cert") + } + tlsConfig := &tls.Config{ + RootCAs: caCertPool, // The CA certificates to verify the server + InsecureSkipVerify: skipInsecureVerify, + } + creds := credentials.NewTLS(tlsConfig) + opts = append(opts, grpc.WithTransportCredentials(creds)) + } cc, err := grpc.NewClient(endpoint, opts...) if err != nil { return nil, err diff --git a/internal/controller/csiaddons/csiaddonsnode_controller.go b/internal/controller/csiaddons/csiaddonsnode_controller.go index d5f0f763e..1c32cf54d 100644 --- a/internal/controller/csiaddons/csiaddonsnode_controller.go +++ b/internal/controller/csiaddons/csiaddonsnode_controller.go @@ -49,8 +49,10 @@ var ( // CSIAddonsNodeReconciler reconciles a CSIAddonsNode object type CSIAddonsNodeReconciler struct { client.Client - Scheme *runtime.Scheme - ConnPool *connection.ConnectionPool + Scheme *runtime.Scheme + ConnPool *connection.ConnectionPool + EnableTLS bool + SkipInsecureVerify bool } //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch @@ -120,7 +122,7 @@ func (r *CSIAddonsNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reques } logger.Info("Connecting to sidecar") - newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name) + newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name, r.EnableTLS, r.SkipInsecureVerify) if err != nil { logger.Error(err, "Failed to establish connection with sidecar") @@ -207,6 +209,10 @@ func (r *CSIAddonsNodeReconciler) removeFinalizer( // by GRPC to connect to the sidecar. func (r *CSIAddonsNodeReconciler) resolveEndpoint(ctx context.Context, rawURL string) (string, error) { namespace, podname, port, err := parseEndpoint(rawURL) + if r.EnableTLS { + // We need to use this name to accept certificates signed for pods + return podname + "." + namespace + ".pod" + ":" + port, nil + } if err != nil && errors.Is(err, errLegacyEndpoint) { return rawURL, nil } else if err != nil { diff --git a/internal/kubernetes/token/grpc.go b/internal/kubernetes/token/grpc.go new file mode 100644 index 000000000..046891068 --- /dev/null +++ b/internal/kubernetes/token/grpc.go @@ -0,0 +1,123 @@ +/* +Copyright 2024 The Kubernetes-CSI-Addons Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package token + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + authv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const bearerPrefix = "Bearer " + +func WithServiceAccountToken() grpc.DialOption { + return grpc.WithUnaryInterceptor(addAuthorizationHeader) +} + +func addAuthorizationHeader(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + token, err := getToken() + if err != nil { + return err + } + + authCtx := metadata.AppendToOutgoingContext(ctx, "Authorization", "Bearer "+token) + return invoker(authCtx, method, req, reply, cc, opts...) +} + +func getToken() (string, error) { + + const tokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + return readFile(tokenPath) +} + +func AuthorizationInterceptor(kubeclient kubernetes.Clientset) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + if err := authorizeConnection(ctx, kubeclient); err != nil { + return nil, err + } + return handler(ctx, req) + } +} + +func authorizeConnection(ctx context.Context, kubeclient kubernetes.Clientset) error { + + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return status.Error(codes.Unauthenticated, "missing metadata") + } + + authHeader, ok := md["authorization"] + if !ok || len(authHeader) == 0 { + return status.Error(codes.Unauthenticated, "missing authorization token") + } + + token := authHeader[0] + isValidated, err := validateBearerToken(ctx, token, kubeclient) + if !isValidated || (err != nil) { + return status.Error(codes.Unauthenticated, fmt.Sprint("invalid token: %w", err)) + } + return nil +} + +func parseToken(authHeader string) string { + return strings.TrimPrefix(authHeader, bearerPrefix) +} + +func validateBearerToken(ctx context.Context, token string, kubeclient kubernetes.Clientset) (bool, error) { + tokenReview := &authv1.TokenReview{ + Spec: authv1.TokenReviewSpec{ + Token: parseToken(token), + }, + } + result, err := kubeclient.AuthenticationV1().TokenReviews().Create(ctx, tokenReview, metav1.CreateOptions{}) + if err != nil { + return false, fmt.Errorf("failed to review token %w", err) + } + + if result.Status.Authenticated { + return true, nil + } + return false, nil +} + +func readFile(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + return "", err + } + return string(data), nil +} + +func GetCACert() (string, error) { + caCertFile := "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + return readFile(caCertFile) +} diff --git a/sidecar/internal/server/server.go b/sidecar/internal/server/server.go index 112554683..918ce6719 100644 --- a/sidecar/internal/server/server.go +++ b/sidecar/internal/server/server.go @@ -18,9 +18,13 @@ package server import ( "errors" + "fmt" "net" + "github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + k8s "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ) @@ -38,15 +42,19 @@ type SidecarServer struct { // URL components to listen on the tcp port scheme string endpoint string + client *k8s.Clientset - server *grpc.Server - services []SidecarService + server *grpc.Server + services []SidecarService + enableTLS bool + tlsCertFile string + tlsKeyFile string } // NewSidecarServer create a new SidecarServer on the given IP-address and // port. If the IP-address is an empty string, the server will listen on all // available IP-addresses. Only tcp ports are supported. -func NewSidecarServer(ip, port string) *SidecarServer { +func NewSidecarServer(ip, port string, client *k8s.Clientset, enableTLS bool, tlsCertFile, tlsKeyFile string) *SidecarServer { ss := &SidecarServer{} if ss.services == nil { @@ -54,8 +62,11 @@ func NewSidecarServer(ip, port string) *SidecarServer { } ss.scheme = "tcp" - ss.endpoint = ip + ":" + port - + ss.endpoint = ip + ":" + fmt.Sprint(port) + ss.client = client + ss.enableTLS = enableTLS + ss.tlsCertFile = tlsCertFile + ss.tlsKeyFile = tlsKeyFile return ss } @@ -69,8 +80,17 @@ func (ss *SidecarServer) RegisterService(svc SidecarService) { // Init creates the internal gRPC server, and registers the SidecarServices. // and starts gRPC server. func (ss *SidecarServer) Start() { - // create the gRPC server and register services - ss.server = grpc.NewServer() + if ss.enableTLS { + creds, err := credentials.NewServerTLSFromFile(ss.tlsCertFile, ss.tlsKeyFile) + if err != nil { + klog.Fatalf("failed to load TLS certificate and key: %v", err) + } + // create the gRPC server and register services + ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(*ss.client)), grpc.Creds(creds)) + } + if !ss.enableTLS { + ss.server = grpc.NewServer() + } for _, svc := range ss.services { svc.RegisterService(ss.server) diff --git a/sidecar/main.go b/sidecar/main.go index 24fd3fe84..b974f2d1c 100644 --- a/sidecar/main.go +++ b/sidecar/main.go @@ -56,6 +56,7 @@ func main() { leaderElectionLeaseDuration = flag.Duration("leader-election-lease-duration", 15*time.Second, "Duration, in seconds, that non-leader candidates will wait to force acquire leadership. Defaults to 15 seconds.") leaderElectionRenewDeadline = flag.Duration("leader-election-renew-deadline", 10*time.Second, "Duration, in seconds, that the acting leader will retry refreshing leadership before giving up. Defaults to 10 seconds.") leaderElectionRetryPeriod = flag.Duration("leader-election-retry-period", 5*time.Second, "Duration, in seconds, the LeaderElector clients should wait between tries of actions. Defaults to 5 seconds.") + enableTLS = flag.Bool("enable-tls", false, "Enable TLS(disabled by default)") ) klog.InitFlags(nil) @@ -110,7 +111,7 @@ func main() { klog.Fatalf("Failed to create csiaddonsnode: %v", err) } - sidecarServer := server.NewSidecarServer(*controllerIP, *controllerPort) + sidecarServer := server.NewSidecarServer(*controllerIP, *controllerPort, kubeClient, *enableTLS) sidecarServer.RegisterService(service.NewIdentityServer(csiClient.GetGRPCClient())) sidecarServer.RegisterService(service.NewReclaimSpaceServer(csiClient.GetGRPCClient(), kubeClient, *stagingPath)) sidecarServer.RegisterService(service.NewNetworkFenceServer(csiClient.GetGRPCClient(), kubeClient))