From 8a1d49050a583e6d5e4eab6d32c40401a796cbde Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Tue, 27 Aug 2024 14:13:49 +0200 Subject: [PATCH] feat: support setting multiple TLS certs for different domains on the interceptor proxy (#1116) Signed-off-by: Jan Wozniak --- CHANGELOG.md | 1 + Makefile | 12 ++ .../interceptor/e2e-test/tls/deployment.yaml | 8 + docs/operate.md | 6 + interceptor/config/serving.go | 2 + interceptor/main.go | 168 ++++++++++++++++-- interceptor/main_test.go | 151 +++++++++++++++- pkg/http/server.go | 12 +- pkg/http/server_test.go | 11 +- tests/utils/setup_test.go | 3 + 10 files changed, 346 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bd7a11d..64d57dcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ This changelog keeps track of work items that have been completed and are ready ### New +- **General**: Support setting multiple TLS certs for different domains on the interceptor proxy ([#1116](https://github.com/kedacore/http-add-on/issues/1116)) - **General**: TODO ([#TODO](https://github.com/kedacore/http-add-on/issues/TODO)) ### Improvements diff --git a/Makefile b/Makefile index e77fec31..de26cf87 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,15 @@ DNS.3 = *.interceptor-tls-test-ns endef export DOMAINS +define ABC_DOMAINS +basicConstraints=CA:FALSE +keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment +subjectAltName = @alt_names +[alt_names] +DNS.1 = abc +endef +export ABC_DOMAINS + # Build targets build-operator: @@ -68,6 +77,9 @@ test-certs: rootca-test-certs echo "$$DOMAINS" > certs/domains.ext openssl req -new -nodes -newkey rsa:2048 -keyout certs/tls.key -out certs/tls.csr -subj "/C=US/ST=KedaState/L=KedaCity/O=Keda-Certificates/CN=keda.local" openssl x509 -req -sha256 -days 1024 -in certs/tls.csr -CA certs/RootCA.pem -CAkey certs/RootCA.key -CAcreateserial -extfile certs/domains.ext -out certs/tls.crt + echo "$$ABC_DOMAINS" > certs/abc_domains.ext + openssl req -new -nodes -newkey rsa:2048 -keyout certs/abc.tls.key -out certs/abc.tls.csr -subj "/C=US/ST=KedaState/L=KedaCity/O=Keda-Certificates/CN=abc" + openssl x509 -req -sha256 -days 1024 -in certs/abc.tls.csr -CA certs/RootCA.pem -CAkey certs/RootCA.key -CAcreateserial -extfile certs/abc_domains.ext -out certs/abc.tls.crt clean-test-certs: rm -r certs || true diff --git a/config/interceptor/e2e-test/tls/deployment.yaml b/config/interceptor/e2e-test/tls/deployment.yaml index 0b836524..2f7a4437 100644 --- a/config/interceptor/e2e-test/tls/deployment.yaml +++ b/config/interceptor/e2e-test/tls/deployment.yaml @@ -18,13 +18,21 @@ spec: value: "/certs/tls.crt" - name: KEDA_HTTP_PROXY_TLS_KEY_PATH value: "/certs/tls.key" + - name: KEDA_HTTP_PROXY_TLS_CERT_STORE_PATHS + value: "/additional-certs" - name: KEDA_HTTP_PROXY_TLS_PORT value: "8443" volumeMounts: - readOnly: true mountPath: "/certs" name: certs + - readOnly: true + mountPath: "/additional-certs/abc-certs" + name: abc-certs volumes: - name: certs secret: secretName: keda-tls + - name: abc-certs + secret: + secretName: abc-certs diff --git a/docs/operate.md b/docs/operate.md index 691c3a32..5a5264c6 100644 --- a/docs/operate.md +++ b/docs/operate.md @@ -23,3 +23,9 @@ If you need to provide any headers such as authentication details in order to ut The interceptor proxy has the ability to run both a HTTP and HTTPS server simultaneously to allow you to scale workloads that use either protocol. By default, the interceptor proxy will only serve over HTTP, but this behavior can be changed by configuring the appropriate environment variables on the deployment. The TLS server can be enabled by setting the environment variable `KEDA_HTTP_PROXY_TLS_ENABLED` to `true` on the interceptor deployment (`false` by default). The TLS server will start on port `8443` by default, but this can be configured by setting `KEDA_HTTP_PROXY_TLS_PORT` to your desired port number. The TLS server will require valid TLS certificates to start, the path to the certificates can be configured via the `KEDA_HTTP_PROXY_TLS_CERT_PATH` and `KEDA_HTTP_PROXY_TLS_KEY_PATH` environment variables (`/certs/tls.crt` and `/certs/tls.key` by default). + +For setting multiple TLS certs, set `KEDA_HTTP_PROXY_TLS_CERT_STORE_PATHS` with comma-separated list of directories that will be recursively searched for any valid cert/key pairs. Currently, two naming patterns are supported +* `XYZ.crt` + `XYZ.key` - this is a convention when using Kubernetes Secrets of type tls +* `XYZ.pem` + `XYZ-key.pem` + +The matching between certs and requests is performed during the TLS ClientHelo message, where the SNI service name is compared to SANs provided in each cert and the first matching cert will be used for the rest of the TLS handshake. diff --git a/interceptor/config/serving.go b/interceptor/config/serving.go index fb191394..d0a2ff5d 100644 --- a/interceptor/config/serving.go +++ b/interceptor/config/serving.go @@ -41,6 +41,8 @@ type Serving struct { TLSCertPath string `envconfig:"KEDA_HTTP_PROXY_TLS_CERT_PATH" default:"/certs/tls.crt"` // TLSKeyPath is the path to read the private key file from for the TLS server TLSKeyPath string `envconfig:"KEDA_HTTP_PROXY_TLS_KEY_PATH" default:"/certs/tls.key"` + // TLSCertStorePaths is a comma separated list of paths to read the certificate/key pairs for the TLS server + TLSCertStorePaths string `envconfig:"KEDA_HTTP_PROXY_TLS_CERT_STORE_PATHS" default:""` // TLSPort is the port that the server should serve on if TLS is enabled TLSPort int `envconfig:"KEDA_HTTP_PROXY_TLS_PORT" default:"8443"` } diff --git a/interceptor/main.go b/interceptor/main.go index 2d98776c..56f45116 100644 --- a/interceptor/main.go +++ b/interceptor/main.go @@ -9,10 +9,13 @@ import ( "fmt" "net/http" "os" + "path/filepath" + "strings" "time" "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus/promhttp" + "golang.org/x/exp/maps" "golang.org/x/sync/errgroup" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" @@ -168,7 +171,7 @@ func main() { // start a proxy server with TLS if proxyTLSEnabled { eg.Go(func() error { - proxyTLSConfig := map[string]string{"certificatePath": servingCfg.TLSCertPath, "keyPath": servingCfg.TLSKeyPath} + proxyTLSConfig := map[string]string{"certificatePath": servingCfg.TLSCertPath, "keyPath": servingCfg.TLSKeyPath, "certstorePaths": servingCfg.TLSCertStorePaths} proxyTLSPort := servingCfg.TLSPort setupLog.Info("starting the proxy server with TLS enabled", "port", proxyTLSPort) @@ -219,7 +222,7 @@ func runAdminServer( addr := fmt.Sprintf("0.0.0.0:%d", port) lggr.Info("admin server starting", "address", addr) - return kedahttp.ServeContext(ctx, addr, adminServer, false, nil) + return kedahttp.ServeContext(ctx, addr, adminServer, nil) } func runMetricsServer( @@ -229,7 +232,135 @@ func runMetricsServer( ) error { lggr.Info("starting the prometheus metrics server", "port", metricsCfg.OtelPrometheusExporterPort, "path", "/metrics") addr := fmt.Sprintf("0.0.0.0:%d", metricsCfg.OtelPrometheusExporterPort) - return kedahttp.ServeContext(ctx, addr, promhttp.Handler(), false, nil) + return kedahttp.ServeContext(ctx, addr, promhttp.Handler(), nil) +} + +// addCert adds a certificate to the map of certificates based on the certificate's SANs +func addCert(m map[string]tls.Certificate, certPath, keyPath string, logger logr.Logger) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, fmt.Errorf("error loading certificate and key: %w", err) + } + if cert.Leaf == nil { + if len(cert.Certificate) == 0 { + return nil, fmt.Errorf("no certificate found in certificate chain") + } + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("error parsing certificate: %w", err) + } + } + for _, d := range cert.Leaf.DNSNames { + logger.Info("adding certificate", "dns", d) + m[d] = cert + } + for _, ip := range cert.Leaf.IPAddresses { + logger.Info("adding certificate", "ip", ip.String()) + m[ip.String()] = cert + } + for _, uri := range cert.Leaf.URIs { + logger.Info("adding certificate", "uri", uri.String()) + m[uri.String()] = cert + } + return &cert, nil +} + +func defaultCertPool(logger logr.Logger) *x509.CertPool { + systemCAs, err := x509.SystemCertPool() + if err == nil { + return systemCAs + } + + logger.Info("error loading system CA pool, using empty pool", "error", err) + return x509.NewCertPool() +} + +// getTLSConfig creates a TLS config from KEDA_HTTP_PROXY_TLS_CERT_PATH, KEDA_HTTP_PROXY_TLS_KEY_PATH and KEDA_HTTP_PROXY_TLS_CERTSTORE_PATHS +// The matching between request and certificate is performed by comparing TLS/SNI server name with x509 SANs +func getTLSConfig(tlsConfig map[string]string, logger logr.Logger) (*tls.Config, error) { + certPath := tlsConfig["certificatePath"] + keyPath := tlsConfig["keyPath"] + certStorePaths := tlsConfig["certstorePaths"] + servingTLS := &tls.Config{ + RootCAs: defaultCertPool(logger), + } + var defaultCert *tls.Certificate + + uriDomainsToCerts := make(map[string]tls.Certificate) + if certPath != "" && keyPath != "" { + cert, err := addCert(uriDomainsToCerts, certPath, keyPath, logger) + if err != nil { + return servingTLS, fmt.Errorf("error adding certificate and key: %w", err) + } + defaultCert = cert + rawCert, err := os.ReadFile(certPath) + if err != nil { + return servingTLS, fmt.Errorf("error reading certificate: %w", err) + } + servingTLS.RootCAs.AppendCertsFromPEM(rawCert) + } + + if certStorePaths != "" { + certFiles := make(map[string]string) + keyFiles := make(map[string]string) + dirPaths := strings.Split(certStorePaths, ",") + for _, dir := range dirPaths { + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + switch { + case strings.HasSuffix(path, "-key.pem"): + certID := path[:len(path)-8] + keyFiles[certID] = path + case strings.HasSuffix(path, ".pem"): + certID := path[:len(path)-4] + certFiles[certID] = path + case strings.HasSuffix(path, ".key"): + certID := path[:len(path)-4] + keyFiles[certID] = path + case strings.HasSuffix(path, ".crt"): + certID := path[:len(path)-4] + certFiles[certID] = path + } + return nil + }) + if err != nil { + return servingTLS, fmt.Errorf("error walking certificate store: %w", err) + } + } + + for certID, certPath := range certFiles { + logger.Info("adding certificate", "certID", certID, "certPath", certPath) + keyPath, ok := keyFiles[certID] + if !ok { + return servingTLS, fmt.Errorf("no key found for certificate %s", certPath) + } + if _, err := addCert(uriDomainsToCerts, certPath, keyPath, logger); err != nil { + return servingTLS, fmt.Errorf("error adding certificate %s: %w", certPath, err) + } + rawCert, err := os.ReadFile(certPath) + if err != nil { + return servingTLS, fmt.Errorf("error reading certificate: %w", err) + } + servingTLS.RootCAs.AppendCertsFromPEM(rawCert) + } + } + + servingTLS.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if cert, ok := uriDomainsToCerts[hello.ServerName]; ok { + return &cert, nil + } + if defaultCert != nil { + return defaultCert, nil + } + return nil, fmt.Errorf("no certificate found for %s", hello.ServerName) + } + servingTLS.Certificates = maps.Values(uriDomainsToCerts) + return servingTLS, nil } func runProxyServer( @@ -251,33 +382,29 @@ func runProxyServer( }) go probeHandler.Start(ctx) - tlsCfg := tls.Config{} + var tlsCfg *tls.Config if tlsEnabled { - caCert, err := os.ReadFile(tlsConfig["certificatePath"]) - if err != nil { - logger.Error(fmt.Errorf("error reading file from TLSCertPath"), "error", err) - os.Exit(1) - } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - cert, err := tls.LoadX509KeyPair(tlsConfig["certificatePath"], tlsConfig["keyPath"]) - + cfg, err := getTLSConfig(tlsConfig, logger) if err != nil { - logger.Error(fmt.Errorf("error creating TLS configuration for proxy server"), "error", err) + logger.Error(fmt.Errorf("error creating certGetter for proxy server"), "error", err) os.Exit(1) } - - tlsCfg.RootCAs = caCertPool - tlsCfg.Certificates = []tls.Certificate{cert} + tlsCfg = cfg } var upstreamHandler http.Handler + forwardingTLSCfg := &tls.Config{} + if tlsCfg != nil { + forwardingTLSCfg.RootCAs = tlsCfg.RootCAs + forwardingTLSCfg.Certificates = tlsCfg.Certificates + } + upstreamHandler = newForwardingHandler( logger, dialContextFunc, waitFunc, newForwardingConfigFromTimeouts(timeouts), - &tlsCfg, + forwardingTLSCfg, ) upstreamHandler = middleware.NewCountingMiddleware( q, @@ -302,5 +429,8 @@ func runProxyServer( addr := fmt.Sprintf("0.0.0.0:%d", port) logger.Info("proxy server starting", "address", addr) - return kedahttp.ServeContext(ctx, addr, rootHandler, tlsEnabled, tlsConfig) + if tlsEnabled { + return kedahttp.ServeContext(ctx, addr, rootHandler, tlsCfg) + } + return kedahttp.ServeContext(ctx, addr, rootHandler, nil) } diff --git a/interceptor/main_test.go b/interceptor/main_test.go index 3a08be53..696ab1de 100644 --- a/interceptor/main_test.go +++ b/interceptor/main_test.go @@ -22,6 +22,8 @@ import ( routingtest "github.com/kedacore/http-add-on/pkg/routing/test" ) +const falseStr = "false" + func TestRunProxyServerCountMiddleware(t *testing.T) { const ( port = 8080 @@ -107,7 +109,7 @@ func TestRunProxyServerCountMiddleware(t *testing.T) { resp.StatusCode, ) } - if resp.Header.Get("X-KEDA-HTTP-Cold-Start") != "false" { + if resp.Header.Get("X-KEDA-HTTP-Cold-Start") != falseStr { return fmt.Errorf("expected X-KEDA-HTTP-Cold-Start false, but got %s", resp.Header.Get("X-KEDA-HTTP-Cold-Start")) } return nil @@ -252,7 +254,152 @@ func TestRunProxyServerWithTLSCountMiddleware(t *testing.T) { resp.StatusCode, ) } - if resp.Header.Get("X-KEDA-HTTP-Cold-Start") != "false" { + if resp.Header.Get("X-KEDA-HTTP-Cold-Start") != falseStr { + return fmt.Errorf("expected X-KEDA-HTTP-Cold-Start false, but got %s", resp.Header.Get("X-KEDA-HTTP-Cold-Start")) + } + return nil + }) + time.Sleep(100 * time.Millisecond) + select { + case hostAndCount := <-q.ResizedCh: + r.Equal(namespacedName, hostAndCount.Host) + r.Equal(1, hostAndCount.Count) + case <-time.After(2000 * time.Millisecond): + r.Fail("timeout waiting for +1 queue resize") + } + + // tell the wait func to proceed + select { + case waiterCh <- struct{}{}: + case <-time.After(5 * time.Second): + r.Fail("timeout producing on waiterCh") + } + + select { + case hostAndCount := <-q.ResizedCh: + r.Equal(namespacedName, hostAndCount.Host) + r.Equal(1, hostAndCount.Count) + case <-time.After(2 * time.Second): + r.Fail("timeout waiting for -1 queue resize") + } + + // check the queue to make sure all counts are at 0 + countsPtr, err := q.Current() + r.NoError(err) + counts := countsPtr.Counts + r.Equal(1, len(counts)) + _, foundHost := counts[namespacedName] + r.True( + foundHost, + "couldn't find host %s in the queue", + host, + ) + r.Equal(0, counts[namespacedName].Concurrency) + + done() + r.Error(g.Wait()) +} + +func TestRunProxyServerWithMultipleCertsTLSCountMiddleware(t *testing.T) { + const ( + port = 8443 + host = "samplehost" + ) + r := require.New(t) + ctx, done := context.WithCancel( + context.Background(), + ) + defer done() + + originHdl := kedanet.NewTestHTTPHandlerWrapper( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }), + ) + originSrv, originURL, err := kedanet.StartTestServer(originHdl) + r.NoError(err) + defer originSrv.Close() + originPort, err := strconv.Atoi(originURL.Port()) + r.NoError(err) + g, ctx := errgroup.WithContext(ctx) + q := queue.NewFakeCounter() + + httpso := targetFromURL( + originURL, + originPort, + "testdepl", + "testsvc", + ) + namespacedName := k8s.NamespacedNameFromObject(httpso).String() + + // set up a fake host that we can spoof + // when we later send request to the proxy, + // so that the proxy calculates a URL for that + // host that points to the (above) fake origin + // server + routingTable := routingtest.NewTable() + routingTable.Memory[host] = httpso + + timeouts := &config.Timeouts{} + waiterCh := make(chan struct{}) + waitFunc := func(_ context.Context, _, _ string) (bool, error) { + <-waiterCh + return false, nil + } + + g.Go(func() error { + return runProxyServer( + ctx, + logr.Discard(), + q, + waitFunc, + routingTable, + timeouts, + port, + true, + map[string]string{"certstorePaths": "../certs"}, + ) + }) + + // wait for server to start + time.Sleep(500 * time.Millisecond) + + // make an HTTPs request in the background + g.Go(func() error { + f, err := os.ReadFile("../certs/RootCA.pem") + if err != nil { + t.Errorf("Unable to find RootCA for test, please run tests via `make test`") + } + rootCAs, _ := x509.SystemCertPool() + rootCAs.AppendCertsFromPEM(f) + + http.DefaultClient.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: rootCAs}, + } + + req, err := http.NewRequest( + "GET", + fmt.Sprintf( + "https://localhost:%d", port, + ), nil, + ) + if err != nil { + return err + } + req.Host = host + // Allow us to use our self made certs + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf( + "unexpected status code: %d", + resp.StatusCode, + ) + } + if resp.Header.Get("X-KEDA-HTTP-Cold-Start") != falseStr { return fmt.Errorf("expected X-KEDA-HTTP-Cold-Start false, but got %s", resp.Header.Get("X-KEDA-HTTP-Cold-Start")) } return nil diff --git a/pkg/http/server.go b/pkg/http/server.go index 062154bd..f037a5d1 100644 --- a/pkg/http/server.go +++ b/pkg/http/server.go @@ -2,15 +2,17 @@ package http import ( "context" + "crypto/tls" "net/http" "github.com/kedacore/http-add-on/pkg/util" ) -func ServeContext(ctx context.Context, addr string, hdl http.Handler, tlsEnabled bool, tlsConfig map[string]string) error { +func ServeContext(ctx context.Context, addr string, hdl http.Handler, tlsConfig *tls.Config) error { srv := &http.Server{ - Handler: hdl, - Addr: addr, + Handler: hdl, + Addr: addr, + TLSConfig: tlsConfig, } go func() { @@ -22,8 +24,8 @@ func ServeContext(ctx context.Context, addr string, hdl http.Handler, tlsEnabled } }() - if tlsEnabled { - return srv.ListenAndServeTLS(tlsConfig["certificatePath"], tlsConfig["keyPath"]) + if tlsConfig != nil { + return srv.ListenAndServeTLS("", "") } return srv.ListenAndServe() diff --git a/pkg/http/server_test.go b/pkg/http/server_test.go index a4b38a3a..ce0d9061 100644 --- a/pkg/http/server_test.go +++ b/pkg/http/server_test.go @@ -2,6 +2,7 @@ package http import ( "context" + "crypto/tls" "errors" "net/http" "testing" @@ -29,7 +30,7 @@ func TestServeContext(t *testing.T) { done() }() start := time.Now() - err := ServeContext(ctx, addr, hdl, false, map[string]string{}) + err := ServeContext(ctx, addr, hdl, nil) elapsed := time.Since(start) r.Error(err) @@ -57,7 +58,13 @@ func TestServeContextWithTLS(t *testing.T) { done() }() start := time.Now() - err := ServeContext(ctx, addr, hdl, true, map[string]string{"certificatePath": "../../certs/tls.crt", "keyPath": "../../certs/tls.key"}) + tlsConfig := tls.Config{ + GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair("../../certs/tls.crt", "../../certs/tls.key") + return &cert, err + }, + } + err := ServeContext(ctx, addr, hdl, &tlsConfig) elapsed := time.Since(start) r.Error(err) diff --git a/tests/utils/setup_test.go b/tests/utils/setup_test.go index 1b30db30..e7e52dcc 100644 --- a/tests/utils/setup_test.go +++ b/tests/utils/setup_test.go @@ -251,6 +251,9 @@ func TestSetupTLSConfiguration(t *testing.T) { _, err = ExecuteCommand("kubectl -n keda create secret tls keda-tls --cert ../../certs/tls.crt --key ../../certs/tls.key") require.NoErrorf(t, err, "could not create tls cert secret in keda namespace - %s", err) + + _, err = ExecuteCommand("kubectl -n keda create secret tls abc-certs --cert ../../certs/abc.tls.crt --key ../../certs/abc.tls.key") + require.NoErrorf(t, err, "could not create tls cert secret in keda namespace - %s", err) } func TestDeployKEDAHttpAddOn(t *testing.T) {