From 00304db2b98927228b8775a7b285a5e22e3949ba Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Tue, 19 Dec 2023 07:24:04 +0100 Subject: [PATCH] Add upstream TLS trust from CM bundles (#1171) * Add upstream TLS trust from CM bundles * Lint warnings * Get Secret and ConfigMaps from system namespace * Use separate CA for ConfigMap * Use networking.TrustBundleLabelKey * Update verbosity logs in happy path * Review improvements * add error when trust-bundle is empty --- pkg/config/config.go | 13 - pkg/generator/caches_test.go | 6 +- pkg/generator/ingress_translator.go | 135 ++++++-- pkg/generator/ingress_translator_test.go | 327 ++++++++++++++++-- pkg/reconciler/ingress/controller.go | 39 +++ .../informers/core/v1/configmap/configmap.go | 50 +++ vendor/modules.txt | 1 + 7 files changed, 495 insertions(+), 76 deletions(-) create mode 100644 vendor/knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/configmap/configmap.go diff --git a/pkg/config/config.go b/pkg/config/config.go index af0e40359..b3bcf7447 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -65,10 +65,6 @@ const ( // to indicate that http2 should not be enabled for it. disableHTTP2AnnotationKey = "kourier.knative.dev/disable-http2" - // ServingNamespaceEnv is an env variable specifying where the serving is deployed. - // e.g. OpenShift deploys Kourier in different namespace so `system.Namespace()` does not work. - ServingNamespaceEnv = "SERVING_NAMESPACE" - // trustedHopsCount Configure the number of additional ingress proxy hops from the // right side of the x-forwarded-for HTTP header to trust. trustedHopsCount = "trusted-hops-count" @@ -98,15 +94,6 @@ func GatewayNamespace() string { return namespace } -// ServingNamespace returns the namespace where the serving is deployed. -func ServingNamespace() string { - namespace := os.Getenv(ServingNamespaceEnv) - if namespace == "" { - return system.Namespace() - } - return namespace -} - // GetDisableHTTP2 specifies whether http2 is going to be disabled func GetDisableHTTP2(annotations map[string]string) (val string) { return disableHTTP2Annotation.Value(annotations) diff --git a/pkg/generator/caches_test.go b/pkg/generator/caches_test.go index b2355a32b..f2b5c3381 100644 --- a/pkg/generator/caches_test.go +++ b/pkg/generator/caches_test.go @@ -289,7 +289,7 @@ func TestLocalTLSListener(t *testing.T) { Name: "test-ca", }, Data: map[string][]byte{ - certificates.CaCertName: cert, + certificates.CaCertName: secretCert, }, } @@ -494,7 +494,7 @@ func createTestDataForIngress( externalSNIMatches: []*envoy.SNIMatch{{ Hosts: []string{"foo.example.com"}, CertSource: types.NamespacedName{Namespace: "secretns", Name: "secretname"}, - CertificateChain: cert, + CertificateChain: secretCert, PrivateKey: privateKey}}, } @@ -530,7 +530,7 @@ func TestValidateIngress(t *testing.T) { externalSNIMatches: []*envoy.SNIMatch{{ Hosts: []string{"foo.example.com"}, CertSource: types.NamespacedName{Namespace: "secretns", Name: "secretname"}, - CertificateChain: cert, + CertificateChain: secretCert, PrivateKey: privateKey}}, } diff --git a/pkg/generator/ingress_translator.go b/pkg/generator/ingress_translator.go index b6af55626..a14b48a3f 100644 --- a/pkg/generator/ingress_translator.go +++ b/pkg/generator/ingress_translator.go @@ -19,6 +19,8 @@ package generator import ( "context" "crypto/tls" + "crypto/x509" + "encoding/pem" "fmt" "os" "strings" @@ -31,6 +33,7 @@ import ( tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" envoymatcherv3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" "github.com/envoyproxy/go-control-plane/pkg/wellknown" + "go.uber.org/zap" "google.golang.org/protobuf/types/known/anypb" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -38,11 +41,13 @@ import ( pkgconfig "knative.dev/net-kourier/pkg/config" envoy "knative.dev/net-kourier/pkg/envoy/api" "knative.dev/net-kourier/pkg/reconciler/ingress/config" + "knative.dev/networking/pkg/apis/networking" "knative.dev/networking/pkg/apis/networking/v1alpha1" "knative.dev/networking/pkg/certificates" netconfig "knative.dev/networking/pkg/config" "knative.dev/pkg/kmeta" "knative.dev/pkg/logging" + "knative.dev/pkg/system" "knative.dev/pkg/tracker" ) @@ -58,22 +63,25 @@ type translatedIngress struct { } type IngressTranslator struct { - secretGetter func(ns, name string) (*corev1.Secret, error) - endpointsGetter func(ns, name string) (*corev1.Endpoints, error) - serviceGetter func(ns, name string) (*corev1.Service, error) - tracker tracker.Interface + secretGetter func(ns, name string) (*corev1.Secret, error) + nsConfigmapGetter func(label string) ([]*corev1.ConfigMap, error) + endpointsGetter func(ns, name string) (*corev1.Endpoints, error) + serviceGetter func(ns, name string) (*corev1.Service, error) + tracker tracker.Interface } func NewIngressTranslator( secretGetter func(ns, name string) (*corev1.Secret, error), + nsConfigmapGetter func(label string) ([]*corev1.ConfigMap, error), endpointsGetter func(ns, name string) (*corev1.Endpoints, error), serviceGetter func(ns, name string) (*corev1.Service, error), tracker tracker.Interface) IngressTranslator { return IngressTranslator{ - secretGetter: secretGetter, - endpointsGetter: endpointsGetter, - serviceGetter: serviceGetter, - tracker: tracker, + secretGetter: secretGetter, + nsConfigmapGetter: nsConfigmapGetter, + endpointsGetter: endpointsGetter, + serviceGetter: serviceGetter, + tracker: tracker, } } @@ -108,6 +116,18 @@ func (translator *IngressTranslator) translateIngress(ctx context.Context, ingre cfg := config.FromContext(ctx) + var err error + var trustChain []byte + if cfg.Network.SystemInternalTLSEnabled() { + trustChain, err = translator.buildTrustChain(logger) + if err != nil { + return nil, err + } + if trustChain == nil { + return nil, fmt.Errorf("failed to build trust-chain, as no valid CA certificate was provided. Please make sure to provide a valid trust-bundle before enabling `system-internal-tls`") + } + } + for i, rule := range ingress.Spec.Rules { ruleName := fmt.Sprintf("(%s/%s).Rules[%d]", ingress.Namespace, ingress.Name, i) @@ -144,10 +164,9 @@ func (translator *IngressTranslator) translateIngress(ctx context.Context, ingre // Match the ingress' port with a port on the Service to find the target. // Also find out if the target supports HTTP2. var ( - externalPort = int32(80) - targetPort = int32(80) - http2 = false - httpsPortUsed = false + externalPort = int32(80) + targetPort = int32(80) + http2 = false ) for _, port := range service.Spec.Ports { if port.Port == split.ServicePort.IntVal || port.Name == split.ServicePort.StrVal { @@ -157,9 +176,6 @@ func (translator *IngressTranslator) translateIngress(ctx context.Context, ingre if port.Name == "http2" || port.Name == "h2c" { http2 = true } - if port.Port == split.ServicePort.IntVal && port.Name == "https" { - httpsPortUsed = true - } } // Disable HTTP2 if the annotation is specified. @@ -196,14 +212,10 @@ func (translator *IngressTranslator) translateIngress(ctx context.Context, ingre var transportSocket *envoycorev3.TransportSocket - // TODO: drop this configmap check - issues/968 - // We could determine whether system-internal-tls is enabled or disabled via the flag only, - // but all conformance tests need to be updated to have the port name so we check the configmap as well. - - // As Ingress with RewriteHost points to ExternalService(kourier-internal), we don't enable TLS. - if (cfg.Network.SystemInternalTLSEnabled() || httpsPortUsed) && httpPath.RewriteHost == "" { + // As Ingress with RewriteHost points to ExternalService(kourier-internal), we don't enable upstream TLS. + if (cfg.Network.SystemInternalTLSEnabled()) && httpPath.RewriteHost == "" { var err error - transportSocket, err = translator.createUpstreamTransportSocket(http2, split.ServiceNamespace) + transportSocket, err = translator.createUpstreamTransportSocket(http2, split.ServiceNamespace, trustChain) if err != nil { return nil, err } @@ -318,16 +330,12 @@ func (translator *IngressTranslator) translateIngressTLS(ingressTLS v1alpha1.Ing return sniMatch, nil } -func (translator *IngressTranslator) createUpstreamTransportSocket(http2 bool, namespace string) (*envoycorev3.TransportSocket, error) { - caSecret, err := translator.secretGetter(pkgconfig.ServingNamespace(), netconfig.ServingRoutingCertName) - if err != nil { - return nil, fmt.Errorf("failed to fetch activator CA secret: %w", err) - } +func (translator *IngressTranslator) createUpstreamTransportSocket(http2 bool, namespace string, trustChain []byte) (*envoycorev3.TransportSocket, error) { var alpnProtocols string if http2 { alpnProtocols = "h2" } - tlsAny, err := anypb.New(createUpstreamTLSContext(caSecret.Data[certificates.CaCertName], namespace, alpnProtocols)) + tlsAny, err := anypb.New(createUpstreamTLSContext(trustChain, namespace, alpnProtocols)) if err != nil { return nil, err } @@ -339,7 +347,7 @@ func (translator *IngressTranslator) createUpstreamTransportSocket(http2 bool, n }, nil } -func createUpstreamTLSContext(caCertificate []byte, namespace string, alpnProtocols ...string) *tlsv3.UpstreamTlsContext { +func createUpstreamTLSContext(trustChain []byte, namespace string, alpnProtocols ...string) *tlsv3.UpstreamTlsContext { return &tlsv3.UpstreamTlsContext{ CommonTlsContext: &tlsv3.CommonTlsContext{ AlpnProtocols: alpnProtocols, @@ -351,7 +359,7 @@ func createUpstreamTLSContext(caCertificate []byte, namespace string, alpnProtoc ValidationContext: &tlsv3.CertificateValidationContext{ TrustedCa: &envoycorev3.DataSource{ Specifier: &envoycorev3.DataSource_InlineBytes{ - InlineBytes: caCertificate, + InlineBytes: trustChain, }, }, MatchTypedSubjectAltNames: []*tlsv3.SubjectAltNameMatcher{{ @@ -377,6 +385,52 @@ func createUpstreamTLSContext(caCertificate []byte, namespace string, alpnProtoc } } +// CA can optionally be in `ca.crt` in the `routing-serving-certs` secret +// and/or configured using a trust-bundle via ConfigMap that has the defined label `knative-ca-trust-bundle`. +// Our upstream TLS context needs to trust them all. +func (translator *IngressTranslator) buildTrustChain(logger *zap.SugaredLogger) ([]byte, error) { + var trustChain []byte + + routingCA, err := translator.secretGetter(system.Namespace(), netconfig.ServingRoutingCertName) + if err != nil { + return nil, fmt.Errorf("failed to fetch Secret %s/%s: %w", system.Namespace(), netconfig.ServingRoutingCertName, err) + } + routingCABytes := routingCA.Data[certificates.CaCertName] + if len(routingCABytes) > 0 { + if err = checkCertBundle(routingCABytes); err != nil { + logger.Warnf("CA from Secret %s/%s[%s] is invalid and will be ignored: %v", + system.Namespace(), netconfig.ServingRoutingCertName, certificates.CaCertName, err) + } else { + logger.Debugf("Adding CA from Secret %s/%s[%s] to trust chain", system.Namespace(), netconfig.ServingRoutingCertName, certificates.CaCertName) + trustChain = routingCABytes + } + } + + cms, err := translator.nsConfigmapGetter(networking.TrustBundleLabelKey) + if err != nil { + return nil, fmt.Errorf("failed to fetch Configmaps with label: %s in namespace: %s: %w", networking.TrustBundleLabelKey, system.Namespace(), err) + } + + newline := []byte("\n") + for _, cm := range cms { + for _, bundle := range cm.Data { + if err = checkCertBundle([]byte(bundle)); err != nil { + logger.Warnf("CA bundle from Configmap %s/%s is invalid and will be ignored: %v", + system.Namespace(), cm.Name, err) + } else { + logger.Debugf("Adding CA bundle from Configmap %s/%s to trust chain", system.Namespace(), cm.Name) + if len(trustChain) > 0 { + // Make sure we always have at least one newline between bundles, multiple ones are ok + trustChain = append(trustChain, newline...) + } + trustChain = append(trustChain, []byte(bundle)...) + } + } + } + + return trustChain, nil +} + func trackSecret(t tracker.Interface, ns, name string, ingress *v1alpha1.Ingress) error { return t.TrackReference(tracker.Reference{ Kind: "Secret", @@ -466,3 +520,24 @@ func domainsForRule(rule v1alpha1.IngressRule) []string { } return domains } + +func checkCertBundle(certs []byte) error { + for block, rest := pem.Decode(certs); block != nil || len(rest) > 0; block, rest = pem.Decode(rest) { + if block != nil { + switch block.Type { + case "CERTIFICATE": + _, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("failed to parse certificate. Invalid certificate found: %s, %v", block.Bytes, err.Error()) + } + + default: + return fmt.Errorf("failed to parse bundle. Bundle contains something other than a certificate. Type: %s, block: %s", block.Type, block.Bytes) + } + } else { + // if the last certificate is parsed, and we still have rest, there are additional unwanted things in the CM + return fmt.Errorf("failed to parse bundle. Bundle contains something other than a certificate: %s", rest) + } + } + return nil +} diff --git a/pkg/generator/ingress_translator_test.go b/pkg/generator/ingress_translator_test.go index ea6d98948..065db4160 100644 --- a/pkg/generator/ingress_translator_test.go +++ b/pkg/generator/ingress_translator_test.go @@ -41,10 +41,12 @@ import ( "k8s.io/client-go/kubernetes/fake" envoy "knative.dev/net-kourier/pkg/envoy/api" "knative.dev/net-kourier/pkg/reconciler/ingress/config" + "knative.dev/networking/pkg/apis/networking" "knative.dev/networking/pkg/apis/networking/v1alpha1" "knative.dev/networking/pkg/certificates" netconfig "knative.dev/networking/pkg/config" pkgtest "knative.dev/pkg/reconciler/testing" + "knative.dev/pkg/system" ) func TestIngressTranslator(t *testing.T) { @@ -164,7 +166,7 @@ func TestIngressTranslator(t *testing.T) { Namespace: "secretns", Name: "secretname", }, - CertificateChain: cert, + CertificateChain: secretCert, PrivateKey: privateKey, }}, localSNIMatches: []*envoy.SNIMatch{}, @@ -239,7 +241,7 @@ func TestIngressTranslator(t *testing.T) { Namespace: "secretns", Name: "secretname", }, - CertificateChain: cert, + CertificateChain: secretCert, PrivateKey: privateKey, }}, externalSNIMatches: []*envoy.SNIMatch{}, @@ -333,7 +335,7 @@ func TestIngressTranslator(t *testing.T) { Namespace: "secretns", Name: "secretname", }, - CertificateChain: cert, + CertificateChain: secretCert, PrivateKey: privateKey, }}, localSNIMatches: []*envoy.SNIMatch{}, @@ -410,7 +412,7 @@ func TestIngressTranslator(t *testing.T) { Namespace: "secretns", Name: "secretname", }, - CertificateChain: cert, + CertificateChain: secretCert, PrivateKey: privateKey, }}, clusters: []*v3.Cluster{ @@ -761,6 +763,9 @@ func TestIngressTranslator(t *testing.T) { func(ns, name string) (*corev1.Secret, error) { return kubeclient.CoreV1().Secrets(ns).Get(ctx, name, metav1.GetOptions{}) }, + func(label string) ([]*corev1.ConfigMap, error) { + return getConfigmaps(ctx, kubeclient) + }, func(ns, name string) (*corev1.Endpoints, error) { return kubeclient.CoreV1().Endpoints(ns).Get(ctx, name, metav1.GetOptions{}) }, @@ -797,6 +802,7 @@ var ( upstreamTLSConfig = &config.Config{ Network: &netconfig.Config{ ExternalDomainTLS: false, + SystemInternalTLS: netconfig.EncryptionEnabled, }, } ) @@ -861,7 +867,7 @@ func TestIngressTranslatorWithHTTPOptionDisabled(t *testing.T) { Namespace: "secretns", Name: "secretname", }, - CertificateChain: cert, + CertificateChain: secretCert, PrivateKey: privateKey, }}, localSNIMatches: []*envoy.SNIMatch{}, @@ -938,7 +944,7 @@ func TestIngressTranslatorWithHTTPOptionDisabled(t *testing.T) { Namespace: "secretns", Name: "secretname", }, - CertificateChain: cert, + CertificateChain: secretCert, PrivateKey: privateKey, }}, clusters: []*v3.Cluster{ @@ -970,6 +976,9 @@ func TestIngressTranslatorWithHTTPOptionDisabled(t *testing.T) { func(ns, name string) (*corev1.Secret, error) { return kubeclient.CoreV1().Secrets(ns).Get(ctx, name, metav1.GetOptions{}) }, + func(label string) ([]*corev1.ConfigMap, error) { + return getConfigmaps(ctx, kubeclient) + }, func(ns, name string) (*corev1.Endpoints, error) { return kubeclient.CoreV1().Endpoints(ns).Get(ctx, name, metav1.GetOptions{}) }, @@ -991,10 +1000,11 @@ func TestIngressTranslatorWithHTTPOptionDisabled(t *testing.T) { func TestIngressTranslatorWithUpstreamTLS(t *testing.T) { tests := []struct { - name string - in *v1alpha1.Ingress - state []runtime.Object - want *translatedIngress + name string + in *v1alpha1.Ingress + state []runtime.Object + want *translatedIngress + wantErr bool }{{ name: "simple", in: ing("simplens", "simplename", func(ing *v1alpha1.Ingress) { @@ -1049,7 +1059,7 @@ func TestIngressTranslatorWithUpstreamTLS(t *testing.T) { false, &envoycorev3.TransportSocket{ Name: wellknown.TransportSocketTls, - ConfigType: typedConfig(false), + ConfigType: typedConfig(false, secretCert), }, v3.Cluster_STATIC, ), @@ -1123,7 +1133,7 @@ func TestIngressTranslatorWithUpstreamTLS(t *testing.T) { true, /* http2 */ &envoycorev3.TransportSocket{ Name: wellknown.TransportSocketTls, - ConfigType: typedConfig(true), + ConfigType: typedConfig(true, secretCert), }, v3.Cluster_STATIC, ), @@ -1198,7 +1208,7 @@ func TestIngressTranslatorWithUpstreamTLS(t *testing.T) { false, /* http2 */ &envoycorev3.TransportSocket{ Name: wellknown.TransportSocketTls, - ConfigType: typedConfig(false), + ConfigType: typedConfig(false, secretCert), }, v3.Cluster_STATIC, ), @@ -1273,7 +1283,143 @@ func TestIngressTranslatorWithUpstreamTLS(t *testing.T) { true, /* http2 */ &envoycorev3.TransportSocket{ Name: wellknown.TransportSocketTls, - ConfigType: typedConfig(true), + ConfigType: typedConfig(true, secretCert), + }, + v3.Cluster_STATIC, + ), + }, + externalVirtualHosts: vHosts, + externalTLSVirtualHosts: []*route.VirtualHost{}, + localVirtualHosts: vHosts, + localTLSVirtualHosts: []*route.VirtualHost{}, + } + }(), + }, { + name: "valid CAs from secret and configmap", + in: ing("simplens", "simplename", func(ing *v1alpha1.Ingress) { + ing.Spec.Rules[0].HTTP.Paths[0].RewriteHost = "" + ing.Spec.Rules[0].HTTP.Paths[0].Splits[0].IngressBackend.ServicePort = intstr.FromInt(443) + }), + state: []runtime.Object{ + svc("servicens", "servicename"), + eps("servicens", "servicename"), + caSecret, + validCAConfigmap, + }, + want: func() *translatedIngress { + vHosts := []*route.VirtualHost{ + envoy.NewVirtualHost( + "(simplens/simplename).Rules[0]", + []string{"foo.example.com", "foo.example.com:*"}, + []*route.Route{envoy.NewRoute( + "(simplens/simplename).Rules[0].Paths[/test]", + []*route.HeaderMatcher{{ + Name: "testheader", + HeaderMatchSpecifier: &route.HeaderMatcher_StringMatch{ + StringMatch: &envoymatcherv3.StringMatcher{ + MatchPattern: &envoymatcherv3.StringMatcher_Exact{ + Exact: "foo", + }, + }, + }, + }}, + "/test", + []*route.WeightedCluster_ClusterWeight{ + envoy.NewWeightedCluster("servicens/servicename", 100, map[string]string{"baz": "gna"}), + }, + 0, + map[string]string{"foo": "bar"}, + ""), + }, + ), + } + + return &translatedIngress{ + name: types.NamespacedName{ + Namespace: "simplens", + Name: "simplename", + }, + externalSNIMatches: []*envoy.SNIMatch{}, + localSNIMatches: []*envoy.SNIMatch{}, + clusters: []*v3.Cluster{ + envoy.NewCluster( + "servicens/servicename", + 5*time.Second, + lbHTTPSEndpoints, + false, + &envoycorev3.TransportSocket{ + Name: wellknown.TransportSocketTls, + ConfigType: typedConfig(false, combineCerts(secretCert, configmapCert)), + }, + v3.Cluster_STATIC, + ), + }, + externalVirtualHosts: vHosts, + externalTLSVirtualHosts: []*route.VirtualHost{}, + localVirtualHosts: vHosts, + localTLSVirtualHosts: []*route.VirtualHost{}, + } + }(), + }, { + name: "valid CA from configmap", + in: ing("simplens", "simplename", func(ing *v1alpha1.Ingress) { + ing.Spec.Rules[0].HTTP.Paths[0].RewriteHost = "" + ing.Spec.Rules[0].HTTP.Paths[0].Splits[0].IngressBackend.ServicePort = intstr.FromInt(443) + }), + state: []runtime.Object{ + svc("servicens", "servicename"), + eps("servicens", "servicename"), + func() runtime.Object { + s := caSecret.DeepCopy() + delete(s.Data, certificates.CaCertName) + return s + }(), + validCAConfigmap, + }, + want: func() *translatedIngress { + vHosts := []*route.VirtualHost{ + envoy.NewVirtualHost( + "(simplens/simplename).Rules[0]", + []string{"foo.example.com", "foo.example.com:*"}, + []*route.Route{envoy.NewRoute( + "(simplens/simplename).Rules[0].Paths[/test]", + []*route.HeaderMatcher{{ + Name: "testheader", + HeaderMatchSpecifier: &route.HeaderMatcher_StringMatch{ + StringMatch: &envoymatcherv3.StringMatcher{ + MatchPattern: &envoymatcherv3.StringMatcher_Exact{ + Exact: "foo", + }, + }, + }, + }}, + "/test", + []*route.WeightedCluster_ClusterWeight{ + envoy.NewWeightedCluster("servicens/servicename", 100, map[string]string{"baz": "gna"}), + }, + 0, + map[string]string{"foo": "bar"}, + ""), + }, + ), + } + + return &translatedIngress{ + name: types.NamespacedName{ + Namespace: "simplens", + Name: "simplename", + }, + externalSNIMatches: []*envoy.SNIMatch{}, + localSNIMatches: []*envoy.SNIMatch{}, + clusters: []*v3.Cluster{ + envoy.NewCluster( + "servicens/servicename", + 5*time.Second, + lbHTTPSEndpoints, + false, + &envoycorev3.TransportSocket{ + Name: wellknown.TransportSocketTls, + ConfigType: typedConfig(false, configmapCert), }, v3.Cluster_STATIC, ), @@ -1284,6 +1430,40 @@ func TestIngressTranslatorWithUpstreamTLS(t *testing.T) { localTLSVirtualHosts: []*route.VirtualHost{}, } }(), + }, { + name: "invalid CA from configmap", + in: ing("simplens", "simplename", func(ing *v1alpha1.Ingress) { + ing.Spec.Rules[0].HTTP.Paths[0].RewriteHost = "" + ing.Spec.Rules[0].HTTP.Paths[0].Splits[0].IngressBackend.ServicePort = intstr.FromInt(443) + }), + state: []runtime.Object{ + svc("servicens", "servicename"), + eps("servicens", "servicename"), + func() runtime.Object { + s := caSecret.DeepCopy() + delete(s.Data, certificates.CaCertName) + return s + }(), + invalidCAConfigmap, + }, + wantErr: true, + }, { + name: "partially valid CA from configmap", + in: ing("simplens", "simplename", func(ing *v1alpha1.Ingress) { + ing.Spec.Rules[0].HTTP.Paths[0].RewriteHost = "" + ing.Spec.Rules[0].HTTP.Paths[0].Splits[0].IngressBackend.ServicePort = intstr.FromInt(443) + }), + state: []runtime.Object{ + svc("servicens", "servicename"), + eps("servicens", "servicename"), + func() runtime.Object { + s := caSecret.DeepCopy() + delete(s.Data, certificates.CaCertName) + return s + }(), + partiallyValidCAConfigmap, + }, + wantErr: true, }} for _, test := range tests { @@ -1297,6 +1477,9 @@ func TestIngressTranslatorWithUpstreamTLS(t *testing.T) { func(ns, name string) (*corev1.Secret, error) { return kubeclient.CoreV1().Secrets(ns).Get(ctx, name, metav1.GetOptions{}) }, + func(label string) ([]*corev1.ConfigMap, error) { + return getConfigmaps(ctx, kubeclient) + }, func(ns, name string) (*corev1.Endpoints, error) { return kubeclient.CoreV1().Endpoints(ns).Get(ctx, name, metav1.GetOptions{}) }, @@ -1307,7 +1490,7 @@ func TestIngressTranslatorWithUpstreamTLS(t *testing.T) { ) got, err := translator.translateIngress(ctx, test.in, false) - assert.NilError(t, err) + assert.Equal(t, err != nil, test.wantErr) assert.DeepEqual(t, got, test.want, cmp.AllowUnexported(translatedIngress{}), protocmp.Transform(), @@ -1396,6 +1579,9 @@ func TestIngressTranslatorHTTP01Challenge(t *testing.T) { func(ns, name string) (*corev1.Secret, error) { return kubeclient.CoreV1().Secrets(ns).Get(ctx, name, metav1.GetOptions{}) }, + func(label string) ([]*corev1.ConfigMap, error) { + return getConfigmaps(ctx, kubeclient) + }, func(ns, name string) (*corev1.Endpoints, error) { return kubeclient.CoreV1().Endpoints(ns).Get(ctx, name, metav1.GetOptions{}) }, @@ -1506,6 +1692,9 @@ func TestIngressTranslatorDomainMappingDisableHTTP2(t *testing.T) { func(ns, name string) (*corev1.Secret, error) { return kubeclient.CoreV1().Secrets(ns).Get(ctx, name, metav1.GetOptions{}) }, + func(label string) ([]*corev1.ConfigMap, error) { + return getConfigmaps(ctx, kubeclient) + }, func(ns, name string) (*corev1.Endpoints, error) { return kubeclient.CoreV1().Endpoints(ns).Get(ctx, name, metav1.GetOptions{}) }, @@ -1662,6 +1851,19 @@ func ingHTTP01Challenge(ns, name string, opts ...func(*v1alpha1.Ingress)) *v1alp return ingress } +func getConfigmaps(ctx context.Context, kubeclient *fake.Clientset) ([]*corev1.ConfigMap, error) { + cms, err := kubeclient.CoreV1().ConfigMaps(system.Namespace()).List(ctx, metav1.ListOptions{LabelSelector: networking.TrustBundleLabelKey}) + if err != nil { + return nil, err + } + result := make([]*corev1.ConfigMap, 0) + for _, c := range cms.Items { + ci := c + result = append(result, &ci) + } + return result, nil +} + var lbEndpoints = []*endpoint.LbEndpoint{ envoy.NewLBEndpoint("2.2.2.2", 8080), envoy.NewLBEndpoint("3.3.3.3", 8080), @@ -1681,16 +1883,27 @@ var lbEndpointHTTP01Challenge = []*endpoint.LbEndpoint{ } var ( - cert = []byte(rsaCertPEM) - invalidCert = []byte(invalidRsaCertPEM) - privateKey = []byte(rsaKeyPEM) - secret = &corev1.Secret{ + secretCert = []byte(rsaSecretCertPEM) + configmapCert = []byte(rsaConfigmapCertPEM) + invalidCert = []byte(invalidRsaCertPEM) + privateKey = []byte(rsaKeyPEM) + secret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "secretns", Name: "secretname", }, Data: map[string][]byte{ - "tls.crt": cert, + "tls.crt": secretCert, + "tls.key": privateKey, + }, + } + invalidSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "secretns", + Name: "secretname", + }, + Data: map[string][]byte{ + "tls.crt": invalidCert, "tls.key": privateKey, }, } @@ -1700,22 +1913,48 @@ var ( Name: netconfig.ServingRoutingCertName, }, Data: map[string][]byte{ - certificates.CaCertName: cert, + certificates.CaCertName: secretCert, }, } - invalidSecret = &corev1.Secret{ + validCAConfigmap = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Namespace: "secretns", - Name: "secretname", + Namespace: "knative-testing", + Name: "valid-ca", + Labels: map[string]string{ + networking.TrustBundleLabelKey: "true", + }, }, - Data: map[string][]byte{ - "tls.crt": invalidCert, - "tls.key": privateKey, + Data: map[string]string{ + certificates.CaCertName: string(configmapCert), + }, + } + invalidCAConfigmap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "knative-testing", + Name: "invalid-ca", + Labels: map[string]string{ + networking.TrustBundleLabelKey: "true", + }, + }, + Data: map[string]string{ + certificates.CaCertName: "NOT A VALID CA", + }, + } + partiallyValidCAConfigmap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "knative-testing", + Name: "partially-valid-ca", + Labels: map[string]string{ + networking.TrustBundleLabelKey: "true", + }, + }, + Data: map[string]string{ + certificates.CaCertName: string(configmapCert) + "\n" + string(invalidCert), }, } ) -func typedConfig(http2 bool) *envoycorev3.TransportSocket_TypedConfig { +func typedConfig(http2 bool, expectedCert []byte) *envoycorev3.TransportSocket_TypedConfig { alpn := []string{""} if http2 { alpn = []string{"h2"} @@ -1731,7 +1970,7 @@ func typedConfig(http2 bool) *envoycorev3.TransportSocket_TypedConfig { ValidationContext: &auth.CertificateValidationContext{ TrustedCa: &envoycorev3.DataSource{ Specifier: &envoycorev3.DataSource_InlineBytes{ - InlineBytes: cert, + InlineBytes: expectedCert, }, }, MatchTypedSubjectAltNames: []*auth.SubjectAltNameMatcher{{ @@ -1766,7 +2005,7 @@ INVALID ` // Copied from https://go.dev/src/crypto/tls/tls_test.go -var rsaCertPEM = `-----BEGIN CERTIFICATE----- +var rsaSecretCertPEM = `-----BEGIN CERTIFICATE----- MIIB0zCCAX2gAwIBAgIJAI/M7BYjwB+uMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX aWRnaXRzIFB0eSBMdGQwHhcNMTIwOTEyMjE1MjAyWhcNMTUwOTEyMjE1MjAyWjBF @@ -1780,6 +2019,27 @@ r5QuVbpQhH6u+0UgcW0jp9QwpxoPTLTWGXEWBBBurxFwiCBhkQ+V -----END CERTIFICATE----- ` +// Selfsigned CA for testing trust-bundles +var rsaConfigmapCertPEM = `-----BEGIN CERTIFICATE----- +MIIDDTCCAfWgAwIBAgIQMQuip05h7NLQq2TB+j9ZmTANBgkqhkiG9w0BAQsFADAW +MRQwEgYDVQQDEwtrbmF0aXZlLmRldjAeFw0yMzExMjIwOTAwNDhaFw0yNDAyMjAw +OTAwNDhaMBYxFDASBgNVBAMTC2tuYXRpdmUuZGV2MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEA3clC3CV7sy0TpUKNuTku6QmP9z8JUCbLCPCLACCUc1zG +FEokqOva6TakgvAntXLkB3TEsbdCJlNm6qFbbko6DBfX6rEggqZs40x3/T+KH66u +4PvMT3fzEtaMJDK/KQOBIvVHrKmPkvccUYK/qWY7rgBjVjjLVSJrCn4dKaEZ2JNr +Fd0KNnaaW/dP9/FvviLqVJvHnTMHH5qyRRr1kUGTrc8njRKwpHcnUdauiDoWRKxo +Zlyy+MhQfdbbyapX984WsDjCvrDXzkdGgbRNAf+erl6yUm6pHpQhyFFo/zndx6Uq +QXA7jYvM2M3qCnXmaFowidoLDsDyhwoxD7WT8zur/QIDAQABo1cwVTAOBgNVHQ8B +Af8EBAMCAgQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAd +BgNVHQ4EFgQU7p4VuECNOcnrP9ulOjc4J37Q2VUwDQYJKoZIhvcNAQELBQADggEB +AAv26Vnk+ptQrppouF7yHV8fZbfnehpm07HIZkmnXO2vAP+MZJDNrHjy8JAVzXjt ++OlzqAL0cRQLsUptB0btoJuw23eq8RXgJo05OLOPQ2iGNbAATQh2kLwBWd/CMg+V +KJ4EIEpF4dmwOohsNR6xa/JoArIYH0D7gh2CwjrdGZr/tq1eMSL+uZcuX5OiE44A +2oXF9/jsqerOcH7QUMejSnB8N7X0LmUvH4jAesQgr7jo1JTOBs7GF6wb+U76NzFa +8ms2iAWhoplQ+EHR52wffWb0k6trXspq4O6v/J+nq9Ky3vC36so+G1ZFkMhCdTVJ +ZmrBsSMWeT2l07qeei2UFRU= +-----END CERTIFICATE-----` + var rsaKeyPEM = testingKey(`-----BEGIN RSA TESTING KEY----- MIIBOwIBAAJBANLJhPHhITqQbPklG3ibCVxwGMRfp/v4XqhfdQHdcVfHap6NQ5Wo k/4xIA+ui35/MmNartNuC+BdZ1tMuVCPFZcCAwEAAQJAEJ2N+zsR0Xn8/Q6twa4G @@ -1792,3 +2052,10 @@ D2lWusoe2/nEqfDVVWGWlyJ7yOmqaVm/iNUN9B2N2g== `) func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") } + +func combineCerts(cert1 []byte, cert2 []byte) []byte { + result := cert1 + result = append(result, []byte("\n")...) + result = append(result, cert2...) + return result +} diff --git a/pkg/reconciler/ingress/controller.go b/pkg/reconciler/ingress/controller.go index 35350bc96..b58be4d6b 100644 --- a/pkg/reconciler/ingress/controller.go +++ b/pkg/reconciler/ingress/controller.go @@ -28,6 +28,8 @@ import ( "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -37,6 +39,7 @@ import ( envoy "knative.dev/net-kourier/pkg/envoy/server" "knative.dev/net-kourier/pkg/generator" store "knative.dev/net-kourier/pkg/reconciler/ingress/config" + "knative.dev/networking/pkg/apis/networking" "knative.dev/networking/pkg/apis/networking/v1alpha1" networkingClientSet "knative.dev/networking/pkg/client/clientset/versioned/typed/networking/v1alpha1" knativeclient "knative.dev/networking/pkg/client/injection/client" @@ -52,6 +55,7 @@ import ( filteredFactory "knative.dev/pkg/client/injection/kube/informers/factory/filtered" "knative.dev/pkg/configmap" "knative.dev/pkg/controller" + nsconfigmapinformer "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/configmap" "knative.dev/pkg/logging" "knative.dev/pkg/reconciler" "knative.dev/pkg/system" @@ -81,6 +85,7 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl serviceInformer := serviceinformer.Get(ctx) podInformer := podinformer.Get(ctx) secretInformer := getSecretInformer(ctx) + nsConfigmapInformer := nsconfigmapinformer.Get(ctx) // this is filtered to SYSTEM_NAMESPACE // Create a new Cache, with the Readiness endpoint enabled, and the list of current Ingresses. caches, err := generator.NewCaches(kubernetesClient, config.ExternalAuthz.Enabled) @@ -183,6 +188,13 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl func(ns, name string) (*corev1.Secret, error) { return secretInformer.Lister().Secrets(ns).Get(name) }, + func(label string) ([]*corev1.ConfigMap, error) { + selector, err := getLabelSelector(label) + if err != nil { + return nil, err + } + return nsConfigmapInformer.Lister().List(selector) + }, func(ns, name string) (*corev1.Endpoints, error) { return endpointsInformer.Lister().Endpoints(ns).Get(name) }, @@ -219,6 +231,13 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl func(ns, name string) (*corev1.Secret, error) { return kubernetesClient.CoreV1().Secrets(ns).Get(ctx, name, metav1.GetOptions{}) }, + func(label string) ([]*corev1.ConfigMap, error) { + selector, err := getLabelSelector(label) + if err != nil { + return nil, err + } + return nsConfigmapInformer.Lister().List(selector) + }, func(ns, name string) (*corev1.Endpoints, error) { return kubernetesClient.CoreV1().Endpoints(ns).Get(ctx, name, metav1.GetOptions{}) }, @@ -310,6 +329,16 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl }, }) + nsConfigmapInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: reconciler.ChainFilterFuncs( + reconciler.LabelExistsFilterFunc(networking.TrustBundleLabelKey), + ), + Handler: controller.HandleAll(func(obj interface{}) { + logger.Info("Doing a global resync due to CA bundle Configmap changes") + impl.FilteredGlobalResync(isKourierIngress, ingressInformer.Informer()) + }), + }) + return impl } @@ -355,6 +384,16 @@ func getSecretInformer(ctx context.Context) v1.SecretInformer { return secretfilteredinformer.Get(ctx, untyped.([]string)[0]) } +func getLabelSelector(label string) (labels.Selector, error) { + selector := labels.NewSelector() + req, err := labels.NewRequirement(label, selection.Exists, nil) + if err != nil { + return nil, err + } + selector = selector.Add(*req) + return selector, nil +} + func ensureCtxWithConfigOrDie(ctx context.Context) context.Context { var err error var cfg *store.Config diff --git a/vendor/knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/configmap/configmap.go b/vendor/knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/configmap/configmap.go new file mode 100644 index 000000000..fd724cad3 --- /dev/null +++ b/vendor/knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/configmap/configmap.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configmap + +import ( + "context" + + v1 "k8s.io/client-go/informers/core/v1" + "knative.dev/pkg/controller" + "knative.dev/pkg/injection" + "knative.dev/pkg/injection/clients/namespacedkube/informers/factory" + "knative.dev/pkg/logging" +) + +func init() { + injection.Default.RegisterInformer(withInformer) +} + +// Key is used for associating the Informer inside the context.Context. +type Key struct{} + +func withInformer(ctx context.Context) (context.Context, controller.Informer) { + f := factory.Get(ctx) + inf := f.Core().V1().ConfigMaps() + return context.WithValue(ctx, Key{}, inf), inf.Informer() +} + +// Get extracts the typed informer from the context. +func Get(ctx context.Context) v1.ConfigMapInformer { + untyped := ctx.Value(Key{}) + if untyped == nil { + logging.FromContext(ctx).Panic( + "Unable to fetch k8s.io/client-go/informers/core/v1.ConfigMapInformer from context.") + } + return untyped.(v1.ConfigMapInformer) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index a4b049752..ab8c50fcc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1080,6 +1080,7 @@ knative.dev/pkg/environment knative.dev/pkg/hack knative.dev/pkg/hash knative.dev/pkg/injection +knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/configmap knative.dev/pkg/injection/clients/namespacedkube/informers/factory knative.dev/pkg/injection/sharedmain knative.dev/pkg/kflag