Merge branch 'main' into SuyashHashiCorp-patch-2

.changelog/21871.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+proxycfg: fix a bug where peered upstreams watches are canceled even when another target needs it.
+```

.changelog/21908.txt

@@ -0,0 +1,3 @@
+```release-note:security
+Resolved issue where hcl would allow duplicates of the same key in acl policy configuration.
+```

.changelog/21909.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+state: ensure that identical manual virtual IP updates result in not bumping the modify indexes
+```

.changelog/21930.txt

@@ -0,0 +1,3 @@
+```release-note:security
+api: Enforces strict content-type header validation to protect against XSS vulnerability.
+```

.changelog/21950.txt

@@ -0,0 +1,3 @@
+```release-note:security
+Removed ability to use bexpr to filter results without ACL read on endpoint
+```

.changelog/21951.txt

@@ -0,0 +1,3 @@
+```release-note:security
+Update `github.com/golang-jwt/jwt/v4` to v4.5.1 to address [GHSA-29wx-vh33-7x7r](https://github.com/golang-jwt/jwt/security/advisories/GHSA-29wx-vh33-7x7r).
+```

.changelog/22001.txt

@@ -0,0 +1,3 @@
+```release-note:security
+Update `golang.org/x/crypto` to v0.31.0 to address [GO-2024-3321](https://pkg.go.dev/vuln/GO-2024-3321).
+```

.changelog/22011.txt

@@ -0,0 +1,4 @@
+```release-note:security
+Update `registry.access.redhat.com/ubi9-minimal` image to 9.5 to address [CVE-2024-3596](https://nvd.nist.gov/vuln/detail/CVE-2024-3596),[CVE-2024-2511](https://nvd.nist.gov/vuln/detail/CVE-2024-2511),[CVE-2024-26458](https://nvd.nist.gov/vuln/detail/CVE-2024-26458).
+```
+

.changelog/22021.txt

@@ -0,0 +1,3 @@
+```release-note:security
+Update `golang.org/x/net` to v0.33.0 to address [GO-2024-3333](https://pkg.go.dev/vuln/GO-2024-3333).
+```

.github/CODEOWNERS

@@ -1,3 +1,5 @@
+* @hashicorp/consul-selfmanage-maintainers
+
 # Techical Writer Review
 
 /website/content/docs/ @hashicorp/consul-docs
@@ -6,36 +8,5 @@
 
 
 # release configuration
-/.release/                              @hashicorp/release-engineering @hashicorp/github-consul-core
-/.github/workflows/build.yml            @hashicorp/release-engineering @hashicorp/github-consul-core
-
-
-# Staff Engineer Review (protocol buffer definitions)
-/proto-public/   @hashicorp/consul-core-staff
-/proto/          @hashicorp/consul-core-staff
-
-# Staff Engineer Review (v1 architecture shared components)
-/agent/cache/                  @hashicorp/consul-core-staff
-/agent/consul/fsm/             @hashicorp/consul-core-staff
-/agent/consul/leader*.go       @hashicorp/consul-core-staff
-/agent/consul/server*.go       @hashicorp/consul-core-staff
-/agent/consul/state/           @hashicorp/consul-core-staff
-/agent/consul/stream/          @hashicorp/consul-core-staff
-/agent/submatview/             @hashicorp/consul-core-staff
-/agent/blockingquery/          @hashicorp/consul-core-staff
-
-# Staff Engineer Review (raft/autopilot)
-/agent/consul/autopilotevents/  @hashicorp/consul-core-staff
-/agent/consul/autopilot*.go     @hashicorp/consul-core-staff
-
-# Staff Engineer Review (v2 architecture shared components)
-/internal/controller/                   @hashicorp/consul-core-staff
-/internal/resource/                     @hashicorp/consul-core-staff
-/internal/storage/                      @hashicorp/consul-core-staff
-/agent/consul/controller/               @hashicorp/consul-core-staff
-/agent/grpc-external/services/resource/ @hashicorp/consul-core-staff
-
-# Staff Engineer Review (v1 security)
-/acl/                   @hashicorp/consul-core-staff
-/agent/xds/rbac*.go     @hashicorp/consul-core-staff
-/agent/xds/jwt*.go      @hashicorp/consul-core-staff
+/.release/                              @hashicorp/team-selfmanaged-releng @hashicorp/consul-selfmanage-maintainers
+/.github/workflows/build.yml            @hashicorp/team-selfmanaged-releng @hashicorp/consul-selfmanage-maintainers

.release/security-scan.hcl

@@ -37,8 +37,8 @@ container {
 	triage {
 		suppress {
 			vulnerabilities = [
-				"CVE-2024-8096", # [email protected],
-				"CVE-2024-9143", # [email protected],
+				"CVE-2024-4067", # libsolv@0:0.7.24-3.el9
+				"CVE-2019-12900" # bzip2-libs@0:1.0.8-8.el9
 			]
 			paths = [
 				"internal/tools/proto-gen-rpc-glue/e2e/consul/*",
@@ -79,6 +79,7 @@ binary {
 	triage {
 		suppress {
 			vulnerabilities = [
+				"GO-2022-0635", // github.com/aws/[email protected]
 			]
 			paths = [
 				"internal/tools/proto-gen-rpc-glue/e2e/consul/*",

Dockerfile

@@ -16,7 +16,7 @@
 # Official docker image that includes binaries from releases.hashicorp.com. This
 # downloads the release from releases.hashicorp.com and therefore requires that
 # the release is published before building the Docker image.
-FROM docker.mirror.hashicorp.services/alpine:3.20 as official
+FROM docker.mirror.hashicorp.services/alpine:3.21 as official
 
 # This is the release of Consul to pull in.
 ARG VERSION
@@ -29,6 +29,13 @@ LABEL org.opencontainers.image.authors="Consul Team <[email protected]>" \
       org.opencontainers.image.vendor="HashiCorp" \
       org.opencontainers.image.title="consul" \
       org.opencontainers.image.description="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \
+      name="Consul" \
+      maintainer="Consul Team <[email protected]>" \
+      vendor="HashiCorp" \
+      release=${PRODUCT_REVISION} \
+      revision=${PRODUCT_REVISION} \
+      summary="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \
+      description="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \
       version=${VERSION}
 
 # This is the location of the releases.
@@ -112,7 +119,7 @@ CMD ["agent", "-dev", "-client", "0.0.0.0"]
 
 # Production docker image that uses CI built binaries.
 # Remember, this image cannot be built locally.
-FROM docker.mirror.hashicorp.services/alpine:3.20 as default
+FROM docker.mirror.hashicorp.services/alpine:3.21 as default
 
 ARG PRODUCT_VERSION
 ARG BIN_NAME
@@ -137,6 +144,13 @@ LABEL org.opencontainers.image.authors="Consul Team <[email protected]>" \
       org.opencontainers.image.title="consul" \
       org.opencontainers.image.description="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \
       org.opencontainers.image.licenses="BSL-1.1" \
+      name="Consul" \
+      maintainer="Consul Team <[email protected]>" \
+      vendor="HashiCorp" \
+      release=${PRODUCT_REVISION} \
+      revision=${PRODUCT_REVISION} \
+      summary="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \
+      description="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \
       version=${PRODUCT_VERSION}
 
 COPY LICENSE /usr/share/doc/$PRODUCT_NAME/LICENSE.txt
@@ -203,7 +217,7 @@ CMD ["agent", "-dev", "-client", "0.0.0.0"]
 
 # Red Hat UBI-based image
 # This target is used to build a Consul image for use on OpenShift.
-FROM registry.access.redhat.com/ubi9-minimal:9.4 as ubi
+FROM registry.access.redhat.com/ubi9-minimal:9.5 as ubi
 
 ARG PRODUCT_VERSION
 ARG PRODUCT_REVISION
@@ -227,6 +241,13 @@ LABEL org.opencontainers.image.authors="Consul Team <[email protected]>" \
       org.opencontainers.image.title="consul" \
       org.opencontainers.image.description="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \
       org.opencontainers.image.licenses="BSL-1.1" \
+      name="Consul" \
+      maintainer="Consul Team <[email protected]>" \
+      vendor="HashiCorp" \
+      release=${PRODUCT_REVISION} \
+      revision=${PRODUCT_REVISION} \
+      summary="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \
+      description="Consul is a datacenter runtime that provides service discovery, configuration, and orchestration." \
       version=${PRODUCT_VERSION}
 
 COPY LICENSE /usr/share/doc/$PRODUCT_NAME/LICENSE.txt

acl/acl.go

@@ -22,6 +22,9 @@ type Config struct {
 	// WildcardName is the string that represents a request to authorize a wildcard permission
 	WildcardName string
 
+	//by default errors, but in certain instances we want to make sure to maintain backwards compatabilty
+	WarnOnDuplicateKey bool
+
 	// embedded enterprise configuration
 	EnterpriseConfig
 }

acl/policy.go

@@ -310,8 +310,8 @@ func (pr *PolicyRules) Validate(conf *Config) error {
 	return nil
 }
 
-func parse(rules string, conf *Config, meta *EnterprisePolicyMeta) (*Policy, error) {
-	p, err := decodeRules(rules, conf, meta)
+func parse(rules string, warnOnDuplicateKey bool, conf *Config, meta *EnterprisePolicyMeta) (*Policy, error) {
+	p, err := decodeRules(rules, warnOnDuplicateKey, conf, meta)
 	if err != nil {
 		return nil, err
 	}
@@ -338,7 +338,11 @@ func NewPolicyFromSource(rules string, conf *Config, meta *EnterprisePolicyMeta)
 
 	var policy *Policy
 	var err error
-	policy, err = parse(rules, conf, meta)
+	warnOnDuplicateKey := false
+	if conf != nil {
+		warnOnDuplicateKey = conf.WarnOnDuplicateKey
+	}
+	policy, err = parse(rules, warnOnDuplicateKey, conf, meta)
 	return policy, err
 }
 

acl/policy_ce.go

@@ -7,8 +7,9 @@ package acl
 
 import (
 	"fmt"
-
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/hcl"
+	"strings"
 )
 
 // EnterprisePolicyMeta stub
@@ -30,12 +31,28 @@ func (r *EnterprisePolicyRules) Validate(*Config) error {
 	return nil
 }
 
-func decodeRules(rules string, _ *Config, _ *EnterprisePolicyMeta) (*Policy, error) {
+func decodeRules(rules string, warnOnDuplicateKey bool, _ *Config, _ *EnterprisePolicyMeta) (*Policy, error) {
 	p := &Policy{}
 
-	if err := hcl.Decode(p, rules); err != nil {
+	err := hcl.DecodeErrorOnDuplicates(p, rules)
+
+	if errIsDuplicateKey(err) && warnOnDuplicateKey {
+		//because the snapshot saves the unparsed rules we have to assume some snapshots exist that shouldn't fail, but
+		// have duplicates
+		if err := hcl.Decode(p, rules); err != nil {
+			hclog.Default().Warn("Warning- Duplicate key in ACL Policy ignored", "errorMessage", err.Error())
+			return nil, fmt.Errorf("Failed to parse ACL rules: %v", err)
+		}
+	} else if err != nil {
 		return nil, fmt.Errorf("Failed to parse ACL rules: %v", err)
 	}
 
 	return p, nil
 }
+
+func errIsDuplicateKey(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), "was already set. Each argument can only be defined once")
+}

acl/policy_test.go

@@ -342,6 +342,12 @@ func TestPolicySourceParse(t *testing.T) {
 			RulesJSON: `{ "acl": "list" }`, // there is no list policy but this helps to exercise another check in isPolicyValid
 			Err:       "Invalid acl policy",
 		},
+		{
+			Name: "Bad Policy - Duplicate ACL Key",
+			Rules: `acl="read"
+					acl="write"`,
+			Err: "Failed to parse ACL rules: The argument \"acl\" at",
+		},
 		{
 			Name:      "Bad Policy - Agent",
 			Rules:     `agent "foo" { policy = "nope" }`,

agent/agent_endpoint.go

@@ -380,16 +380,14 @@ func (s *HTTPHandlers) AgentServices(resp http.ResponseWriter, req *http.Request
 		return nil, err
 	}
 
-	raw, err := filter.Execute(agentSvcs)
-	if err != nil {
-		return nil, err
-	}
-	agentSvcs = raw.(map[string]*api.AgentService)
-
-	// Note: we filter the results with ACLs *after* applying the user-supplied
-	// bexpr filter, to ensure total (and the filter-by-acls header we set below)
-	// do not include results that would be filtered out even if the user did have
-	// permission.
+	// Note: we filter the results with ACLs *before* applying the user-supplied
+	// bexpr filter to ensure that the user can only run expressions on data that
+	// they have access to.  This is a security measure to prevent users from
+	// running arbitrary expressions on data they don't have access to.
+	// QueryMeta.ResultsFilteredByACLs being true already indicates to the user
+	// that results they don't have access to have been removed.  If they were
+	// also allowed to run the bexpr filter on the data, they could potentially
+	// infer the specific attributes of data they don't have access to.
 	total := len(agentSvcs)
 	if err := s.agent.filterServicesWithAuthorizer(authz, agentSvcs); err != nil {
 		return nil, err
@@ -407,6 +405,12 @@ func (s *HTTPHandlers) AgentServices(resp http.ResponseWriter, req *http.Request
 		setResultsFilteredByACLs(resp, total != len(agentSvcs))
 	}
 
+	raw, err := filter.Execute(agentSvcs)
+	if err != nil {
+		return nil, err
+	}
+	agentSvcs = raw.(map[string]*api.AgentService)
+
 	return agentSvcs, nil
 }
 
@@ -540,16 +544,14 @@ func (s *HTTPHandlers) AgentChecks(resp http.ResponseWriter, req *http.Request)
 		}
 	}
 
-	raw, err := filter.Execute(agentChecks)
-	if err != nil {
-		return nil, err
-	}
-	agentChecks = raw.(map[types.CheckID]*structs.HealthCheck)
-
-	// Note: we filter the results with ACLs *after* applying the user-supplied
-	// bexpr filter, to ensure total (and the filter-by-acls header we set below)
-	// do not include results that would be filtered out even if the user did have
-	// permission.
+	// Note: we filter the results with ACLs *before* applying the user-supplied
+	// bexpr filter to ensure that the user can only run expressions on data that
+	// they have access to.  This is a security measure to prevent users from
+	// running arbitrary expressions on data they don't have access to.
+	// QueryMeta.ResultsFilteredByACLs being true already indicates to the user
+	// that results they don't have access to have been removed.  If they were
+	// also allowed to run the bexpr filter on the data, they could potentially
+	// infer the specific attributes of data they don't have access to.
 	total := len(agentChecks)
 	if err := s.agent.filterChecksWithAuthorizer(authz, agentChecks); err != nil {
 		return nil, err
@@ -567,6 +569,12 @@ func (s *HTTPHandlers) AgentChecks(resp http.ResponseWriter, req *http.Request)
 		setResultsFilteredByACLs(resp, total != len(agentChecks))
 	}
 
+	raw, err := filter.Execute(agentChecks)
+	if err != nil {
+		return nil, err
+	}
+	agentChecks = raw.(map[types.CheckID]*structs.HealthCheck)
+
 	return agentChecks, nil
 }
 
@@ -623,21 +631,14 @@ func (s *HTTPHandlers) AgentMembers(resp http.ResponseWriter, req *http.Request)
 		}
 	}
 
-	// filter the members by parsed filter expression
-	var filterExpression string
-	s.parseFilter(req, &filterExpression)
-	if filterExpression != "" {
-		filter, err := bexpr.CreateFilter(filterExpression, nil, members)
-		if err != nil {
-			return nil, err
-		}
-		raw, err := filter.Execute(members)
-		if err != nil {
-			return nil, err
-		}
-		members = raw.([]serf.Member)
-	}
-
+	// Note: we filter the results with ACLs *before* applying the user-supplied
+	// bexpr filter to ensure that the user can only run expressions on data that
+	// they have access to.  This is a security measure to prevent users from
+	// running arbitrary expressions on data they don't have access to.
+	// QueryMeta.ResultsFilteredByACLs being true already indicates to the user
+	// that results they don't have access to have been removed.  If they were
+	// also allowed to run the bexpr filter on the data, they could potentially
+	// infer the specific attributes of data they don't have access to.
 	total := len(members)
 	if err := s.agent.filterMembers(token, &members); err != nil {
 		return nil, err
@@ -655,6 +656,21 @@ func (s *HTTPHandlers) AgentMembers(resp http.ResponseWriter, req *http.Request)
 		setResultsFilteredByACLs(resp, total != len(members))
 	}
 
+	// filter the members by parsed filter expression
+	var filterExpression string
+	s.parseFilter(req, &filterExpression)
+	if filterExpression != "" {
+		filter, err := bexpr.CreateFilter(filterExpression, nil, members)
+		if err != nil {
+			return nil, err
+		}
+		raw, err := filter.Execute(members)
+		if err != nil {
+			return nil, err
+		}
+		members = raw.([]serf.Member)
+	}
+
 	return members, nil
 }