Merge branch 'kubernetes:master' into master

clusterloader2/pkg/measurement/common/generic_query_measurement.go

@@ -83,6 +83,7 @@ type GenericQuery struct {
 	Name           string
 	Query          string
 	Threshold      *float64
+	LowerBound     bool
 	RequireSamples bool
 }
 
@@ -132,6 +133,26 @@ func getOrCreate(dataItems map[string]*measurementutil.DataItem, key, unit strin
 	return dataItem
 }
 
+func (g *genericQueryGatherer) validateSample(q GenericQuery, val float64) error {
+	thresholdMsg := "none"
+	if q.Threshold != nil {
+		thresholdMsg = fmt.Sprintf("%v (upper bound)", *q.Threshold)
+		if q.LowerBound {
+			thresholdMsg = fmt.Sprintf("%v (lower bound)", *q.Threshold)
+		}
+	}
+	klog.V(2).Infof("metric: %v: %v, value: %v, threshold: %v", g.MetricName, q.Name, val, thresholdMsg)
+	if q.Threshold != nil {
+		if q.LowerBound && val < *q.Threshold {
+			return errors.NewMetricViolationError(q.Name, fmt.Sprintf("sample below threshold: want: greater or equal than %v, got: %v", *q.Threshold, val))
+		}
+		if !q.LowerBound && val > *q.Threshold {
+			return errors.NewMetricViolationError(q.Name, fmt.Sprintf("sample above threshold: want: less or equal than %v, got: %v", *q.Threshold, val))
+		}
+	}
+	return nil
+}
+
 func (g *genericQueryGatherer) Gather(executor QueryExecutor, startTime, endTime time.Time, config *measurement.Config) ([]measurement.Summary, error) {
 	var errs []error
 	dataItems := map[string]*measurementutil.DataItem{}
@@ -161,13 +182,8 @@ func (g *genericQueryGatherer) Gather(executor QueryExecutor, startTime, endTime
 				dataItem.Data[q.Name] = val
 			}
 
-			thresholdMsg := "none"
-			if q.Threshold != nil {
-				thresholdMsg = fmt.Sprintf("%v", *q.Threshold)
-			}
-			klog.V(2).Infof("metric: %v: %v, value: %v, threshold: %v", g.MetricName, q.Name, val, thresholdMsg)
-			if q.Threshold != nil && val > *q.Threshold {
-				errs = append(errs, errors.NewMetricViolationError(q.Name, fmt.Sprintf("sample above threshold: want: less or equal than %v, got: %v", *q.Threshold, val)))
+			if err := g.validateSample(q, val); err != nil {
+				errs = append(errs, err)
 			}
 		}
 	}

clusterloader2/pkg/measurement/common/generic_query_measurement_test.go

@@ -73,9 +73,9 @@ func TestGather(t *testing.T) {
 				},
 			},
 			samples: map[string][]*model.Sample{
-				"below-threshold-query[1m]":             {{Value: model.SampleValue(7)}},
-				"no-threshold-query[1m]":                {{Value: model.SampleValue(120)}},
-				"placeholder-a[1m] + placeholder-b[1m]": {{Value: model.SampleValue(5)}},
+				"below-threshold-query[60s]":              {{Value: model.SampleValue(7)}},
+				"no-threshold-query[60s]":                 {{Value: model.SampleValue(120)}},
+				"placeholder-a[60s] + placeholder-b[60s]": {{Value: model.SampleValue(5)}},
 			},
 			wantDataItems: []measurementutil.DataItem{
 				{
@@ -132,7 +132,7 @@ func TestGather(t *testing.T) {
 				},
 			},
 			samples: map[string][]*model.Sample{
-				"many-samples-query[1m]": {
+				"many-samples-query[60s]": {
 					{Value: model.SampleValue(1)},
 					{Value: model.SampleValue(2)},
 				},
@@ -163,7 +163,7 @@ func TestGather(t *testing.T) {
 				},
 			},
 			samples: map[string][]*model.Sample{
-				"above-threshold-query[1m]": {{Value: model.SampleValue(123)}},
+				"above-threshold-query[60s]": {{Value: model.SampleValue(123)}},
 			},
 			wantErr: "sample above threshold: want: less or equal than 60, got: 123",
 			wantDataItems: []measurementutil.DataItem{
@@ -175,6 +175,61 @@ func TestGather(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc: "sample above lower bound",
+			params: map[string]interface{}{
+				"metricName":    "below-threshold",
+				"metricVersion": "v1",
+				"unit":          "ms",
+				"queries": []map[string]interface{}{
+					{
+						"name":       "above-threshold",
+						"query":      "above-threshold-query[%v]",
+						"threshold":  60,
+						"lowerBound": true,
+					},
+				},
+			},
+			samples: map[string][]*model.Sample{
+				"above-threshold-query[60s]": {{Value: model.SampleValue(74)}},
+			},
+			wantDataItems: []measurementutil.DataItem{
+				{
+					Unit: "ms",
+					Data: map[string]float64{
+						"above-threshold": 74.0,
+					},
+				},
+			},
+		},
+		{
+			desc: "sample below lower bound",
+			params: map[string]interface{}{
+				"metricName":    "below-threshold",
+				"metricVersion": "v1",
+				"unit":          "ms",
+				"queries": []map[string]interface{}{
+					{
+						"name":       "below-threshold",
+						"query":      "below-threshold-query[%v]",
+						"threshold":  60,
+						"lowerBound": true,
+					},
+				},
+			},
+			samples: map[string][]*model.Sample{
+				"below-threshold-query[60s]": {{Value: model.SampleValue(42)}},
+			},
+			wantErr: "sample below threshold: want: greater or equal than 60, got: 42",
+			wantDataItems: []measurementutil.DataItem{
+				{
+					Unit: "ms",
+					Data: map[string]float64{
+						"below-threshold": 42.0,
+					},
+				},
+			},
+		},
 		{
 			desc: "missing field metricName",
 			params: map[string]interface{}{
@@ -221,7 +276,7 @@ func TestGather(t *testing.T) {
 				},
 			},
 			samples: map[string][]*model.Sample{
-				"query-perc99[1m]": {
+				"query-perc99[60s]": {
 					{
 						Metric: model.Metric{
 							model.LabelName("d1"): model.LabelValue("d1-val1"),
@@ -238,7 +293,7 @@ func TestGather(t *testing.T) {
 						Value: model.SampleValue(2),
 					},
 				},
-				"query-perc90[1m]": {
+				"query-perc90[60s]": {
 					{
 						Metric: model.Metric{
 							model.LabelName("d1"): model.LabelValue("d1-val1"),
@@ -316,7 +371,7 @@ func TestGather(t *testing.T) {
 				},
 			},
 			samples: map[string][]*model.Sample{
-				"query-perc99[1m]": {
+				"query-perc99[60s]": {
 					{
 						Metric: model.Metric{
 							model.LabelName("d1"): model.LabelValue("d1-val1"),

clusterloader2/pkg/measurement/util/prometheus.go

@@ -174,8 +174,5 @@ func (qr *promResponseData) UnmarshalJSON(b []byte) error {
 
 // ToPrometheusTime returns prometheus string representation of given time.
 func ToPrometheusTime(t time.Duration) string {
-	if t < time.Minute {
-		return fmt.Sprintf("%ds", int64(t)/int64(time.Second))
-	}
-	return fmt.Sprintf("%dm", int64(t)/int64(time.Minute))
+	return fmt.Sprintf("%ds", int64(t)/int64(time.Second))
 }

clusterloader2/testing/request-benchmark/config.yaml

@@ -5,6 +5,7 @@
 {{$benchmarkReplicas := DefaultParam .CL2_BENCHMARK_PODS 1}}
 
 {{$inflight := DefaultParam .CL2_BENCHMARK_INFLIGHT 10}}
+{{$qps := DefaultParam .CL2_BENCHMARK_QPS -1}}
 {{$uri := DefaultParam .CL2_BENCHMARK_URI ""}} # URI example: /api/v1/namespaces/%namespace%/pods
 
 name: Request benchmark
@@ -13,7 +14,7 @@ namespace:
 tuningSets:
 - name: Sequence
   parallelismLimitedLoad:
-    parallelismLimit: 1
+    parallelismLimit: 10
 steps:
 - name: Setup permissions
   phases:
@@ -43,7 +44,7 @@ steps:
     objectBundle:
     - basename: {{$configMapGroup}}
       objectTemplatePath: configmap.yaml
-      params:
+      templateFillMap:
         bytes: {{$configMapBytes}}
         group: {{$configMapGroup}}
 - module:
@@ -56,6 +57,7 @@ steps:
       replicas: {{$benchmarkReplicas}}
       inflight: {{$inflight}}
       uri: {{$uri}}
+      qps: {{$qps}}
 - module:
     path: modules/measurements.yaml
     params:

clusterloader2/testing/request-benchmark/configmap.yaml

@@ -1,4 +1,4 @@
-{{$bytes := DefaultParam .bytes 0}}
+{{$bytes := .bytes}}
 {{$group := DefaultParam .group .Name}}
 
 apiVersion: v1
@@ -8,4 +8,4 @@ metadata:
   labels:
     group: {{$group}}
 data:
-  key: "{{range $i := Loop $bytes}}{{RandIntRange 0 9}}{{end}}"
+  key: "{{RandData $bytes}}"

clusterloader2/testing/request-benchmark/deployment.yaml

@@ -27,6 +27,7 @@ spec:
         - --inflight={{.Inflight}}
         - --namespace={{.Namespace}}
         - --uri={{.Uri}}
+        - --qps={{.QPS}}
         resources:
           requests:
             cpu: {{$cpu}}

clusterloader2/testing/request-benchmark/modules/benchmark-deployment.yaml

@@ -1,6 +1,7 @@
 {{$replicas := DefaultParam .replicas 0}}
 {{$inflight := DefaultParam .inflight 0}}
 {{$uri := DefaultParam .uri "/"}}
+{{$qps := DefaultParam .qps -1}}
 
 steps:
 - name: Creating WaitForControlledPodsRunning measurement
@@ -28,6 +29,7 @@ steps:
         Replicas: {{$replicas}}
         Inflight: {{$inflight}}
         Uri: {{$uri}}
+        QPS: {{$qps}}
 - name: Waiting for WaitForControlledPodsRunning gather
   measurements:
   - Identifier: WaitForBenchmarkDeployment

clusterloader2/testing/request-benchmark/modules/measurements.yaml

@@ -9,6 +9,12 @@ steps:
         duration: 1m
 - name: "Starting measurement - {{.name}}"
   measurements:
+  - Identifier: APIResponsivenessPrometheusSimple
+    Method: APIResponsivenessPrometheus
+    Params:
+      action: start
+      enableViolations: false
+      useSimpleLatencyQuery: true
   - Identifier: ContainerCPU-{{.name}}
     Method: GenericPrometheusQuery
     Params:
@@ -49,3 +55,9 @@ steps:
         query: quantile_over_time(0.90, sum by (container) (rate(container_cpu_usage_seconds_total[1m]))[%v:])
       - name: Perc50
         query: quantile_over_time(0.50, sum by (container) (rate(container_cpu_usage_seconds_total[1m]))[%v:])
+  - Identifier: APIResponsivenessPrometheusSimple
+    Method: APIResponsivenessPrometheus
+    Params:
+      action: gather
+      enableViolations: false
+      useSimpleLatencyQuery: true

clusterloader2/testing/watch-list/config.yaml

@@ -42,7 +42,8 @@ steps:
         min: 1
         max: 1
         basename: watch-list
-      replicasPerNamespace: 2
+      # TODO(p0lyn0mial): bring back 2 replicas
+      replicasPerNamespace: 1
       tuningSet: Uniform10qps
       objectBundle:
         - basename: watch-list-secret

clusterloader2/testing/watch-list/job.yaml

@@ -21,5 +21,6 @@ spec:
               memory: "16Gi"
               cpu: "6"
           command: [ "watch-list" ]
-          args: [ "--alsologtostderr=true", "--v=4", "--timeout={{.Duration}}", "--count=16", "--namespace=watch-list-1", "--enableWatchListFeature={{.EnableWatchListFeature}}"]
+          # TODO(p0lyn0mial): bring back 16 informers
+          args: [ "--alsologtostderr=true", "--v=4", "--timeout={{.Duration}}", "--count=6", "--namespace=watch-list-1", "--enableWatchListFeature={{.EnableWatchListFeature}}"]
       restartPolicy: Never

golang/Makefile

@@ -15,7 +15,7 @@
 .PHONY: build-go build push
 
 export GCS_BUCKET?=k8s-infra-scale-golang-builds
-export GO_COMPILER_PKG?=go1.18.5.linux-amd64.tar.gz
+export GO_COMPILER_PKG?=go1.20.6.linux-amd64.tar.gz
 export GO_COMPILER_URL?=https://dl.google.com/go/$(GO_COMPILER_PKG)
 export ROOT_DIR?=/home/prow/go/src
 

util-images/request-benchmark/Makefile

@@ -1,6 +1,6 @@
 PROJECT ?= k8s-testimages
 IMG = gcr.io/$(PROJECT)/perf-tests-util/request-benchmark
-TAG = v0.0.5
+TAG = v0.0.6
 
 all: build push
 

util-images/request-benchmark/client.go

@@ -48,13 +48,14 @@ func (c *Client) TimedRequest(ctx context.Context, request *rest.Request) Respon
 	}
 }
 
-func GetClient() (*Client, error) {
+func GetClient(qps float32) (*Client, error) {
 	config, err := getConfig()
 	if err != nil {
 		return nil, err
 	}
 
-	config.QPS = -1 // Disable rate limiter
+	config.QPS = qps
+	config.Burst = 10
 
 	client, err := kubernetes.NewForConfig(config)
 	return (*Client)(client), err

util-images/request-benchmark/main.go

@@ -34,14 +34,15 @@ var (
 	namespace = flag.String("namespace", "", "Replace %namespace% in URI with provided namespace")
 	URI       = flag.String("uri", "", "Request URI")
 	verb      = flag.String("verb", "GET", "A verb to be used in requests.")
+	qps       = flag.Float64("qps", -1, "The qps limit for all requests")
 )
 
 func init() {
 	flag.Parse()
 }
 
 func main() {
-	client, err := GetClient()
+	client, err := GetClient(float32(*qps))
 	if err != nil {
 		panic(err)
 	}