add support for gateway-api (#72)

* add support for gateway-api

* Simplify endpoint creation

* remove redundant applicationNamespace var

* get ingress domain from go commons lib

* go mod rm 1.23

* Fix linting

Signed-off-by: Raul Sevilla <rsevilla@redhat.com>

---------

Signed-off-by: Raul Sevilla <rsevilla@redhat.com>
Co-authored-by: Raul Sevilla <rsevilla@redhat.com>

README.md

@@ -76,9 +76,15 @@ Ingress-perf is compatible with the OpenShift implementation of the Istio ingres
 
 At the time of writing these lines only the `http` and `edge` terminations are supported.
 
+## Gateway API
+
+Ingress-perf is compatible with the OpenShift implementation of the Gateway API, provided by OpenShift Service Mesh. To enable this, it is necessary to pass the flag `--gateway-api=true`. When this flag specified, `ingress-perf` will create its httproute in the `ingress-perf` namespace.
+
+At the time of writing these lines only the `http` and `edge` terminations are supported.
+
 ## Compile
 
-Go 1.19 is required
+Go 1.22 is required
 
 ```console
 $ make build

cmd/ingress-perf.go

@@ -44,7 +44,7 @@ var versionCmd = &cobra.Command{
 
 func run() *cobra.Command {
 	var cfg, uuid, esServer, esIndex, logLevel, outputDir, igNamespace string
-	var cleanup, podMetrics, serviceMesh bool
+	var cleanup, podMetrics, serviceMesh, gatewayAPI bool
 	cmd := &cobra.Command{
 		Use:           "run",
 		Short:         "Run benchmark",
@@ -67,6 +67,7 @@ func run() *cobra.Command {
 				uuid, cleanup,
 				runner.WithIndexer(esServer, esIndex, outputDir, podMetrics),
 				runner.WithServiceMesh(serviceMesh, igNamespace),
+				runner.WithGatewayAPI(gatewayAPI),
 			)
 			return r.Start()
 		},
@@ -81,6 +82,7 @@ func run() *cobra.Command {
 	cmd.Flags().StringVar(&logLevel, "loglevel", "info", "Log level. Allowed levels are error, info and debug")
 	cmd.Flags().StringVar(&igNamespace, "gw-ns", "istio-system", "Ingress gateway namespace")
 	cmd.Flags().BoolVar(&serviceMesh, "service-mesh", false, "Enable service mesh mode")
+	cmd.Flags().BoolVar(&gatewayAPI, "gateway-api", false, "Enable gateway api mode")
 	cmd.MarkFlagRequired("cfg")
 	return cmd
 }

go.mod

@@ -1,6 +1,8 @@
 module github.com/cloud-bulldozer/ingress-perf
 
-go 1.22
+go 1.22.0
+
+toolchain go1.22.7
 
 require (
 	github.com/cloud-bulldozer/go-commons v1.0.10
@@ -9,65 +11,68 @@ require (
 	github.com/prometheus/common v0.45.0
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.1
 	gopkg.in/yaml.v3 v3.0.1
 	istio.io/api v1.20.1
 	istio.io/client-go v1.20.1
-	k8s.io/api v0.28.3
-	k8s.io/apimachinery v0.28.3
-	k8s.io/client-go v0.28.3
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
+	k8s.io/api v0.31.1
+	k8s.io/apimachinery v0.31.1
+	k8s.io/client-go v0.31.1
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+	sigs.k8s.io/gateway-api v1.2.0
 )
 
 require (
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/prometheus/client_golang v1.17.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
 	google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13 // indirect
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
-	github.com/go-openapi/jsonpointer v0.20.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.3.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/spdystream v0.4.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opensearch-project/opensearch-go v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/oauth2 v0.13.0 // indirect
-	golang.org/x/sync v0.4.0
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/sync v0.8.0
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/term v0.23.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/klog/v2 v2.100.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240423202451-8948a665c108 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )

pkg/runner/exec.go

@@ -38,18 +38,42 @@
 
 var lock = &sync.Mutex{}
 
-func runBenchmark(cfg config.Config, clusterMetadata tools.ClusterMetadata, p *prometheus.Prometheus, podMetrics bool) ([]tools.Result, error) {
+func runBenchmark(
+	cfg config.Config,
+	clusterMetadata tools.ClusterMetadata,
+	p *prometheus.Prometheus,
+	podMetrics bool,
+	isGatewayAPIEnabled bool,
+) ([]tools.Result, error) {
 	var aggAvgRps, aggAvgLatency, aggP95Latency float64
 	var timeouts, httpErrors int64
 	var benchmarkResult []tools.Result
 	var clientPods []corev1.Pod
 	var ep string
-	r, err := orClientSet.RouteV1().Routes(routesNamespace).Get(context.TODO(), fmt.Sprintf("%s-%s", serverName, cfg.Termination), metav1.GetOptions{})
-	if err != nil {
-		return benchmarkResult, err
+	var host string
+	if isGatewayAPIEnabled {
+		hr, err := hrClientSet.GatewayV1beta1().HTTPRoutes(routesNamespace).
+			Get(context.TODO(), serverName, metav1.GetOptions{})
+		if err != nil {
+			return benchmarkResult, err
+		}
+		host = string(hr.Spec.Hostnames[0])
+	} else {
+		r, err := orClientSet.RouteV1().Routes(routesNamespace).
+			Get(context.TODO(), fmt.Sprintf("%s-%s", serverName, cfg.Termination), metav1.GetOptions{})
+		if err != nil {
+			return benchmarkResult, err
+		}
+		host = r.Spec.Host
 	}
+	protocol := "http"
+	if cfg.Termination != "http" {
+		protocol = "https"
+	}
+
+	ep = fmt.Sprintf("%s://%v%v", protocol, host, cfg.Path)
 	allClientPods, err := clientSet.CoreV1().Pods(benchmarkNs.Name).List(context.TODO(), metav1.ListOptions{
 		LabelSelector: fmt.Sprintf("app=%s", clientName),
 	})
 	if err != nil {
 		return benchmarkResult, err
@@ -81,11 +105,6 @@
 			for i := 0; i < cfg.Procs; i++ {
 				func(p corev1.Pod) {
 					errGroup.Go(func() error {
-						if cfg.Termination == "http" {
-							ep = fmt.Sprintf("http://%v%v", r.Spec.Host, cfg.Path)
-						} else {
-							ep = fmt.Sprintf("https://%v%v", r.Spec.Host, cfg.Path)
-						}
 						tool, err := tools.New(cfg, ep)
 						if err != nil {
 							return err

pkg/runner/runner.go

@@ -43,13 +43,16 @@
 	openshiftrouteclientset "github.com/openshift/client-go/route/clientset/versioned"
 	istioclient "istio.io/client-go/pkg/clientset/versioned"
 	"k8s.io/client-go/tools/clientcmd"
+	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+	gatewayApiClientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned"
 )
 
 var restConfig *rest.Config
 var clientSet *kubernetes.Clientset
 var istioClient *istioclient.Clientset
 var dynamicClient *dynamic.DynamicClient
 var orClientSet *openshiftrouteclientset.Clientset
+var hrClientSet *gatewayApiClientset.Clientset
 var currentTuning string
 
 func New(uuid string, cleanup bool, opts ...OptsFunctions) *Runner {
@@ -101,6 +104,12 @@
 	}
 }
 
+func WithGatewayAPI(enable bool) OptsFunctions {
+	return func(r *Runner) {
+		r.gatewayAPI = enable
+	}
+}
+
 func (r *Runner) Start() error {
 	var err error
 	var kubeconfig string
@@ -122,6 +131,7 @@
 	clientSet = kubernetes.NewForConfigOrDie(restConfig)
 	istioClient = istioclient.NewForConfigOrDie(restConfig)
 	orClientSet = openshiftrouteclientset.NewForConfigOrDie(restConfig)
+	hrClientSet = gatewayApiClientset.NewForConfigOrDie(restConfig)
 	dynamicClient = dynamic.NewForConfigOrDie(restConfig)
 	ocpMetadata, err := ocpmetadata.NewMetadata(restConfig)
 	if err != nil {
@@ -172,7 +182,7 @@
 				return err
 			}
 		}
-		if benchmarkResult, err = runBenchmark(cfg, clusterMetadata, p, r.podMetrics); err != nil {
+		if benchmarkResult, err = runBenchmark(cfg, clusterMetadata, p, r.podMetrics, r.gatewayAPI); err != nil {
 			return err
 		}
 		if r.indexer != nil && !cfg.Warmup {
@@ -201,7 +211,7 @@
 	if passed {
 		return nil
 	}
 	return fmt.Errorf("some benchmark comparisons failed")
 }
 
 func indexDocuments(indexer indexers.Indexer, documents []interface{}, indexingOpts indexers.IndexingOpts) error {
@@ -218,7 +228,7 @@
 	if err := clientSet.CoreV1().Namespaces().Delete(context.TODO(), benchmarkNs.Name, metav1.DeleteOptions{}); err != nil {
 		return err
 	}
 	err := wait.PollUntilContextTimeout(context.TODO(), time.Second, timeout, true, func(ctx context.Context) (bool, error) {
 		_, err := clientSet.CoreV1().Namespaces().Get(context.TODO(), benchmarkNs.Name, metav1.GetOptions{})
 		if err != nil {
 			if errors.IsNotFound(err) {
@@ -234,11 +244,14 @@
 	return clientSet.RbacV1().ClusterRoleBindings().Delete(context.Background(), clientCRB.Name, metav1.DeleteOptions{})
 }
 
+//nolint:gocyclo
 func (r *Runner) deployAssets() error {
 	log.Infof("Deploying benchmark assets")
 	if r.serviceMesh {
 		log.Info("Service mesh mode enabled")
 		benchmarkNs.Labels["istio-injection"] = "enabled"
+	} else if r.gatewayAPI {
+		log.Info("Gateway API mode enabled")
 	}
 	_, err := clientSet.CoreV1().Namespaces().Create(context.TODO(), &benchmarkNs, metav1.CreateOptions{})
 	if err != nil && !errors.IsAlreadyExists(err) {
@@ -260,19 +273,22 @@
 	if err != nil && !errors.IsAlreadyExists(err) {
 		return err
 	}
-	for _, route := range routes {
-		if r.serviceMesh {
-			route.Spec.To = v1.RouteTargetReference{
-				Name: "istio-ingressgateway",
+	if !r.gatewayAPI {
+		for _, route := range routes {
+			if r.serviceMesh {
+				route.Spec.To = v1.RouteTargetReference{
+					Name: "istio-ingressgateway",
+				}
+				route.Spec.Port.TargetPort = intstr.FromString("http2")
+				routesNamespace = r.igNamespace
+			}
+			_, err := orClientSet.RouteV1().Routes(routesNamespace).Create(context.TODO(), &route, metav1.CreateOptions{})
+			if err != nil && !errors.IsAlreadyExists(err) {
+				return err
 			}
-			route.Spec.Port.TargetPort = intstr.FromString("http2")
-			routesNamespace = r.igNamespace
-		}
-		_, err := orClientSet.RouteV1().Routes(routesNamespace).Create(context.TODO(), &route, metav1.CreateOptions{})
-		if err != nil && !errors.IsAlreadyExists(err) {
-			return err
 		}
 	}
+
 	if r.serviceMesh {
 		routes, _ := orClientSet.RouteV1().Routes(routesNamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=ingress-perf"})
 		for _, r := range routes.Items {
@@ -287,6 +303,33 @@
 			return err
 		}
 	}
+	if r.gatewayAPI {
+		ocpMetadata, _ := ocpmetadata.NewMetadata(restConfig)
+		ingressDomain, err = ocpMetadata.GetDefaultIngressDomain()
+		if err != nil {
+			return err
+		}
+		listenerHostName = gatewayv1beta1.Hostname("*.gwapi." + ingressDomain)
+		httproutes.Spec.Hostnames = append(httproutes.Spec.Hostnames, gatewayv1beta1.Hostname("nginx.gwapi."+ingressDomain))
+		log.Debugf("Creating GatewayClass...")
+		_, err = hrClientSet.GatewayV1beta1().GatewayClasses().Create(context.TODO(), gatewayClass, metav1.CreateOptions{})
+		if err != nil && !errors.IsAlreadyExists(err) {
+			return err
+		}
+		time.Sleep(5 * time.Second) // wait for ServiceMeshControlPlane to be ready
+		log.Debugf("Creating Gateway...")
+		_, err = hrClientSet.GatewayV1beta1().Gateways(string(gatewayNamespace)).Create(context.TODO(), &gateway, metav1.CreateOptions{})
+		if err != nil && !errors.IsAlreadyExists(err) {
+			return err
+		}
+		log.Debugf("Waiting 4 minutes for Gateway to be ready...")
+		time.Sleep(4 * time.Minute)
+		log.Debugf("Creating HTTPRoute...")
+		_, err := hrClientSet.GatewayV1beta1().HTTPRoutes(routesNamespace).Create(context.TODO(), &httproutes, metav1.CreateOptions{})
+		if err != nil && !errors.IsAlreadyExists(err) {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -317,7 +360,7 @@
 	var dep *appsv1.Deployment
 	var err error
 	log.Infof("Waiting for replicas from deployment %s in ns %s to be ready", deployment, ns)
 	err = wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (bool, error) {
 		dep, err = clientSet.AppsV1().Deployments(ns).Get(context.TODO(), deployment, metav1.GetOptions{})
 		if err != nil {
 			return false, err

pkg/runner/types.go

@@ -27,6 +27,7 @@
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
+	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 )
 
 const (
@@ -42,12 +43,20 @@
 	podMetrics  bool
 	cleanup     bool
 	serviceMesh bool
+	gatewayAPI  bool
 	igNamespace string
 }
 
 type OptsFunctions func(r *Runner)
 
 var routesNamespace = benchmarkNs.Name
+var gatewayClassName = "openshift-default"
+var gatewayNamespace gatewayv1beta1.Namespace = "openshift-ingress"
+var portNumber gatewayv1beta1.PortNumber = 8080
+var tlsType gatewayv1beta1.TLSModeType = "Terminate"
+var fromNamespaces gatewayv1beta1.FromNamespaces = "All"
+var listenerHostName gatewayv1beta1.Hostname
+var ingressDomain string
 
 var benchmarkNs = corev1.Namespace{
 	ObjectMeta: metav1.ObjectMeta{
@@ -213,7 +222,7 @@
 var routes = []routev1.Route{
 	{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: fmt.Sprintf("%s-edge", serverName),
 			Labels: map[string]string{
 				"app": "ingress-perf",
 			},
@@ -230,7 +239,7 @@
 	},
 	{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: fmt.Sprintf("%s-reencrypt", serverName),
 			Labels: map[string]string{
 				"app": "ingress-perf",
 			},
@@ -333,3 +342,93 @@
 		},
 	},
 }
+
+var gatewayClass = &gatewayv1beta1.GatewayClass{
+	TypeMeta: metav1.TypeMeta{
+		APIVersion: "gateway.networking.k8s.io/v1beta1",
+		Kind:       "GatewayClass",
+	},
+	ObjectMeta: metav1.ObjectMeta{
+		Name: gatewayClassName,
+	},
+	Spec: gatewayv1beta1.GatewayClassSpec{
+		ControllerName: "openshift.io/gateway-controller",
+	},
+}
+
+var gateway = gatewayv1beta1.Gateway{
+	TypeMeta: metav1.TypeMeta{
+		APIVersion: "gateway.networking.k8s.io/v1beta1",
+		Kind:       "Gateway",
+	},
+	ObjectMeta: metav1.ObjectMeta{
+		Name:      "gateway",
+		Namespace: string(gatewayNamespace),
+	},
+	Spec: gatewayv1beta1.GatewaySpec{
+		GatewayClassName: gatewayv1beta1.ObjectName(gatewayClassName),
+		Listeners: []gatewayv1beta1.Listener{
+
+			{
+				Name:     "http",
+				Port:     80,
+				Protocol: gatewayv1beta1.ProtocolType("HTTP"),
+				Hostname: &listenerHostName,
+				AllowedRoutes: &gatewayv1beta1.AllowedRoutes{
+					Namespaces: &gatewayv1beta1.RouteNamespaces{
+						From: &fromNamespaces,
+					},
+				},
+			},
+			{
+				Name:     "https",
+				Port:     443,
+				Protocol: gatewayv1beta1.ProtocolType("HTTPS"),
+				Hostname: &listenerHostName,
+				AllowedRoutes: &gatewayv1beta1.AllowedRoutes{
+					Namespaces: &gatewayv1beta1.RouteNamespaces{
+						From: &fromNamespaces,
+					},
+				},
+				TLS: &gatewayv1beta1.GatewayTLSConfig{
+					Mode: &tlsType,
+					CertificateRefs: []gatewayv1beta1.SecretObjectReference{
+						{
+							Name: "router-certs-default",
+						},
+					},
+				},
+			},
+		},
+	},
+}
+
+var httproutes = gatewayv1beta1.HTTPRoute{
+	ObjectMeta: metav1.ObjectMeta{
+		Name: service.Name,
+	},
+	Spec: gatewayv1beta1.HTTPRouteSpec{
+		CommonRouteSpec: gatewayv1beta1.CommonRouteSpec{
+			ParentRefs: []gatewayv1beta1.ParentReference{
+				{
+					Namespace: &gatewayNamespace,
+					Name:      gatewayv1beta1.ObjectName("gateway"),
+				},
+			},
+		},
+		Rules: []gatewayv1beta1.HTTPRouteRule{
+			{
+				BackendRefs: []gatewayv1beta1.HTTPBackendRef{
+					{
+						BackendRef: gatewayv1beta1.BackendRef{
+							BackendObjectReference: gatewayv1beta1.BackendObjectReference{
+								Name: gatewayv1beta1.ObjectName(service.Name),
+								Port: &portNumber,
+							},
+						},
+					},
+				},
+			},
+		},
+	},
+}