From 6a5f41d02490c0b84117660f4ce5b600235c9945 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Thu, 13 Feb 2025 13:05:07 +0530 Subject: [PATCH 1/2] Fix API definition fetching for gRPC APIs --- gateway/enforcer/internal/extproc/ext_proc.go | 69 ++++++++++++++++--- .../enforcer/internal/requestconfig/api.go | 5 ++ 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/gateway/enforcer/internal/extproc/ext_proc.go b/gateway/enforcer/internal/extproc/ext_proc.go index d8b130638..933ae9dfd 100644 --- a/gateway/enforcer/internal/extproc/ext_proc.go +++ b/gateway/enforcer/internal/extproc/ext_proc.go @@ -20,9 +20,11 @@ package extproc import ( "bytes" "compress/gzip" + "encoding/base64" "encoding/json" "fmt" "io" + "io/ioutil" "strconv" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -222,14 +224,9 @@ func (s *ExternalProcessingServer) Process(srv envoy_service_proc_v3.ExternalPro dynamicMetadataKeyValuePairs[analytics.APIOrganizationIDKey] = requestConfigHolder.MatchedAPI.OrganizationID dynamicMetadataKeyValuePairs[analytics.APICreatorTenantDomainKey] = requestConfigHolder.MatchedAPI.OrganizationID - requestConfigHolder.ExternalProcessingEnvoyAttributes = attributes if requestConfigHolder.MatchedAPI != nil && requestConfigHolder.MatchedAPI.APIDefinitionPath != "" { definitionPath := requestConfigHolder.MatchedAPI.APIDefinitionPath - fileName := "attachment; filename=\"api_definition.json\"" - if requestConfigHolder.MatchedAPI.IsGraphQLAPI() { - fileName = "attachment; filename=\"api_definition.graphql\"" - } s.cfg.Logger.Info(fmt.Sprintf("definition Path: %v", definitionPath)) fullPath := requestConfigHolder.MatchedAPI.BasePath + requestConfigHolder.MatchedAPI.APIDefinitionPath if attributes.Path == fullPath { @@ -239,6 +236,15 @@ func (s *ExternalProcessingServer) Process(srv envoy_service_proc_v3.ExternalPro if err != nil { s.cfg.Logger.Error(err, "Error reading api definition gzip") } + fileName, contentType := getFileNameAndContentTypeForDef(requestConfigHolder.MatchedAPI) + responseBody := []byte(decompressedStr) + // for grpc apis, the definition might be a zip file + if contentType == "application/zip" { + reader, _ := gzip.NewReader(bytes.NewReader([]byte(requestConfigHolder.MatchedAPI.APIDefinition))) + defer reader.Close() + decompressedData, _ := io.ReadAll(reader) + responseBody, _ = base64.StdEncoding.DecodeString(string(decompressedData)) + } s.cfg.Logger.Info(fmt.Sprintf("decompressed definition: %v", decompressedStr)) if definition != nil { resp = &envoy_service_proc_v3.ProcessingResponse{ @@ -252,7 +258,7 @@ func (s *ExternalProcessingServer) Process(srv envoy_service_proc_v3.ExternalPro { Header: &corev3.HeaderValue{ Key: "Content-Type", - RawValue: []byte("application/octet-stream"), + RawValue: []byte(contentType), }, }, { @@ -263,7 +269,7 @@ func (s *ExternalProcessingServer) Process(srv envoy_service_proc_v3.ExternalPro }, }, }, - Body: []byte(decompressedStr), + Body: responseBody, }, }, } @@ -272,15 +278,15 @@ func (s *ExternalProcessingServer) Process(srv envoy_service_proc_v3.ExternalPro } } s.cfg.Logger.Info(fmt.Sprintf("Metadata context : %+v", req.GetMetadataContext())) - + requestConfigHolder.MatchedResource = httpHandler.GetMatchedResource(requestConfigHolder.MatchedAPI, *requestConfigHolder.ExternalProcessingEnvoyAttributes) if requestConfigHolder.MatchedResource != nil { requestConfigHolder.MatchedResource.RouteMetadataAttributes = attributes dynamicMetadataKeyValuePairs[matchedResourceMetadataKey] = requestConfigHolder.MatchedResource.GetResourceIdentifier() dynamicMetadataKeyValuePairs[analytics.APIResourceTemplateKey] = requestConfigHolder.MatchedResource.Path s.log.Info(fmt.Sprintf("Matched Resource Endpoints: %+v", requestConfigHolder.MatchedResource.Endpoints)) - if requestConfigHolder.MatchedResource.Endpoints!= nil && len(requestConfigHolder.MatchedResource.Endpoints.URLs) > 0 { - dynamicMetadataKeyValuePairs[analytics.DestinationKey] = requestConfigHolder.MatchedResource.Endpoints.URLs[0] + if requestConfigHolder.MatchedResource.Endpoints != nil && len(requestConfigHolder.MatchedResource.Endpoints.URLs) > 0 { + dynamicMetadataKeyValuePairs[analytics.DestinationKey] = requestConfigHolder.MatchedResource.Endpoints.URLs[0] } } metadata, err := extractExternalProcessingMetadata(req.GetMetadataContext()) @@ -290,7 +296,7 @@ func (s *ExternalProcessingServer) Process(srv envoy_service_proc_v3.ExternalPro break } requestConfigHolder.ExternalProcessingEnvoyMetadata = metadata - + // s.log.Info(fmt.Sprintf("Matched api bjc: %v", requestConfigHolder.MatchedAPI.BackendJwtConfiguration)) // s.log.Info(fmt.Sprintf("Matched Resource: %v", requestConfigHolder.MatchedResource)) // s.log.Info(fmt.Sprintf("req holderrr: %+v\n s: %+v", &requestConfigHolder, &s)) @@ -893,6 +899,47 @@ func (s *ExternalProcessingServer) Process(srv envoy_service_proc_v3.ExternalPro } } +func getFileNameAndContentTypeForDef(matchedAPI *requestconfig.API) (string, string) { + fileName := "attachment; filename=\"api_definition.json\"" + contentType := "application/octet-stream" + if matchedAPI.IsGraphQLAPI() { + fileName = "attachment; filename=\"api_definition.graphql\"" + } + if matchedAPI.IsgRPCAPI() { + fileType, _ := DetectFileType([]byte(matchedAPI.APIDefinition)) + + if fileType == "proto" { + return "attachment; filename=\"api_definition.proto\"", contentType + } + if fileType == "zip" { + return "attachment; filename=\"api_definition.zip\"", "application/zip" + } + } + + return fileName, contentType +} + +// DetectFileType detects if the file is a .proto or .zip +func DetectFileType(data []byte) (string, error) { + + reader, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return "", err + } + defer reader.Close() + + decompressedData, err := ioutil.ReadAll(reader) + if err != nil { + return "", err + } + + if bytes.Contains(decompressedData, []byte("syntax = ")) { + return "proto", nil + } + + return "zip", nil +} + // extractExternalProcessingMetadata extracts the external processing metadata from the given data. func extractExternalProcessingMetadata(data *corev3.Metadata) (*dto.ExternalProcessingEnvoyMetadata, error) { filterMatadata := data.GetFilterMetadata() diff --git a/gateway/enforcer/internal/requestconfig/api.go b/gateway/enforcer/internal/requestconfig/api.go index 27ecce4e5..5976ac707 100644 --- a/gateway/enforcer/internal/requestconfig/api.go +++ b/gateway/enforcer/internal/requestconfig/api.go @@ -62,3 +62,8 @@ type API struct { func (api *API) IsGraphQLAPI() bool { return strings.ToLower(api.APIType) == "graphql" } + +// IsgRPCAPI checks whether the API is graphql +func (api *API) IsgRPCAPI() bool { + return strings.ToLower(api.APIType) == "grpc" +} From 438ce6020e2706ec3e95ac58afc110846e10e850 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Thu, 13 Feb 2025 13:15:44 +0530 Subject: [PATCH 2/2] Add API definition test cases for GraphQL and gRPC --- .../src/test/resources/tests/api/GRPC.feature | 25 ++++++++++++++++++- .../test/resources/tests/api/GraphQL.feature | 21 ++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/test/cucumber-tests/src/test/resources/tests/api/GRPC.feature b/test/cucumber-tests/src/test/resources/tests/api/GRPC.feature index 886101dd2..66999bff8 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/GRPC.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/GRPC.feature @@ -46,7 +46,7 @@ Feature: Generating APK conf for gRPC API And I eventually receive 200 response code, not accepting | 429 | | 500 | - And the response body should contain endpoint definition for student.proto + And the response body should be "artifacts/definitions/student.proto" in resources Scenario: Undeploy API @@ -55,6 +55,29 @@ Feature: Generating APK conf for gRPC API When I undeploy the API whose ID is "grpc-basic-api" Then the response status code should be 202 + Scenario: Checking api-definition endpoint to get zip API definition file + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/grpc/order-with-endpoints.apk-conf" + And the definition file "artifacts/definitions/order-definition.zip" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | Bearer ${accessToken} | + | Host | default.gw.wso2.com | + And I send "GET" request to "https://default.gw.wso2.com:9095/grpcapi.v1/api-definition/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And the response body should be "artifacts/definitions/order-definition.zip" in resources + + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "grpc-order-api" + Then the response status code should be 202 + Scenario: Deploying gRPC API with OAuth2 disabled Given The system is ready And I have a valid subscription diff --git a/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature b/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature index 598fa44c4..4cecac6ff 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature @@ -5,6 +5,27 @@ Feature: Generating APK conf for GraphQL API And generate the APK conf file for a "GRAPHQL" API Then the response status code should be 200 + Scenario: Get API definition from a GraphQL API + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_conf_without_sub.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | Bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/graphql/3.14/api-definition" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And the response body should be "artifacts/definitions/graphql_sample_api.graphql" in resources + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "graphql-without-sub" + Then the response status code should be 202 + Scenario: Deploying APK conf using a valid GraphQL API definition without a subscription resource Given The system is ready And I have a valid subscription