From 58a6df59990930d0f1f847bd3207cda333204b26 Mon Sep 17 00:00:00 2001 From: DinethH Date: Thu, 18 Jan 2024 12:23:56 +0530 Subject: [PATCH 01/52] add GRPC api type for API CR --- common-go-libs/apis/dp/v1alpha2/api_types.go | 4 ++-- common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml | 3 ++- helm-charts/templates/crds/dp.wso2.com_apis.yaml | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/common-go-libs/apis/dp/v1alpha2/api_types.go b/common-go-libs/apis/dp/v1alpha2/api_types.go index 4579e5e0b..c5df17cb2 100644 --- a/common-go-libs/apis/dp/v1alpha2/api_types.go +++ b/common-go-libs/apis/dp/v1alpha2/api_types.go @@ -80,9 +80,9 @@ type APISpec struct { Sandbox []EnvConfig `json:"sandbox"` // APIType denotes the type of the API. - // Possible values could be REST, GraphQL, Async + // Possible values could be REST, GraphQL, Async, GRPC etc. // - // +kubebuilder:validation:Enum=REST;GraphQL + // +kubebuilder:validation:Enum=REST;GraphQL;GRPC APIType string `json:"apiType"` // BasePath denotes the basepath of the API. diff --git a/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml b/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml index 7846de3ca..353ad94fa 100644 --- a/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml +++ b/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml @@ -243,10 +243,11 @@ spec: type: array apiType: description: APIType denotes the type of the API. Possible values - could be REST, GraphQL, Async + could be REST, GraphQL, Async, GRPC etc. enum: - REST - GraphQL + - GRPC type: string apiVersion: description: APIVersion is the version number of the API. diff --git a/helm-charts/templates/crds/dp.wso2.com_apis.yaml b/helm-charts/templates/crds/dp.wso2.com_apis.yaml index 399c0c01f..c398f3988 100644 --- a/helm-charts/templates/crds/dp.wso2.com_apis.yaml +++ b/helm-charts/templates/crds/dp.wso2.com_apis.yaml @@ -256,10 +256,11 @@ spec: type: array apiType: description: APIType denotes the type of the API. Possible values - could be REST, GraphQL, Async + could be REST, GraphQL, Async, GRPC etc. enum: - REST - GraphQL + - GRPC type: string apiVersion: description: APIVersion is the version number of the API. From 8287d0fe1ef6e374280f0ee33ad827a42eebf475 Mon Sep 17 00:00:00 2001 From: DinethH Date: Thu, 18 Jan 2024 12:29:00 +0530 Subject: [PATCH 02/52] gitignore router/resources folder --- gateway/router/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 gateway/router/.gitignore diff --git a/gateway/router/.gitignore b/gateway/router/.gitignore new file mode 100644 index 000000000..e47e1b33a --- /dev/null +++ b/gateway/router/.gitignore @@ -0,0 +1 @@ +/resources From 29ee66ca3aba7ef8dbe27043339fc0de0bb100e1 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 19 Mar 2024 13:48:49 +0530 Subject: [PATCH 03/52] c1 after rebase --- .../operator/controllers/dp/api_controller.go | 204 +++++++++++++++++- .../operator/synchronizer/api_state.go | 13 ++ common-go-libs/apis/dp/v1alpha2/api_types.go | 1 - .../config/crd/bases/dp.wso2.com_apis.yaml | 1 - .../serviceAccount/apk-cluster-role.yaml | 2 +- 5 files changed, 217 insertions(+), 4 deletions(-) diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 0c13b4570..c12cbbce1 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -54,6 +54,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" ctrl "sigs.k8s.io/controller-runtime" @@ -67,6 +68,7 @@ import ( ) const ( + grpcRouteAPIIndex = "grpcRouteAPIIndex" httpRouteAPIIndex = "httpRouteAPIIndex" gqlRouteAPIIndex = "gqlRouteAPIIndex" // apiAuthenticationIndex Index for API level authentications @@ -157,6 +159,13 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera return err } + if err := c.Watch(source.Kind(mgr.GetCache(), &gwapiv1a2.GRPCRoute{}), handler.EnqueueRequestsFromMapFunc(apiReconciler.populateAPIReconcileRequestsForGRPCRoute), + predicates...); err != nil { + //TODO change the error number + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2667, logging.BLOCKER, "Error watching GRPCRoute resources: %v", err)) + return err + } + if err := c.Watch(source.Kind(mgr.GetCache(), &gwapiv1b1.Gateway{}), handler.EnqueueRequestsFromMapFunc(apiReconciler.getAPIsForGateway), predicates...); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2611, logging.BLOCKER, "Error watching API resources: %v", err)) @@ -232,6 +241,9 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera // +kubebuilder:rbac:groups=dp.wso2.com,resources=gqlroutes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=dp.wso2.com,resources=gqlroutes/status,verbs=get;update;patch // +kubebuilder:rbac:groups=dp.wso2.com,resources=gqlroutes/finalizers,verbs=update +// +kubebuilder:rbac:groups=dp.wso2.com,resources=grpcroutes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=dp.wso2.com,resources=grpcroutes/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=dp.wso2.com,resources=grpcroutes/finalizers,verbs=update // +kubebuilder:rbac:groups=dp.wso2.com,resources=authentications,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=dp.wso2.com,resources=authentications/status,verbs=get;update;patch // +kubebuilder:rbac:groups=dp.wso2.com,resources=authentications/finalizers,verbs=update @@ -419,6 +431,24 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 } } + //handle grpc apis + if len(prodRouteRefs) > 0 && apiState.APIDefinition.Spec.APIType == "GRPC" { + apiState.ProdGRPCRoute = &synchronizer.GRPCRouteState{} + if apiState.ProdGRPCRoute, err = apiReconciler.resolveGRPCRouteRefs(ctx, prodRouteRefs, + namespace, api); err != nil { + return nil, fmt.Errorf("error while resolving production grpcRouteref %s in namespace :%s has not found. %s", + prodRouteRefs, namespace, err.Error()) + } + } + if len(sandRouteRefs) > 0 && apiState.APIDefinition.Spec.APIType == "GRPC" { + apiState.SandGRPCRoute = &synchronizer.GRPCRouteState{} + if apiState.SandGRPCRoute, err = apiReconciler.resolveGRPCRouteRefs(ctx, sandRouteRefs, + namespace, api); err != nil { + return nil, fmt.Errorf("error while resolving sandbox grpcRouteref %s in namespace :%s has not found. %s", + sandRouteRefs, namespace, err.Error()) + } + } + // handle gql apis if len(prodRouteRefs) > 0 && apiState.APIDefinition.Spec.APIType == "GraphQL" { if apiState.ProdGQLRoute, err = apiReconciler.resolveGQLRouteRefs(ctx, prodRouteRefs, namespace, @@ -516,6 +546,16 @@ func (apiReconciler *APIReconciler) resolveGQLRouteRefs(ctx context.Context, gql return &gqlRouteState, err } +func (apiReconciler *APIReconciler) resolveGRPCRouteRefs(ctx context.Context, grpcRouteRefs []string, + namespace string, api dpv1alpha2.API) (*synchronizer.GRPCRouteState, error) { + grpcRouteState, err := apiReconciler.concatGRPCRoutes(ctx, grpcRouteRefs, namespace, api) + if err != nil { + return nil, err + } + grpcRouteState.Scopes, err = apiReconciler.getScopesForGRPCRoute(ctx, grpcRouteState.GRPCRouteCombined, api) + return &grpcRouteState, err +} + // resolveHTTPRouteRefs validates following references related to the API // - Authentications func (apiReconciler *APIReconciler) resolveHTTPRouteRefs(ctx context.Context, httpRouteState *synchronizer.HTTPRouteState, @@ -534,7 +574,42 @@ func (apiReconciler *APIReconciler) resolveHTTPRouteRefs(ctx context.Context, ht return httpRouteState, err } +func (apiReconciler *APIReconciler) concatGRPCRoutes(ctx context.Context, grpcRouteRefs []string, + namespace string, api dpv1alpha2.API) (synchronizer.GRPCRouteState, error) { + grpcRouteState := synchronizer.GRPCRouteState{} + grpcRoutePartitions := make(map[string]*gwapiv1a2.GRPCRoute) + for _, grpcRouteRef := range grpcRouteRefs { + var grpcRoute gwapiv1a2.GRPCRoute + namespacedName := types.NamespacedName{Namespace: namespace, Name: grpcRouteRef} + if err := utils.ResolveRef(ctx, apiReconciler.client, &api, namespacedName, true, &grpcRoute); err != nil { + return grpcRouteState, fmt.Errorf("error while getting grpcroute %s in namespace :%s, %s", grpcRouteRef, + namespace, err.Error()) + } + grpcRoutePartitions[namespacedName.String()] = &grpcRoute + if grpcRouteState.GRPCRouteCombined == nil { + grpcRouteState.GRPCRouteCombined = &grpcRoute + } else { + grpcRouteState.GRPCRouteCombined.Spec.Rules = append(grpcRouteState.GRPCRouteCombined.Spec.Rules, + grpcRoute.Spec.Rules...) + } + } + grpcRouteState.GRPCRoutePartitions = grpcRoutePartitions + backendNamespacedName := types.NamespacedName{ + //TODO: replace with appropriate attributes in the grpcRoute + //Name: string(grpcRouteState.GRPCRouteCombined.Spec.BackendRefs[0].Name), + //Name: "grpc-backend", + Namespace: namespace, + } + resolvedBackend := utils.GetResolvedBackend(ctx, apiReconciler.client, backendNamespacedName, &api) + if resolvedBackend != nil { + grpcRouteState.BackendMapping = map[string]*dpv1alpha1.ResolvedBackend{ + backendNamespacedName.String(): resolvedBackend, + } + return grpcRouteState, nil + } + return grpcRouteState, errors.New("error while resolving backend for grpcroute") +} func (apiReconciler *APIReconciler) concatGQLRoutes(ctx context.Context, gqlRouteRefs []string, namespace string, api dpv1alpha2.API) (synchronizer.GQLRouteState, error) { gqlRouteState := synchronizer.GQLRouteState{} @@ -623,7 +698,25 @@ func (apiReconciler *APIReconciler) getRatelimitPoliciesForAPI(ctx context.Conte } return ratelimitPolicies, nil } - +func (apiReconciler *APIReconciler) getScopesForGRPCRoute(ctx context.Context, + grpcRoute *gwapiv1a2.GRPCRoute, api dpv1alpha2.API) (map[string]dpv1alpha1.Scope, error) { + scopes := make(map[string]dpv1alpha1.Scope) + for _, rule := range grpcRoute.Spec.Rules { + for _, filter := range rule.Filters { + if filter.ExtensionRef != nil && filter.ExtensionRef.Kind == constants.KindScope { + scope := &dpv1alpha1.Scope{} + if err := utils.ResolveRef(ctx, apiReconciler.client, &api, + types.NamespacedName{Namespace: grpcRoute.Namespace, Name: string(filter.ExtensionRef.Name)}, false, + scope); err != nil { + return nil, fmt.Errorf("error while getting scope %s in namespace :%s, %s", filter.ExtensionRef.Name, + grpcRoute.Namespace, err.Error()) + } + scopes[utils.NamespacedName(scope).String()] = *scope + } + } + } + return scopes, nil +} func (apiReconciler *APIReconciler) getScopesForGQLRoute(ctx context.Context, gqlRoute *dpv1alpha2.GQLRoute, api dpv1alpha2.API) (map[string]dpv1alpha1.Scope, error) { scopes := make(map[string]dpv1alpha1.Scope) @@ -876,6 +969,12 @@ func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForHTTPRoute(ctx return requests } +func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForGRPCRoute(ctx context.Context, obj k8client.Object) []reconcile.Request { + requests := apiReconciler.getAPIForGRPCRoute(ctx, obj) + apiReconciler.handleOwnerReference(ctx, obj, &requests) + return requests +} + func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForConfigMap(ctx context.Context, obj k8client.Object) []reconcile.Request { requests := apiReconciler.getAPIsForConfigMap(ctx, obj) apiReconciler.handleOwnerReference(ctx, obj, &requests) @@ -952,6 +1051,16 @@ func (apiReconciler *APIReconciler) traverseAPIStateAndUpdateOwnerReferences(ctx apiReconciler.retriveParentAPIsAndUpdateOwnerReferene(ctx, gqlRoute) } } + if apiState.ProdGRPCRoute != nil { + for _, grpcRoute := range apiState.ProdGRPCRoute.GRPCRoutePartitions { + apiReconciler.retriveParentAPIsAndUpdateOwnerReferene(ctx, grpcRoute) + } + } + if apiState.SandGRPCRoute != nil { + for _, grpcRoute := range apiState.SandGRPCRoute.GRPCRoutePartitions { + apiReconciler.retriveParentAPIsAndUpdateOwnerReferene(ctx, grpcRoute) + } + } for _, auth := range apiState.Authentications { apiReconciler.retriveParentAPIsAndUpdateOwnerReferene(ctx, &auth) } @@ -1001,6 +1110,20 @@ func (apiReconciler *APIReconciler) traverseAPIStateAndUpdateOwnerReferences(ctx } } } + if apiState.ProdGRPCRoute != nil { + for _, backend := range apiState.ProdGRPCRoute.BackendMapping { + if &backend != nil { + apiReconciler.retriveParentAPIsAndUpdateOwnerReferene(ctx, &backend.Backend) + } + } + } + if apiState.SandGRPCRoute != nil { + for _, backend := range apiState.SandGRPCRoute.BackendMapping { + if &backend != nil { + apiReconciler.retriveParentAPIsAndUpdateOwnerReferene(ctx, &backend.Backend) + } + } + } for _, backendJwt := range apiState.BackendJWTMapping { apiReconciler.retriveParentAPIsAndUpdateOwnerReferene(ctx, &backendJwt) } @@ -1142,6 +1265,16 @@ func (apiReconciler *APIReconciler) retriveParentAPIsAndUpdateOwnerReferene(ctx } requests = apiReconciler.getAPIForGQLRoute(ctx, &gqlRoute) apiReconciler.handleOwnerReference(ctx, &gqlRoute, &requests) + case *gwapiv1a2.GRPCRoute: + var grpcRoute gwapiv1a2.GRPCRoute + namespaceName := types.NamespacedName{ + Name: string(obj.GetName()), + Namespace: string(obj.GetNamespace()), + } + if err := apiReconciler.client.Get(ctx, namespaceName, &grpcRoute); err != nil { + loggers.LoggerAPKOperator.Errorf("Unexpected error occured while loading the cr object from cluster %+v", err) + return + } default: loggers.LoggerAPKOperator.Errorf("Unexpected type found while processing owner reference %+v", obj) } @@ -1219,6 +1352,41 @@ func (apiReconciler *APIReconciler) getAPIForHTTPRoute(ctx context.Context, obj return requests } +func (apiReconciler *APIReconciler) getAPIForGRPCRoute(ctx context.Context, obj k8client.Object) []reconcile.Request { + grpcRoute, ok := obj.(*gwapiv1a2.GRPCRoute) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2622, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", grpcRoute)) + return []reconcile.Request{} + } + + apiList := &dpv1alpha2.APIList{} + + if err := apiReconciler.client.List(ctx, apiList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(grpcRouteAPIIndex, utils.NamespacedName(grpcRoute).String()), + }); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2623, logging.CRITICAL, "Unable to find associated APIs: %s", utils.NamespacedName(grpcRoute).String())) + return []reconcile.Request{} + } + + if len(apiList.Items) == 0 { + loggers.LoggerAPKOperator.Debugf("APIs for GRPCRoute not found: %s", utils.NamespacedName(grpcRoute).String()) + return []reconcile.Request{} + } + + requests := []reconcile.Request{} + for _, api := range apiList.Items { + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: api.Name, + Namespace: api.Namespace}, + } + requests = append(requests, req) + loggers.LoggerAPKOperator.Infof("Adding reconcile request for API: %s/%s with API UUID: %v", api.Namespace, api.Name, + string(api.ObjectMeta.UID)) + } + return requests +} + // getAPIsForConfigMap triggers the API controller reconcile method based on the changes detected // in configMap resources. func (apiReconciler *APIReconciler) getAPIsForConfigMap(ctx context.Context, obj k8client.Object) []reconcile.Request { @@ -1656,6 +1824,40 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { }); err != nil { return err } + if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha2.API{}, grpcRouteAPIIndex, + func(rawObj k8client.Object) []string { + //check Spec.Kind + api := rawObj.(*dpv1alpha2.API) + if api.Spec.APIType != "GRPC" { + return nil + } + var grpcRoutes []string + if len(api.Spec.Production) > 0 { + for _, ref := range api.Spec.Production[0].RouteRefs { + if ref != "" { + grpcRoutes = append(grpcRoutes, + types.NamespacedName{ + Namespace: api.Namespace, + Name: ref, + }.String()) + } + } + } + if len(api.Spec.Sandbox) > 0 { + for _, ref := range api.Spec.Sandbox[0].RouteRefs { + if ref != "" { + grpcRoutes = append(grpcRoutes, + types.NamespacedName{ + Namespace: api.Namespace, + Name: ref, + }.String()) + } + } + } + return grpcRoutes + }); err != nil { + return err + } if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha2.API{}, configMapAPIDefinition, func(rawObj k8client.Object) []string { diff --git a/adapter/internal/operator/synchronizer/api_state.go b/adapter/internal/operator/synchronizer/api_state.go index 10e50c223..cbf42bc2a 100644 --- a/adapter/internal/operator/synchronizer/api_state.go +++ b/adapter/internal/operator/synchronizer/api_state.go @@ -20,6 +20,7 @@ package synchronizer import ( "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) @@ -32,6 +33,8 @@ type APIState struct { SandHTTPRoute *HTTPRouteState ProdGQLRoute *GQLRouteState SandGQLRoute *GQLRouteState + ProdGRPCRoute *GRPCRouteState + SandGRPCRoute *GRPCRouteState Authentications map[string]v1alpha2.Authentication RateLimitPolicies map[string]v1alpha1.RateLimitPolicy ResourceAuthentications map[string]v1alpha2.Authentication @@ -64,3 +67,13 @@ type GQLRouteState struct { BackendMapping map[string]*v1alpha1.ResolvedBackend Scopes map[string]v1alpha1.Scope } + +// GRPCRouteState holds the state of the deployed grpcRoutes. This state is compared with +// the state of the Kubernetes controller cache to detect updates. +// +k8s:deepcopy-gen=true +type GRPCRouteState struct { + GRPCRouteCombined *gwapiv1a2.GRPCRoute + GRPCRoutePartitions map[string]*gwapiv1a2.GRPCRoute + BackendMapping map[string]*v1alpha1.ResolvedBackend + Scopes map[string]v1alpha1.Scope +} diff --git a/common-go-libs/apis/dp/v1alpha2/api_types.go b/common-go-libs/apis/dp/v1alpha2/api_types.go index c5df17cb2..2943fa784 100644 --- a/common-go-libs/apis/dp/v1alpha2/api_types.go +++ b/common-go-libs/apis/dp/v1alpha2/api_types.go @@ -89,7 +89,6 @@ type APISpec struct { // e.g: /pet-store-api/1.0.6 // // +kubectl:validation:MaxLength=232 - // +kubebuilder:validation:Pattern=^[/][a-zA-Z0-9~/_.-]*$ BasePath string `json:"basePath"` // Organization denotes the organization. diff --git a/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml b/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml index 353ad94fa..385c45e7e 100644 --- a/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml +++ b/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml @@ -257,7 +257,6 @@ spec: type: string basePath: description: 'BasePath denotes the basepath of the API. e.g: /pet-store-api/1.0.6' - pattern: ^[/][a-zA-Z0-9~/_.-]*$ type: string definitionFileRef: description: DefinitionFileRef contains the definition of the API diff --git a/helm-charts/templates/serviceAccount/apk-cluster-role.yaml b/helm-charts/templates/serviceAccount/apk-cluster-role.yaml index ccbb849a5..14f7ec717 100644 --- a/helm-charts/templates/serviceAccount/apk-cluster-role.yaml +++ b/helm-charts/templates/serviceAccount/apk-cluster-role.yaml @@ -24,7 +24,7 @@ rules: resources: ["services","configmaps","secrets"] verbs: ["get","list","watch","update","delete","create"] - apiGroups: ["gateway.networking.k8s.io"] - resources: ["httproutes","gateways","gatewayclasses"] + resources: ["httproutes","gateways","grpcroutes","gatewayclasses"] verbs: ["get","list","watch","update","delete","create"] - apiGroups: [ "gateway.networking.k8s.io" ] resources: [ "gateways/status","gatewayclasses/status","httproutes/status" ] From b63f91a2453266c3e6a92eda9e0e4c93485d50bb Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 29 Jan 2024 15:34:22 +0530 Subject: [PATCH 04/52] c2 --- .../operator/controllers/dp/api_controller.go | 82 ++++++++++++++++++- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index c12cbbce1..f05978e00 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -81,6 +81,8 @@ const ( apiRateLimitResourceIndex = "apiRateLimitResourceIndex" // gatewayHTTPRouteIndex Index for gateway httproutes gatewayHTTPRouteIndex = "gatewayHTTPRouteIndex" + // gatewayGRPCRouteIndex Index for gateway grpcroutes + gatewayGRPCRouteIndex = "gatewayGRPCRouteIndex" // apiAPIPolicyIndex Index for API level apipolicies apiAPIPolicyIndex = "apiAPIPolicyIndex" // apiAPIPolicyResourceIndex Index for resource level apipolicies @@ -95,6 +97,7 @@ const ( secretAuthentication = "secretAuthentication" backendHTTPRouteIndex = "backendHTTPRouteIndex" backendGQLRouteIndex = "backendGQLRouteIndex" + backendGRPCRouteIndex = "backendGRPCRouteIndex" interceptorServiceAPIPolicyIndex = "interceptorServiceAPIPolicyIndex" backendInterceptorServiceIndex = "backendInterceptorServiceIndex" backendJWTAPIPolicyIndex = "backendJWTAPIPolicyIndex" @@ -289,6 +292,7 @@ func (apiReconciler *APIReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, nil } + //TODO there's another error here associated with the deployment of grpcroutes if apiState, err := apiReconciler.resolveAPIRefs(ctx, apiCR); err != nil { loggers.LoggerAPKOperator.Warnf("Error retrieving ref CRs for API in namespace : %s with API UUID : %v, %v", req.NamespacedName.String(), string(apiCR.ObjectMeta.UID), err) @@ -596,9 +600,8 @@ func (apiReconciler *APIReconciler) concatGRPCRoutes(ctx context.Context, grpcRo } grpcRouteState.GRPCRoutePartitions = grpcRoutePartitions backendNamespacedName := types.NamespacedName{ - //TODO: replace with appropriate attributes in the grpcRoute - //Name: string(grpcRouteState.GRPCRouteCombined.Spec.BackendRefs[0].Name), - //Name: "grpc-backend", + //TODO check if this is correct + Name: string(grpcRouteState.GRPCRouteCombined.Spec.Rules[0].BackendRefs[0].BackendRef.Name), Namespace: namespace, } resolvedBackend := utils.GetResolvedBackend(ctx, apiReconciler.client, backendNamespacedName, &api) @@ -1352,6 +1355,9 @@ func (apiReconciler *APIReconciler) getAPIForHTTPRoute(ctx context.Context, obj return requests } +// getAPIForGRPCRoute triggers the API controller reconcile method based on the changes detected +// from GRPCRoute objects. If the changes are done for an API stored in the Operator Data store, +// a new reconcile event will be created and added to the reconcile event queue. func (apiReconciler *APIReconciler) getAPIForGRPCRoute(ctx context.Context, obj k8client.Object) []reconcile.Request { grpcRoute, ok := obj.(*gwapiv1a2.GRPCRoute) if !ok { @@ -1675,15 +1681,30 @@ func (apiReconciler *APIReconciler) getAPIsForBackend(ctx context.Context, obj k return []reconcile.Request{} } + grpcRouteList := &gwapiv1a2.GRPCRouteList{} + if err := apiReconciler.client.List(ctx, grpcRouteList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(backendGRPCRouteIndex, utils.NamespacedName(backend).String()), + }); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2625, logging.CRITICAL, "Unable to find associated GRPCRoutes: %s", utils.NamespacedName(backend).String())) + return []reconcile.Request{} + } + if len(httpRouteList.Items) == 0 { loggers.LoggerAPKOperator.Debugf("HTTPRoutes for Backend not found: %s", utils.NamespacedName(backend).String()) } + if len(grpcRouteList.Items) == 0 { + loggers.LoggerAPKOperator.Debugf("GRPCRoutes for Backend not found: %s", utils.NamespacedName(backend).String()) + } requests := []reconcile.Request{} for item := range httpRouteList.Items { httpRoute := httpRouteList.Items[item] requests = append(requests, apiReconciler.getAPIForHTTPRoute(ctx, &httpRoute)...) } + for item := range grpcRouteList.Items { + grpcRoute := grpcRouteList.Items[item] + requests = append(requests, apiReconciler.getAPIForGRPCRoute(ctx, &grpcRoute)...) + } gqlRouteList := &dpv1alpha2.GQLRouteList{} if err := apiReconciler.client.List(ctx, gqlRouteList, &k8client.ListOptions{ @@ -1738,17 +1759,33 @@ func (apiReconciler *APIReconciler) getAPIsForGateway(ctx context.Context, obj k loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2625, logging.CRITICAL, "Unable to find associated HTTPRoutes: %s", utils.NamespacedName(gateway).String())) return []reconcile.Request{} } + grpcRouteList := &gwapiv1a2.GRPCRouteList{} + if err := apiReconciler.client.List(ctx, grpcRouteList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(gatewayGRPCRouteIndex, utils.NamespacedName(gateway).String()), + }); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2625, logging.CRITICAL, "Unable to find associated GRPCRoutes: %s", utils.NamespacedName(gateway).String())) + return []reconcile.Request{} + } if len(httpRouteList.Items) == 0 { loggers.LoggerAPKOperator.Debugf("HTTPRoutes for Gateway not found: %s", utils.NamespacedName(gateway).String()) return []reconcile.Request{} } + if len(grpcRouteList.Items) == 0 { + loggers.LoggerAPKOperator.Debugf("GRPCRoutes for Gateway not found: %s", utils.NamespacedName(gateway).String()) + return []reconcile.Request{} + } requests := []reconcile.Request{} for item := range httpRouteList.Items { httpRoute := httpRouteList.Items[item] requests = append(requests, apiReconciler.getAPIForHTTPRoute(ctx, &httpRoute)...) } + for item := range grpcRouteList.Items { + grpcRoute := grpcRouteList.Items[item] + requests = append(requests, apiReconciler.getAPIForGRPCRoute(ctx, &grpcRoute)...) + } + return requests } @@ -1936,6 +1973,28 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { return err } + // Backend to GRPCRoute indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1a2.GRPCRoute{}, backendGRPCRouteIndex, + func(rawObj k8client.Object) []string { + grpcRoute := rawObj.(*gwapiv1a2.GRPCRoute) + var backends []string + for _, rule := range grpcRoute.Spec.Rules { + for _, backendRef := range rule.BackendRefs { + //TODO fix this if condition + if backendRef.Kind != nil && *backendRef.Kind == constants.KindBackend { + backends = append(backends, types.NamespacedName{ + Namespace: utils.GetNamespace(backendRef.Namespace, + grpcRoute.ObjectMeta.Namespace), + Name: string(backendRef.Name), + }.String()) + } + } + } + return backends + }); err != nil { + return err + } + // Backend to GQLRoute indexer if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha2.GQLRoute{}, backendGQLRouteIndex, func(rawObj k8client.Object) []string { @@ -1973,6 +2032,23 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { return err } + //Gateway to GRPCRoute indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1a2.GRPCRoute{}, gatewayGRPCRouteIndex, + func(rawObj k8client.Object) []string { + grpcRoute := rawObj.(*gwapiv1a2.GRPCRoute) + var gateways []string + for _, parentRef := range grpcRoute.Spec.ParentRefs { + gateways = append(gateways, types.NamespacedName{ + Namespace: utils.GetNamespace(parentRef.Namespace, + grpcRoute.Namespace), + Name: string(parentRef.Name), + }.String()) + } + return gateways + }); err != nil { + return err + } + // ConfigMap to Backend indexer if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha1.Backend{}, configMapBackend, func(rawObj k8client.Object) []string { From 1f62729552c4a7693c77c90b10712d5ffed9ec74 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 30 Jan 2024 11:51:39 +0530 Subject: [PATCH 05/52] reconciled grpc backend --- adapter/internal/operator/controllers/dp/api_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index f05978e00..8d27b83cc 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -1980,7 +1980,6 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { var backends []string for _, rule := range grpcRoute.Spec.Rules { for _, backendRef := range rule.BackendRefs { - //TODO fix this if condition if backendRef.Kind != nil && *backendRef.Kind == constants.KindBackend { backends = append(backends, types.NamespacedName{ Namespace: utils.GetNamespace(backendRef.Namespace, From ea2814e62c114f19fdb9f012ca6d9ba8f75ab9b1 Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 31 Jan 2024 10:51:12 +0530 Subject: [PATCH 06/52] add function SetInfoGRPCRouteCR --- .../oasparser/model/adapter_internal_api.go | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index a5318d66a..c3245d7eb 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "net/url" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "strconv" "strings" "time" @@ -42,6 +43,7 @@ import ( // AdapterInternalAPI represents the object structure holding the information related to the // adapter internal representation. The values are populated from the operator. The pathItem level information is represented // by the resources array which contains the Resource entries. +// TODO task 1 type AdapterInternalAPI struct { UUID string apiType string @@ -940,6 +942,164 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGQLRouteCR(gqlRoute *dpv1al return nil } +// SetInfoGRPCRouteCR populates resources and endpoints of adapterInternalAPI. httpRoute.Spec.Rules.Matches +// are used to create resources and httpRoute.Spec.Rules.BackendRefs are used to create EndpointClusters. +func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwapiv1a2.GRPCRoute, resourceParams ResourceParams) error { + var resources []*Resource + outputAuthScheme := utils.TieBreaker(utils.GetPtrSlice(maps.Values(resourceParams.AuthSchemes))) + outputAPIPolicy := utils.TieBreaker(utils.GetPtrSlice(maps.Values(resourceParams.APIPolicies))) + outputRatelimitPolicy := utils.TieBreaker(utils.GetPtrSlice(maps.Values(resourceParams.RateLimitPolicies))) + + disableScopes := true + config := config.ReadConfigs() + + var authScheme *dpv1alpha2.Authentication + if outputAuthScheme != nil { + authScheme = *outputAuthScheme + } + var apiPolicy *dpv1alpha2.APIPolicy + if outputAPIPolicy != nil { + apiPolicy = *outputAPIPolicy + } + var ratelimitPolicy *dpv1alpha1.RateLimitPolicy + if outputRatelimitPolicy != nil { + ratelimitPolicy = *outputRatelimitPolicy + } + + //We are only supporting one backend for now + //TODO fix this later (Dineth) + backend := grpcRoute.Spec.Rules[0].BackendRefs[0] + backendName := types.NamespacedName{ + Name: string(backend.Name), + Namespace: utils.GetNamespace(backend.Namespace, grpcRoute.Namespace), + } + resolvedBackend, ok := resourceParams.BackendMapping[backendName.String()] + if ok { + endpointConfig := &EndpointConfig{} + if resolvedBackend.CircuitBreaker != nil { + endpointConfig.CircuitBreakers = &CircuitBreakers{ + MaxConnections: int32(resolvedBackend.CircuitBreaker.MaxConnections), + MaxRequests: int32(resolvedBackend.CircuitBreaker.MaxRequests), + MaxPendingRequests: int32(resolvedBackend.CircuitBreaker.MaxPendingRequests), + MaxRetries: int32(resolvedBackend.CircuitBreaker.MaxRetries), + MaxConnectionPools: int32(resolvedBackend.CircuitBreaker.MaxConnectionPools), + } + } + if resolvedBackend.Timeout != nil { + endpointConfig.TimeoutInMillis = resolvedBackend.Timeout.UpstreamResponseTimeout * 1000 + endpointConfig.IdleTimeoutInSeconds = resolvedBackend.Timeout.DownstreamRequestIdleTimeout + } + if resolvedBackend.Retry != nil { + statusCodes := config.Envoy.Upstream.Retry.StatusCodes + if len(resolvedBackend.Retry.StatusCodes) > 0 { + statusCodes = resolvedBackend.Retry.StatusCodes + } + endpointConfig.RetryConfig = &RetryConfig{ + Count: int32(resolvedBackend.Retry.Count), + StatusCodes: statusCodes, + BaseIntervalInMillis: int32(resolvedBackend.Retry.BaseIntervalMillis), + } + } + adapterInternalAPI.Endpoints = &EndpointCluster{ + Endpoints: GetEndpoints(backendName, resourceParams.BackendMapping), + Config: endpointConfig, + } + if resolvedBackend.HealthCheck != nil { + adapterInternalAPI.Endpoints.HealthCheck = &HealthCheck{ + Interval: resolvedBackend.HealthCheck.Interval, + Timeout: resolvedBackend.HealthCheck.Timeout, + UnhealthyThreshold: resolvedBackend.HealthCheck.UnhealthyThreshold, + HealthyThreshold: resolvedBackend.HealthCheck.HealthyThreshold, + } + } + + var securityConfig []EndpointSecurity + switch resolvedBackend.Security.Type { + case "Basic": + securityConfig = append(securityConfig, EndpointSecurity{ + Password: string(resolvedBackend.Security.Basic.Password), + Username: string(resolvedBackend.Security.Basic.Username), + Type: string(resolvedBackend.Security.Type), + Enabled: true, + }) + } + adapterInternalAPI.EndpointSecurity = utils.GetPtrSlice(securityConfig) + } else { + return fmt.Errorf("backend: %s has not been resolved", backendName) + } + + for _, rule := range grpcRoute.Spec.Rules { + var policies = OperationPolicies{} + resourceAuthScheme := authScheme + resourceRatelimitPolicy := ratelimitPolicy + var scopes []string + + for _, filter := range rule.Filters { + if filter.ExtensionRef != nil && filter.ExtensionRef.Kind == constants.KindAuthentication { + if ref, found := resourceParams.ResourceAuthSchemes[types.NamespacedName{ + Name: string(filter.ExtensionRef.Name), + Namespace: grpcRoute.Namespace, + }.String()]; found { + resourceAuthScheme = concatAuthSchemes(authScheme, &ref) + } else { + return fmt.Errorf(`auth scheme: %s has not been resolved, spec.targetRef.kind should be + 'Resource' in resource level Authentications`, filter.ExtensionRef.Name) + } + } + if filter.ExtensionRef != nil && filter.ExtensionRef.Kind == constants.KindScope { + if ref, found := resourceParams.ResourceScopes[types.NamespacedName{ + Name: string(filter.ExtensionRef.Name), + Namespace: grpcRoute.Namespace, + }.String()]; found { + scopes = ref.Spec.Names + disableScopes = false + } else { + return fmt.Errorf("scope: %s has not been resolved in namespace %s", filter.ExtensionRef.Name, grpcRoute.Namespace) + } + } + if filter.ExtensionRef != nil && filter.ExtensionRef.Kind == constants.KindRateLimitPolicy { + if ref, found := resourceParams.ResourceRateLimitPolicies[types.NamespacedName{ + Name: string(filter.ExtensionRef.Name), + Namespace: grpcRoute.Namespace, + }.String()]; found { + resourceRatelimitPolicy = concatRateLimitPolicies(ratelimitPolicy, &ref) + } else { + return fmt.Errorf(`ratelimitpolicy: %s has not been resolved, spec.targetRef.kind should be + 'Resource' in resource level RateLimitPolicies`, filter.ExtensionRef.Name) + } + } + } + resourceAuthScheme = concatAuthSchemes(resourceAuthScheme, nil) + resourceRatelimitPolicy = concatRateLimitPolicies(resourceRatelimitPolicy, nil) + + loggers.LoggerOasparser.Debugf("Calculating auths for API ..., API_UUID = %v", adapterInternalAPI.UUID) + apiAuth := getSecurity(resourceAuthScheme) + + for _, match := range rule.Matches { + resourcePath := *match.Method.Service + resource := &Resource{path: resourcePath, + methods: []*Operation{{iD: uuid.New().String(), method: string(*match.Method.Type), policies: policies, + auth: apiAuth, RateLimitPolicy: parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes: scopes}}, + iD: uuid.New().String(), + } + resources = append(resources, resource) + } + } + + ratelimitPolicy = concatRateLimitPolicies(ratelimitPolicy, nil) + apiPolicy = concatAPIPolicies(apiPolicy, nil) + authScheme = concatAuthSchemes(authScheme, nil) + + adapterInternalAPI.RateLimitPolicy = parseRateLimitPolicyToInternal(ratelimitPolicy) + adapterInternalAPI.resources = resources + adapterInternalAPI.xWso2Cors = getCorsConfigFromAPIPolicy(apiPolicy) + if authScheme.Spec.Override != nil && authScheme.Spec.Override.Disabled != nil { + adapterInternalAPI.disableAuthentications = *authScheme.Spec.Override.Disabled + } + adapterInternalAPI.disableScopes = disableScopes + return nil +} + func (endpoint *Endpoint) validateEndpoint() error { if endpoint.Port == 0 || endpoint.Port > 65535 { return errors.New("endpoint port value should be between 0 and 65535") From bdcbf93e1859f056190807cdecf91ed444b0a9f4 Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 31 Jan 2024 11:38:31 +0530 Subject: [PATCH 07/52] duplicate methods from GQL to grpc_api.go --- .../operator/synchronizer/grpc_api.go | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 adapter/internal/operator/synchronizer/grpc_api.go diff --git a/adapter/internal/operator/synchronizer/grpc_api.go b/adapter/internal/operator/synchronizer/grpc_api.go new file mode 100644 index 000000000..114328cbb --- /dev/null +++ b/adapter/internal/operator/synchronizer/grpc_api.go @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +//Todo task 3 dupliate for grpc +package synchronizer + +import ( + "errors" + "fmt" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/wso2/apk/adapter/config" + "github.com/wso2/apk/adapter/internal/dataholder" + "github.com/wso2/apk/adapter/internal/discovery/xds" + "github.com/wso2/apk/adapter/internal/discovery/xds/common" + "github.com/wso2/apk/adapter/internal/loggers" + "github.com/wso2/apk/adapter/internal/oasparser/model" + "github.com/wso2/apk/adapter/pkg/logging" + "k8s.io/apimachinery/pkg/types" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// generateGRPCAdapterInternalAPI this will populate a AdapterInternalAPI representation for an GRPCRoute +func generateGRPCAdapterInternalAPI(apiState APIState, grpcRoute *GRPCRouteState, envType string) (*model.AdapterInternalAPI, map[string]struct{}, error) { + var adapterInternalAPI model.AdapterInternalAPI + adapterInternalAPI.SetIsDefaultVersion(apiState.APIDefinition.Spec.IsDefaultVersion) + adapterInternalAPI.SetInfoAPICR(*apiState.APIDefinition) + adapterInternalAPI.SetAPIDefinitionFile(apiState.APIDefinitionFile) + adapterInternalAPI.SetAPIDefinitionEndpoint(apiState.APIDefinition.Spec.DefinitionPath) + adapterInternalAPI.SetSubscriptionValidation(apiState.SubscriptionValidation) + adapterInternalAPI.EnvType = envType + + environment := apiState.APIDefinition.Spec.Environment + if environment == "" { + conf := config.ReadConfigs() + environment = conf.Adapter.Environment + } + adapterInternalAPI.SetEnvironment(environment) + + resourceParams := model.ResourceParams{ + AuthSchemes: apiState.Authentications, + ResourceAuthSchemes: apiState.ResourceAuthentications, + BackendMapping: grpcRoute.BackendMapping, + APIPolicies: apiState.APIPolicies, + ResourceAPIPolicies: apiState.ResourceAPIPolicies, + ResourceScopes: grpcRoute.Scopes, + InterceptorServiceMapping: apiState.InterceptorServiceMapping, + BackendJWTMapping: apiState.BackendJWTMapping, + RateLimitPolicies: apiState.RateLimitPolicies, + ResourceRateLimitPolicies: apiState.ResourceRateLimitPolicies, + } + if err := adapterInternalAPI.SetInfoGRPCRouteCR(grpcRoute.GRPCRouteCombined, resourceParams); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2631, logging.MAJOR, "Error setting GRPCRoute CR info to adapterInternalAPI. %v", err)) + return nil, nil, err + } + if err := adapterInternalAPI.Validate(); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2632, logging.MAJOR, "Error validating adapterInternalAPI intermediate representation. %v", err)) + return nil, nil, err + } + vHosts := getVhostsForGRPCAPI(grpcRoute.GRPCRouteCombined) + labels := getLabelsForGRPCAPI(grpcRoute.GRPCRouteCombined) + listeners, relativeSectionNames := getListenersForGRPCAPI(grpcRoute.GRPCRouteCombined, adapterInternalAPI.UUID) + // We don't have a use case where a perticular API's two different grpc routes refer to two different gateway. Hence get the first listener name for the list for processing. + if len(listeners) == 0 || len(relativeSectionNames) == 0 { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2633, logging.MINOR, "Failed to find a matching listener for grpc route: %v. ", + grpcRoute.GRPCRouteCombined.Name)) + return nil, nil, errors.New("failed to find matching listener name for the provided grpc route") + } + + updatedLabelsMap := make(map[string]struct{}) + listenerName := listeners[0] + sectionName := relativeSectionNames[0] + if len(listeners) != 0 { + updatedLabels, err := xds.UpdateAPICache(vHosts, labels, listenerName, sectionName, adapterInternalAPI) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2633, logging.MAJOR, "Error updating the API : %s:%s in vhosts: %s, API_UUID: %v. %v", + adapterInternalAPI.GetTitle(), adapterInternalAPI.GetVersion(), vHosts, adapterInternalAPI.UUID, err)) + return nil, nil, err + } + for newLabel := range updatedLabels { + updatedLabelsMap[newLabel] = struct{}{} + } + } + + return &adapterInternalAPI, updatedLabelsMap, nil +} + +// getVhostForAPI returns the vHosts related to an API. +func getVhostsForGRPCAPI(grpcRoute *gwapiv1a2.GRPCRoute) []string { + var vHosts []string + for _, hostName := range grpcRoute.Spec.Hostnames { + vHosts = append(vHosts, string(hostName)) + } + fmt.Println("vhosts size: ", len(vHosts)) + return vHosts +} + +// getLabelsForAPI returns the labels related to an API. +func getLabelsForGRPCAPI(grpcRoute *gwapiv1a2.GRPCRoute) []string { + var labels []string + var err error + for _, parentRef := range grpcRoute.Spec.ParentRefs { + err = xds.SanitizeGateway(string(parentRef.Name), false) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2653, logging.CRITICAL, "Gateway Label is invalid: %s", string(parentRef.Name))) + } else { + labels = append(labels, string(parentRef.Name)) + } + } + return labels +} + +// getListenersForGRPCAPI returns the listeners related to an API. +func getListenersForGRPCAPI(grpcRoute *gwapiv1a2.GRPCRoute, apiUUID string) ([]string, []string) { + var listeners []string + var sectionNames []string + for _, parentRef := range grpcRoute.Spec.ParentRefs { + namespace := grpcRoute.GetNamespace() + if parentRef.Namespace != nil && *parentRef.Namespace != "" { + namespace = string(*parentRef.Namespace) + } + gateway, found := dataholder.GetGatewayMap()[types.NamespacedName{ + Namespace: namespace, + Name: string(parentRef.Name), + }.String()] + if found { + // find the matching listener + matchedListener, listenerFound := common.FindElement(gateway.Spec.Listeners, func(listener gwapiv1b1.Listener) bool { + if string(listener.Name) == string(*parentRef.SectionName) { + return true + } + return false + }) + if listenerFound { + sectionNames = append(sectionNames, string(matchedListener.Name)) + listeners = append(listeners, common.GetEnvoyListenerName(string(matchedListener.Protocol), uint32(matchedListener.Port))) + continue + } + } + loggers.LoggerAPKOperator.Errorf("Failed to find matching listeners for the grpcroute: %+v", grpcRoute.Name) + } + return listeners, sectionNames +} + +func deleteGRPCAPIFromEnv(grpcRoute *gwapiv1a2.GRPCRoute, apiState APIState) error { + labels := getLabelsForGRPCAPI(grpcRoute) + org := apiState.APIDefinition.Spec.Organization + uuid := string(apiState.APIDefinition.ObjectMeta.UID) + return xds.DeleteAPICREvent(labels, uuid, org) +} + +// undeployGRPCAPIInGateway undeploys the related API in CREATE and UPDATE events. +func undeployGRPCAPIInGateway(apiState APIState) error { + var err error + if apiState.ProdGRPCRoute != nil { + err = deleteGRPCAPIFromEnv(apiState.ProdGRPCRoute.GRPCRouteCombined, apiState) + } + if err != nil { + loggers.LoggerXds.ErrorC(logging.PrintError(logging.Error2630, logging.MAJOR, "Error undeploying prod grpcRoute of API : %v in Organization %v from environments."+ + " Hence not checking on deleting the sand grpcRoute of the API", string(apiState.APIDefinition.ObjectMeta.UID), apiState.APIDefinition.Spec.Organization)) + return err + } + if apiState.SandGRPCRoute != nil { + err = deleteGRPCAPIFromEnv(apiState.SandGRPCRoute.GRPCRouteCombined, apiState) + } + return err +} From 9e85a0386c61d9e32c8058adfa4b204b564c38d8 Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 31 Jan 2024 11:38:55 +0530 Subject: [PATCH 08/52] minor changes --- adapter/internal/oasparser/model/adapter_internal_api.go | 1 + adapter/internal/operator/controllers/dp/api_controller.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index c3245d7eb..feb4e13e5 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -1076,6 +1076,7 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap apiAuth := getSecurity(resourceAuthScheme) for _, match := range rule.Matches { + //TODO double check this resourcePath := *match.Method.Service resource := &Resource{path: resourcePath, methods: []*Operation{{iD: uuid.New().String(), method: string(*match.Method.Type), policies: policies, diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 8d27b83cc..ce7846ffb 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -292,7 +292,6 @@ func (apiReconciler *APIReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, nil } - //TODO there's another error here associated with the deployment of grpcroutes if apiState, err := apiReconciler.resolveAPIRefs(ctx, apiCR); err != nil { loggers.LoggerAPKOperator.Warnf("Error retrieving ref CRs for API in namespace : %s with API UUID : %v, %v", req.NamespacedName.String(), string(apiCR.ObjectMeta.UID), err) From 7186f273aa3443d48687c8f7df83f45aabb5b3ba Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 31 Jan 2024 11:44:05 +0530 Subject: [PATCH 09/52] changed synchronizer.go. Builds without errors --- .../operator/synchronizer/synchronizer.go | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/adapter/internal/operator/synchronizer/synchronizer.go b/adapter/internal/operator/synchronizer/synchronizer.go index a61aa181c..34aeddfc1 100644 --- a/adapter/internal/operator/synchronizer/synchronizer.go +++ b/adapter/internal/operator/synchronizer/synchronizer.go @@ -117,12 +117,16 @@ func undeployAPIInGateway(apiEvent *APIEvent) error { if apiState.APIDefinition.Spec.APIType == "GraphQL" { err = undeployGQLAPIInGateway(apiState) } + if apiState.APIDefinition.Spec.APIType == "GRPC" { + return undeployGRPCAPIInGateway(apiState) + } if err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2629, logging.CRITICAL, "API deployment failed for %s event : %v, %v", apiEvent.EventType, apiState.APIDefinition.Name, err)) } else if config.ReadConfigs().PartitionServer.Enabled { paritionCh <- apiEvent } + return nil } @@ -199,6 +203,44 @@ func deployMultipleAPIsInGateway(event *APIEvent, successChannel *chan SuccessEv } } } + if apiState.APIDefinition.Spec.APIType == "GRPC" { + if apiState.ProdGRPCRoute == nil { + var adapterInternalAPI model.AdapterInternalAPI + adapterInternalAPI.SetInfoAPICR(*apiState.APIDefinition) + xds.RemoveAPICacheForEnv(adapterInternalAPI, constants.Production) + } + if apiState.SandGRPCRoute == nil { + var adapterInternalAPI model.AdapterInternalAPI + adapterInternalAPI.SetInfoAPICR(*apiState.APIDefinition) + xds.RemoveAPICacheForEnv(adapterInternalAPI, constants.Sandbox) + } + if apiState.ProdGRPCRoute != nil { + _, updatedLabels, err := generateGRPCAdapterInternalAPI(apiState, apiState.ProdGRPCRoute, constants.Production) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2665, logging.CRITICAL, + "Error deploying prod grpcRoute of API : %v in Organization %v from environments %v. Error: %v", + string(apiState.APIDefinition.Spec.APIName), apiState.APIDefinition.Spec.Organization, + getLabelsForGRPCAPI(apiState.ProdGRPCRoute.GRPCRouteCombined), err)) + continue + } + for label := range updatedLabels { + updatedLabelsMap[label] = struct{}{} + } + } + if apiState.SandGRPCRoute != nil { + _, updatedLabels, err := generateGRPCAdapterInternalAPI(apiState, apiState.SandGRPCRoute, constants.Sandbox) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2665, logging.CRITICAL, + "Error deploying sand grpcRoute of API : %v in Organization %v from environments %v. Error: %v", + string(apiState.APIDefinition.Spec.APIName), apiState.APIDefinition.Spec.Organization, + getLabelsForGRPCAPI(apiState.SandGRPCRoute.GRPCRouteCombined), err)) + continue + } + for label := range updatedLabels { + updatedLabelsMap[label] = struct{}{} + } + } + } updatedAPIs = append(updatedAPIs, utils.NamespacedName(apiState.APIDefinition)) } @@ -254,6 +296,13 @@ func SendEventToPartitionServer() { for _, hostName := range httpRoute.HTTPRouteCombined.Spec.Hostnames { hostNames = append(hostNames, string(hostName)) } + grpcRoute := api.ProdGRPCRoute + if grpcRoute == nil { + grpcRoute = api.SandGRPCRoute + } + for _, hostName := range grpcRoute.GRPCRouteCombined.Spec.Hostnames { + hostNames = append(hostNames, string(hostName)) + } data := PartitionEvent{ EventType: eventType, BasePath: basePath, From db8d739f58aa2c0eabfbccc118e4d3c2f79de960 Mon Sep 17 00:00:00 2001 From: DinethH Date: Fri, 2 Feb 2024 14:35:45 +0530 Subject: [PATCH 10/52] so far everything works --- .../internal/oasparser/constants/constants.go | 1 + .../envoyconf/routes_with_clusters.go | 44 ++++++++ .../oasparser/model/adapter_internal_api.go | 9 +- adapter/internal/oasparser/model/common.go | 2 +- .../config/crd/bases/dp.wso2.com_apis.yaml | 5 +- .../operator/synchronizer/data_store.go | 102 ++++++++++++++++++ .../config/crd/bases/dp.wso2.com_apis.yaml | 1 + .../serviceAccount/apk-cluster-role.yaml | 9 ++ 8 files changed, 168 insertions(+), 5 deletions(-) diff --git a/adapter/internal/oasparser/constants/constants.go b/adapter/internal/oasparser/constants/constants.go index df863dc5a..3893d473a 100644 --- a/adapter/internal/oasparser/constants/constants.go +++ b/adapter/internal/oasparser/constants/constants.go @@ -98,6 +98,7 @@ const ( SOAP string = "SOAP" WS string = "WS" GRAPHQL string = "GraphQL" + GRPC string = "GRPC" WEBHOOK string = "WEBHOOK" SSE string = "SSE" Prototyped string = "prototyped" diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index acb0edb51..05a7ac3c9 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -197,6 +197,50 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte } return routes, clusters, endpoints, nil } + if adapterInternalAPI.GetAPIType() == constants.GRPC { + basePath := strings.TrimSuffix(adapterInternalAPI.Endpoints.Endpoints[0].Basepath, "/") + + clusterName := getClusterName(adapterInternalAPI.Endpoints.EndpointPrefix, organizationID, vHost, + adapterInternalAPI.GetTitle(), apiVersion, "") + cluster, address, err := processEndpoints(clusterName, adapterInternalAPI.Endpoints, timeout, basePath) + + if err != nil { + logger.LoggerOasparser.ErrorC(logging.PrintError(logging.Error2239, logging.MAJOR, + "Error while adding grpc endpoints for %s:%v. %v", apiTitle, apiVersion, err.Error())) + return nil, nil, nil, fmt.Errorf("error while adding grpc endpoints for %s:%v. %v", apiTitle, apiVersion, + err.Error()) + } + clusters = append(clusters, cluster) + endpoints = append(endpoints, address...) + + // The current code requires to create policy for all routes to support backend endpoint. + policyParameters := make(map[string]interface{}) + policyParameters[constants.RewritePathType] = gwapiv1b1.FullPathHTTPPathModifier + policyParameters[constants.IncludeQueryParams] = true + policyParameters[constants.RewritePathResourcePath] = basePath + var policies = model.OperationPolicies{ + Request: []model.Policy{ + { + PolicyName: string(gwapiv1b1.HTTPRouteFilterURLRewrite), + Action: constants.ActionRewritePath, + Parameters: policyParameters, + }, + }, + } + + grpcop := model.NewOperationWithPolicies("POST", policies) + resource := model.CreateMinimalResource(adapterInternalAPI.GetXWso2Basepath(), []*model.Operation{grpcop}, "", adapterInternalAPI.Endpoints, true, gwapiv1b1.PathMatchExact) + routesP, err := createRoutes(genRouteCreateParams(&adapterInternalAPI, &resource, vHost, basePath, clusterName, nil, + nil, organizationID, false, false)) + if err != nil { + logger.LoggerXds.ErrorC(logging.PrintError(logging.Error2231, logging.MAJOR, + "Error while creating routes for GRPC API %s %s Error: %s", adapterInternalAPI.GetTitle(), + adapterInternalAPI.GetVersion(), err.Error())) + return nil, nil, nil, fmt.Errorf("error while creating routes. %v", err) + } + routes = append(routes, routesP...) + return routes, clusters, endpoints, nil + } for _, resource := range adapterInternalAPI.GetResources() { var clusterName string resourcePath := resource.GetPath() diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index feb4e13e5..2c94f6715 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -1030,10 +1030,11 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap for _, rule := range grpcRoute.Spec.Rules { var policies = OperationPolicies{} + var endPoints []Endpoint resourceAuthScheme := authScheme resourceRatelimitPolicy := ratelimitPolicy var scopes []string - + //TODO remove policies for _, filter := range rule.Filters { if filter.ExtensionRef != nil && filter.ExtensionRef.Kind == constants.KindAuthentication { if ref, found := resourceParams.ResourceAuthSchemes[types.NamespacedName{ @@ -1076,13 +1077,17 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap apiAuth := getSecurity(resourceAuthScheme) for _, match := range rule.Matches { - //TODO double check this resourcePath := *match.Method.Service + endPoints = append(endPoints, GetEndpoints(backendName, resourceParams.BackendMapping)...) resource := &Resource{path: resourcePath, methods: []*Operation{{iD: uuid.New().String(), method: string(*match.Method.Type), policies: policies, auth: apiAuth, RateLimitPolicy: parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes: scopes}}, iD: uuid.New().String(), } + endpoints := GetEndpoints(backendName, resourceParams.BackendMapping) + resource.endpoints = &EndpointCluster{ + Endpoints: endpoints, + } resources = append(resources, resource) } } diff --git a/adapter/internal/oasparser/model/common.go b/adapter/internal/oasparser/model/common.go index d1d3ba664..8c5c2f54c 100644 --- a/adapter/internal/oasparser/model/common.go +++ b/adapter/internal/oasparser/model/common.go @@ -108,7 +108,7 @@ func getHostandBasepathandPort(apiType string, rawURL string) (*Endpoint, error) rawURL = strings.Trim(rawURL, " ") if !strings.Contains(rawURL, "://") { - if apiType == constants.REST || apiType == constants.GRAPHQL { + if apiType == constants.REST || apiType == constants.GRAPHQL || apiType == constants.GRPC { rawURL = "http://" + rawURL } else if apiType == constants.WS { rawURL = "ws://" + rawURL diff --git a/adapter/internal/operator/config/crd/bases/dp.wso2.com_apis.yaml b/adapter/internal/operator/config/crd/bases/dp.wso2.com_apis.yaml index fc4e0a2a9..c2a145323 100644 --- a/adapter/internal/operator/config/crd/bases/dp.wso2.com_apis.yaml +++ b/adapter/internal/operator/config/crd/bases/dp.wso2.com_apis.yaml @@ -72,7 +72,7 @@ spec: type: array apiType: description: APIType denotes the type of the API. Possible values - could be REST, GraphQL, Async + could be REST, GraphQL, GRPC Async enum: - REST type: string @@ -244,10 +244,11 @@ spec: type: array apiType: description: APIType denotes the type of the API. Possible values - could be REST, GraphQL, Async + could be REST, GraphQL, GRPC Async enum: - REST - GraphQL + - GRPC type: string apiVersion: description: APIVersion is the version number of the API. diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 9b3afab1d..8c2c23d4b 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -18,6 +18,7 @@ package synchronizer import ( + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "sync" "github.com/wso2/apk/adapter/internal/loggers" @@ -115,6 +116,23 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced } cachedAPI.ProdGQLRoute = nil } + if apiState.ProdGRPCRoute != nil { + if cachedAPI.ProdGRPCRoute == nil { + cachedAPI.ProdGRPCRoute = apiState.ProdGRPCRoute + updated = true + events = append(events, "Production") + } else if routeEvents, routesUpdated := updateGRPCRoute(apiState.ProdGRPCRoute, cachedAPI.ProdGRPCRoute, + "Production"); routesUpdated { + updated = true + events = append(events, routeEvents...) + } + } else { + if cachedAPI.ProdGRPCRoute != nil { + updated = true + events = append(events, "Production") + } + cachedAPI.ProdGRPCRoute = nil + } if apiState.SandHTTPRoute != nil { if cachedAPI.SandHTTPRoute == nil { cachedAPI.SandHTTPRoute = apiState.SandHTTPRoute @@ -147,6 +165,22 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced } cachedAPI.SandGQLRoute = nil } + if apiState.SandGRPCRoute != nil { + if cachedAPI.SandGRPCRoute == nil { + cachedAPI.SandGRPCRoute = apiState.SandGRPCRoute + updated = true + events = append(events, "Sandbox") + } else if routeEvents, routesUpdated := updateGRPCRoute(apiState.SandGRPCRoute, cachedAPI.SandGRPCRoute, "Sandbox"); routesUpdated { + updated = true + events = append(events, routeEvents...) + } + } else { + if cachedAPI.SandGRPCRoute != nil { + updated = true + events = append(events, "Sandbox") + } + cachedAPI.SandGRPCRoute = nil + } if len(apiState.Authentications) != len(cachedAPI.Authentications) { cachedAPI.Authentications = apiState.Authentications cachedAPI.MutualSSL = apiState.MutualSSL @@ -443,6 +477,61 @@ func updateGQLRoute(gqlRoute *GQLRouteState, cachedGQLRoute *GQLRouteState, endp return events, updated } +func updateGRPCRoute(grpcRoute *GRPCRouteState, cachedGRPCRoute *GRPCRouteState, endpointType string) ([]string, bool) { + var updated bool + events := []string{} + if cachedGRPCRoute.GRPCRouteCombined == nil || !isEqualGRPCRoutes(cachedGRPCRoute.GRPCRoutePartitions, grpcRoute.GRPCRoutePartitions) { + cachedGRPCRoute.GRPCRouteCombined = grpcRoute.GRPCRouteCombined + cachedGRPCRoute.GRPCRoutePartitions = grpcRoute.GRPCRoutePartitions + updated = true + events = append(events, endpointType+" Endpoint") + } + + if len(grpcRoute.Scopes) != len(cachedGRPCRoute.Scopes) { + cachedGRPCRoute.Scopes = grpcRoute.Scopes + updated = true + events = append(events, "Resource Scopes") + } else { + for key, scope := range grpcRoute.Scopes { + if existingScope, found := cachedGRPCRoute.Scopes[key]; found { + if scope.UID != existingScope.UID || scope.Generation > existingScope.Generation { + cachedGRPCRoute.Scopes = grpcRoute.Scopes + updated = true + events = append(events, "Resource Scopes") + break + } + } else { + cachedGRPCRoute.Scopes = grpcRoute.Scopes + updated = true + events = append(events, "Resource Scopes") + break + } + } + } + + if len(grpcRoute.BackendMapping) != len(cachedGRPCRoute.BackendMapping) { + cachedGRPCRoute.BackendMapping = grpcRoute.BackendMapping + updated = true + events = append(events, endpointType+" Backend Properties") + } else { + for key, backend := range grpcRoute.BackendMapping { + if existingBackend, found := cachedGRPCRoute.BackendMapping[key]; found { + if backend.Backend.UID != existingBackend.Backend.UID || backend.Backend.Generation > existingBackend.Backend.Generation { + cachedGRPCRoute.BackendMapping = grpcRoute.BackendMapping + updated = true + events = append(events, endpointType+" Backend Properties") + break + } + } else { + cachedGRPCRoute.BackendMapping = grpcRoute.BackendMapping + updated = true + events = append(events, endpointType+" Backend Properties") + break + } + } + } + return events, updated +} func isEqualHTTPRoutes(cachedHTTPRoutes, newHTTPRoutes map[string]*gwapiv1b1.HTTPRoute) bool { for key, cachedHTTPRoute := range cachedHTTPRoutes { if newHTTPRoutes[key] == nil { @@ -469,6 +558,19 @@ func isEqualGQLRoutes(cachedGQLRoutes, newGQLRoutes map[string]*dpv1alpha2.GQLRo return true } +func isEqualGRPCRoutes(cachedGRPCRoutes, newGRPCRoutes map[string]*gwapiv1a2.GRPCRoute) bool { + for key, cachedGRPCRoute := range cachedGRPCRoutes { + if newGRPCRoutes[key] == nil { + return false + } + if newGRPCRoutes[key].UID == cachedGRPCRoute.UID && + newGRPCRoutes[key].Generation > cachedGRPCRoute.Generation { + return false + } + } + return true +} + // GetCachedAPI get cached apistate func (ods *OperatorDataStore) GetCachedAPI(apiName types.NamespacedName) (APIState, bool) { if cachedAPI, found := ods.apiStore[apiName]; found { diff --git a/common-controller/internal/operator/config/crd/bases/dp.wso2.com_apis.yaml b/common-controller/internal/operator/config/crd/bases/dp.wso2.com_apis.yaml index 304529c66..a674ff1d2 100644 --- a/common-controller/internal/operator/config/crd/bases/dp.wso2.com_apis.yaml +++ b/common-controller/internal/operator/config/crd/bases/dp.wso2.com_apis.yaml @@ -247,6 +247,7 @@ spec: enum: - REST - GraphQL + - GRPC type: string apiVersion: description: APIVersion is the version number of the API. diff --git a/helm-charts/templates/serviceAccount/apk-cluster-role.yaml b/helm-charts/templates/serviceAccount/apk-cluster-role.yaml index 14f7ec717..428ce7abf 100644 --- a/helm-charts/templates/serviceAccount/apk-cluster-role.yaml +++ b/helm-charts/templates/serviceAccount/apk-cluster-role.yaml @@ -122,6 +122,15 @@ rules: - apiGroups: ["dp.wso2.com"] resources: ["gqlroutes/status"] verbs: ["get","patch","update"] + - apiGroups: ["dp.wso2.com"] + resources: ["grpcroutes"] + verbs: ["get","list","watch","update","delete","create"] + - apiGroups: ["dp.wso2.com"] + resources: ["grpcroutes/finalizers"] + verbs: ["update"] + - apiGroups: ["dp.wso2.com"] + resources: ["grpcroutes/status"] + verbs: ["get","patch","update"] - apiGroups: ["cp.wso2.com"] resources: ["applications"] verbs: ["get","list","watch","update","delete","create"] From 93d1d756225efc9f7795565bac0a0241f0e3f186 Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 7 Feb 2024 13:21:44 +0530 Subject: [PATCH 11/52] removed policies --- .../envoyconf/routes_with_clusters.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 05a7ac3c9..48c55cd88 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -213,23 +213,7 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte clusters = append(clusters, cluster) endpoints = append(endpoints, address...) - // The current code requires to create policy for all routes to support backend endpoint. - policyParameters := make(map[string]interface{}) - policyParameters[constants.RewritePathType] = gwapiv1b1.FullPathHTTPPathModifier - policyParameters[constants.IncludeQueryParams] = true - policyParameters[constants.RewritePathResourcePath] = basePath - var policies = model.OperationPolicies{ - Request: []model.Policy{ - { - PolicyName: string(gwapiv1b1.HTTPRouteFilterURLRewrite), - Action: constants.ActionRewritePath, - Parameters: policyParameters, - }, - }, - } - - grpcop := model.NewOperationWithPolicies("POST", policies) - resource := model.CreateMinimalResource(adapterInternalAPI.GetXWso2Basepath(), []*model.Operation{grpcop}, "", adapterInternalAPI.Endpoints, true, gwapiv1b1.PathMatchExact) + resource := model.CreateMinimalResource(adapterInternalAPI.GetXWso2Basepath(), []*model.Operation{}, "", adapterInternalAPI.Endpoints, true, gwapiv1b1.PathMatchExact) routesP, err := createRoutes(genRouteCreateParams(&adapterInternalAPI, &resource, vHost, basePath, clusterName, nil, nil, organizationID, false, false)) if err != nil { From b16758e76652755ec3362c7ab66c61863b7ca3ba Mon Sep 17 00:00:00 2001 From: DinethH Date: Thu, 8 Feb 2024 08:49:06 +0530 Subject: [PATCH 12/52] enable http2 backend for grpc --- adapter/internal/oasparser/envoyconf/routes_with_clusters.go | 1 + 1 file changed, 1 insertion(+) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 48c55cd88..eeb56dee5 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -202,6 +202,7 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte clusterName := getClusterName(adapterInternalAPI.Endpoints.EndpointPrefix, organizationID, vHost, adapterInternalAPI.GetTitle(), apiVersion, "") + adapterInternalAPI.Endpoints.HTTP2BackendEnabled = true cluster, address, err := processEndpoints(clusterName, adapterInternalAPI.Endpoints, timeout, basePath) if err != nil { From 7e3b2c8e1445fbda96ce5da43bc2c07766852a97 Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 19 Feb 2024 14:57:43 +0530 Subject: [PATCH 13/52] remove comment --- adapter/internal/operator/synchronizer/grpc_api.go | 1 - 1 file changed, 1 deletion(-) diff --git a/adapter/internal/operator/synchronizer/grpc_api.go b/adapter/internal/operator/synchronizer/grpc_api.go index 114328cbb..3c0671ed8 100644 --- a/adapter/internal/operator/synchronizer/grpc_api.go +++ b/adapter/internal/operator/synchronizer/grpc_api.go @@ -14,7 +14,6 @@ * limitations under the License. * */ -//Todo task 3 dupliate for grpc package synchronizer import ( From b4a5ba9b00d31c4adbbe69352c96ac88bea5ca32 Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 19 Feb 2024 15:26:25 +0530 Subject: [PATCH 14/52] add constants --- .../org/wso2/apk/enforcer/commons/model/ResourceConfig.java | 2 +- .../src/main/java/org/wso2/apk/enforcer/api/APIFactory.java | 6 ++++++ .../java/org/wso2/apk/enforcer/constants/APIConstants.java | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/gateway/enforcer/org.wso2.apk.enforcer.commons/src/main/java/org/wso2/apk/enforcer/commons/model/ResourceConfig.java b/gateway/enforcer/org.wso2.apk.enforcer.commons/src/main/java/org/wso2/apk/enforcer/commons/model/ResourceConfig.java index 2d938a80e..44428a76b 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer.commons/src/main/java/org/wso2/apk/enforcer/commons/model/ResourceConfig.java +++ b/gateway/enforcer/org.wso2.apk.enforcer.commons/src/main/java/org/wso2/apk/enforcer/commons/model/ResourceConfig.java @@ -41,7 +41,7 @@ public class ResourceConfig { */ public enum HttpMethods { GET("get"), POST("post"), PUT("put"), DELETE("delete"), HEAD("head"), - PATCH("patch"), OPTIONS("options"), QUERY("query"), MUTATION("mutation"), + PATCH("patch"), OPTIONS("options"), QUERY("query"), MUTATION("mutation"), EXACT("Exact"), SUBSCRIPTION("subscription"); private String value; diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java index 58c9c3dd2..83096ba22 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java @@ -78,6 +78,12 @@ public void addApis(List apis) { graphQLAPI.init(api); String apiKey = getApiKey(graphQLAPI); newApis.put(apiKey, graphQLAPI); + } else if (APIConstants.ApiType.GRPC.equals(api.getApiType())) { + GRPCAPI grpcAPI = new GRPCAPI(); + grpcAPI.init(api); + String apiKey = getApiKey(grpcAPI); + newApis.put(apiKey, grpcAPI); + logger.error("GRPC API is not supported Yet"); } else { RestAPI enforcerApi = new RestAPI(); enforcerApi.init(api); diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/APIConstants.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/APIConstants.java index 348f29150..a911a06af 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/APIConstants.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/APIConstants.java @@ -270,6 +270,7 @@ public static class ApiType { public static final String WEB_SOCKET = "WS"; public static final String GRAPHQL = "GraphQL"; + public static final String GRPC = "GRPC"; } /** From 302b36d5387924d7799d37ae1eea8e83c7888f23 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 20 Feb 2024 10:10:07 +0530 Subject: [PATCH 15/52] commented out unimplemented function in enforcer --- .../main/java/org/wso2/apk/enforcer/api/APIFactory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java index 83096ba22..7c50e2c2d 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java @@ -79,10 +79,10 @@ public void addApis(List apis) { String apiKey = getApiKey(graphQLAPI); newApis.put(apiKey, graphQLAPI); } else if (APIConstants.ApiType.GRPC.equals(api.getApiType())) { - GRPCAPI grpcAPI = new GRPCAPI(); - grpcAPI.init(api); - String apiKey = getApiKey(grpcAPI); - newApis.put(apiKey, grpcAPI); + // GRPCAPI grpcAPI = new GRPCAPI(); + // grpcAPI.init(api); + // String apiKey = getApiKey(grpcAPI); + // newApis.put(apiKey, grpcAPI); logger.error("GRPC API is not supported Yet"); } else { RestAPI enforcerApi = new RestAPI(); From 0c548bb1b1210d667e02cc6b468ac21d6619c99a Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 20 Feb 2024 19:45:54 +0530 Subject: [PATCH 16/52] bug fixes after rebasing --- adapter/internal/oasparser/envoyconf/routes_with_clusters.go | 2 +- adapter/internal/oasparser/model/adapter_internal_api.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index eeb56dee5..f62d3e05c 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -215,7 +215,7 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte endpoints = append(endpoints, address...) resource := model.CreateMinimalResource(adapterInternalAPI.GetXWso2Basepath(), []*model.Operation{}, "", adapterInternalAPI.Endpoints, true, gwapiv1b1.PathMatchExact) - routesP, err := createRoutes(genRouteCreateParams(&adapterInternalAPI, &resource, vHost, basePath, clusterName, nil, + routesP, err := createRoutes(genRouteCreateParams(adapterInternalAPI, &resource, vHost, basePath, clusterName, nil, nil, organizationID, false, false)) if err != nil { logger.LoggerXds.ErrorC(logging.PrintError(logging.Error2231, logging.MAJOR, diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 2c94f6715..00f50ac8d 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -43,7 +43,6 @@ import ( // AdapterInternalAPI represents the object structure holding the information related to the // adapter internal representation. The values are populated from the operator. The pathItem level information is represented // by the resources array which contains the Resource entries. -// TODO task 1 type AdapterInternalAPI struct { UUID string apiType string @@ -1081,7 +1080,7 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap endPoints = append(endPoints, GetEndpoints(backendName, resourceParams.BackendMapping)...) resource := &Resource{path: resourcePath, methods: []*Operation{{iD: uuid.New().String(), method: string(*match.Method.Type), policies: policies, - auth: apiAuth, RateLimitPolicy: parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes: scopes}}, + auth: apiAuth, rateLimitPolicy: parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes: scopes}}, iD: uuid.New().String(), } endpoints := GetEndpoints(backendName, resourceParams.BackendMapping) From 3e443ae230baa3c314d1c1d0ac00563a293d7891 Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 26 Feb 2024 11:26:39 +0530 Subject: [PATCH 17/52] fixed envoy configurations --- .../envoyconf/routes_with_clusters.go | 24 +++++++++---------- .../oasparser/model/adapter_internal_api.go | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index f62d3e05c..041ba9249 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -213,18 +213,18 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte } clusters = append(clusters, cluster) endpoints = append(endpoints, address...) - - resource := model.CreateMinimalResource(adapterInternalAPI.GetXWso2Basepath(), []*model.Operation{}, "", adapterInternalAPI.Endpoints, true, gwapiv1b1.PathMatchExact) - routesP, err := createRoutes(genRouteCreateParams(adapterInternalAPI, &resource, vHost, basePath, clusterName, nil, - nil, organizationID, false, false)) - if err != nil { - logger.LoggerXds.ErrorC(logging.PrintError(logging.Error2231, logging.MAJOR, - "Error while creating routes for GRPC API %s %s Error: %s", adapterInternalAPI.GetTitle(), - adapterInternalAPI.GetVersion(), err.Error())) - return nil, nil, nil, fmt.Errorf("error while creating routes. %v", err) - } - routes = append(routes, routesP...) - return routes, clusters, endpoints, nil + //TODO check if creating a minimal resource and appeding this to routes is necessary + //resource := model.CreateMinimalResource(adapterInternalAPI.GetXWso2Basepath(), []*model.Operation{}, "", adapterInternalAPI.Endpoints, false, gwapiv1b1.PathMatchExact) + //routesP, err := createRoutes(genRouteCreateParams(adapterInternalAPI, &resource, vHost, basePath, clusterName, nil, + // nil, organizationID, false, false)) + //if err != nil { + // logger.LoggerXds.ErrorC(logging.PrintError(logging.Error2231, logging.MAJOR, + // "Error while creating routes for GRPC API %s %s Error: %s", adapterInternalAPI.GetTitle(), + // adapterInternalAPI.GetVersion(), err.Error())) + // return nil, nil, nil, fmt.Errorf("error while creating routes. %v", err) + //} + //routes = append(routes, routesP...) + //return routes, clusters, endpoints, nil } for _, resource := range adapterInternalAPI.GetResources() { var clusterName string diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 00f50ac8d..74362ca6b 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -1076,7 +1076,7 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap apiAuth := getSecurity(resourceAuthScheme) for _, match := range rule.Matches { - resourcePath := *match.Method.Service + resourcePath := *match.Method.Service + "/" + *match.Method.Method endPoints = append(endPoints, GetEndpoints(backendName, resourceParams.BackendMapping)...) resource := &Resource{path: resourcePath, methods: []*Operation{{iD: uuid.New().String(), method: string(*match.Method.Type), policies: policies, From 7d7af556320c27910f4d328a0343acbd1f019b29 Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 28 Feb 2024 17:35:48 +0530 Subject: [PATCH 18/52] configured grpc routes on adapter side --- .../envoyconf/routes_with_clusters.go | 41 +++++++++++++------ .../oasparser/model/adapter_internal_api.go | 4 +- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 041ba9249..f5d16e182 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -213,18 +213,6 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte } clusters = append(clusters, cluster) endpoints = append(endpoints, address...) - //TODO check if creating a minimal resource and appeding this to routes is necessary - //resource := model.CreateMinimalResource(adapterInternalAPI.GetXWso2Basepath(), []*model.Operation{}, "", adapterInternalAPI.Endpoints, false, gwapiv1b1.PathMatchExact) - //routesP, err := createRoutes(genRouteCreateParams(adapterInternalAPI, &resource, vHost, basePath, clusterName, nil, - // nil, organizationID, false, false)) - //if err != nil { - // logger.LoggerXds.ErrorC(logging.PrintError(logging.Error2231, logging.MAJOR, - // "Error while creating routes for GRPC API %s %s Error: %s", adapterInternalAPI.GetTitle(), - // adapterInternalAPI.GetVersion(), err.Error())) - // return nil, nil, nil, fmt.Errorf("error while creating routes. %v", err) - //} - //routes = append(routes, routesP...) - //return routes, clusters, endpoints, nil } for _, resource := range adapterInternalAPI.GetResources() { var clusterName string @@ -1074,6 +1062,35 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error routes = append(routes, route) } } + } else if apiType == "GRPC" { + //TODO this is only a temporary measure, gets triggered by grpc api + logger.LoggerOasparser.Debugf("Creating routes for resource : %s that has no policies", resourcePath) + // No policies defined for the resource. Therefore, create one route for all operations. + methodRegex := strings.Join(resourceMethods, "|") + if !strings.Contains(methodRegex, "OPTIONS") { + methodRegex = methodRegex + "|OPTIONS" + } + match := generateRouteMatch(routePath) + //match.Headers = generateHTTPMethodMatcher(methodRegex, clusterName) + action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria) + rewritePath := generateRoutePathForReWrite(basePath, resourcePath, pathMatchType) + action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, resourcePath, pathMatchType) + + //TODO think of a better way to do this + //remove xws2basepaath from resourcepath + suffix := strings.TrimPrefix(resourcePath, xWso2Basepath) + action.Route.RegexRewrite.Substitution = suffix + action.Route.RegexRewrite.Pattern.EngineType = &envoy_type_matcherv3.RegexMatcher_GoogleRe2{} + action.Route.UpgradeConfigs = []*routev3.RouteAction_UpgradeConfig{} + action.Route.ClusterSpecifier = &routev3.RouteAction_Cluster{Cluster: clusterName} + //remove autohostrewrite value + action.Route.HostRewriteSpecifier = nil + decorator = nil + perRouteFilterConfigs = nil + // general headers to add and remove are included in this methods + route := generateRouteConfig(xWso2Basepath, match, action, nil, decorator, perRouteFilterConfigs, nil, nil, nil, nil) + + routes = append(routes, route) } else { logger.LoggerOasparser.Debugf("Creating routes for resource : %s that has no policies", resourcePath) // No policies defined for the resource. Therefore, create one route for all operations. diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 74362ca6b..42dc4afaf 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -1076,9 +1076,9 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap apiAuth := getSecurity(resourceAuthScheme) for _, match := range rule.Matches { - resourcePath := *match.Method.Service + "/" + *match.Method.Method + resourcePath := adapterInternalAPI.GetXWso2Basepath() + "/" + *match.Method.Service + "/" + *match.Method.Method endPoints = append(endPoints, GetEndpoints(backendName, resourceParams.BackendMapping)...) - resource := &Resource{path: resourcePath, + resource := &Resource{path: resourcePath, pathMatchType: "PathPrefix", methods: []*Operation{{iD: uuid.New().String(), method: string(*match.Method.Type), policies: policies, auth: apiAuth, rateLimitPolicy: parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes: scopes}}, iD: uuid.New().String(), From c2b99c8a10ecd3ffebad0eba0b1a6034802997b4 Mon Sep 17 00:00:00 2001 From: DinethH Date: Fri, 1 Mar 2024 10:04:55 +0530 Subject: [PATCH 19/52] debugged context extensions not being set --- .../internal/oasparser/envoyconf/routes_with_clusters.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index f5d16e182..9417a80aa 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -1080,13 +1080,6 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error //remove xws2basepaath from resourcepath suffix := strings.TrimPrefix(resourcePath, xWso2Basepath) action.Route.RegexRewrite.Substitution = suffix - action.Route.RegexRewrite.Pattern.EngineType = &envoy_type_matcherv3.RegexMatcher_GoogleRe2{} - action.Route.UpgradeConfigs = []*routev3.RouteAction_UpgradeConfig{} - action.Route.ClusterSpecifier = &routev3.RouteAction_Cluster{Cluster: clusterName} - //remove autohostrewrite value - action.Route.HostRewriteSpecifier = nil - decorator = nil - perRouteFilterConfigs = nil // general headers to add and remove are included in this methods route := generateRouteConfig(xWso2Basepath, match, action, nil, decorator, perRouteFilterConfigs, nil, nil, nil, nil) From 035e9efe4e6bef7819faf6c64554abdd0cd27ae4 Mon Sep 17 00:00:00 2001 From: DinethH Date: Fri, 1 Mar 2024 10:24:52 +0530 Subject: [PATCH 20/52] Request gets matched to grpc api correctly --- .../org/wso2/apk/enforcer/api/APIFactory.java | 9 +- .../org/wso2/apk/enforcer/api/GRPCAPI.java | 223 ++++++++++++++++++ 2 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/GRPCAPI.java diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java index 7c50e2c2d..ab20bffc3 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java @@ -79,11 +79,10 @@ public void addApis(List apis) { String apiKey = getApiKey(graphQLAPI); newApis.put(apiKey, graphQLAPI); } else if (APIConstants.ApiType.GRPC.equals(api.getApiType())) { - // GRPCAPI grpcAPI = new GRPCAPI(); - // grpcAPI.init(api); - // String apiKey = getApiKey(grpcAPI); - // newApis.put(apiKey, grpcAPI); - logger.error("GRPC API is not supported Yet"); + GRPCAPI grpcAPI = new GRPCAPI(); + grpcAPI.init(api); + String apiKey = getApiKey(grpcAPI); + newApis.put(apiKey, grpcAPI); } else { RestAPI enforcerApi = new RestAPI(); enforcerApi.init(api); diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/GRPCAPI.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/GRPCAPI.java new file mode 100644 index 000000000..ca68c9a59 --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/GRPCAPI.java @@ -0,0 +1,223 @@ +package org.wso2.apk.enforcer.api; + +import graphql.schema.idl.SchemaParser; +import graphql.schema.idl.TypeDefinitionRegistry; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.wso2.apk.enforcer.analytics.AnalyticsFilter; +import org.wso2.apk.enforcer.commons.Filter; +import org.wso2.apk.enforcer.commons.dto.ClaimValueDTO; +import org.wso2.apk.enforcer.commons.dto.JWTConfigurationDto; +import org.wso2.apk.enforcer.commons.model.*; +import org.wso2.apk.enforcer.commons.model.EndpointCluster; +import org.wso2.apk.enforcer.config.ConfigHolder; +import org.wso2.apk.enforcer.config.EnforcerConfig; +import org.wso2.apk.enforcer.constants.APIConstants; +import org.wso2.apk.enforcer.constants.HttpConstants; +import org.wso2.apk.enforcer.cors.CorsFilter; +import org.wso2.apk.enforcer.discovery.api.*; +import org.wso2.apk.enforcer.security.AuthFilter; +import org.wso2.apk.enforcer.security.mtls.MtlsUtils; +import org.wso2.apk.enforcer.server.swagger.APIDefinitionUtils; +import org.wso2.apk.enforcer.util.EndpointUtils; +import org.wso2.apk.enforcer.util.FilterUtils; + +import java.io.IOException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GRPCAPI implements API{ + + private static final Logger logger = LogManager.getLogger(GRPCAPI.class); + private final List filters = new ArrayList<>(); + private APIConfig apiConfig; + @Override + public List getFilters() { + return filters; + } + + @Override + public String init(Api api) { + String vhost = api.getVhost(); + String basePath = api.getBasePath(); + String name = api.getTitle(); + String version = api.getVersion(); + String apiType = api.getApiType(); + List resources = new ArrayList<>(); + Map mtlsCertificateTiers = new HashMap<>(); + String mutualSSL = api.getMutualSSL(); + boolean applicationSecurity = api.getApplicationSecurity(); + + EndpointCluster endpoints = Utils.processEndpoints(api.getEndpoints()); + EndpointSecurity[] endpointSecurity = APIProcessUtils.convertProtoEndpointSecurity( + api.getEndpointSecurityList()); + + for (Resource res : api.getResourcesList()) { + for (Operation operation : res.getMethodsList()) { + ResourceConfig resConfig = Utils.buildResource(operation, res.getPath(), endpointSecurity); + resConfig.setEndpoints(endpoints); + resConfig.setPolicyConfig(Utils.genPolicyConfig(operation.getPolicies())); + resources.add(resConfig); + } + } + + KeyStore trustStore; + try { + trustStore = MtlsUtils.createTrustStore(api.getClientCertificatesList()); + } catch (KeyStoreException e) { + throw new SecurityException(e); + } + + BackendJWTTokenInfo backendJWTTokenInfo = api.getBackendJWTTokenInfo(); + JWTConfigurationDto jwtConfigurationDto = new JWTConfigurationDto(); + + // If backendJWTTokeInfo is available + if (api.hasBackendJWTTokenInfo()) { + Map claims = backendJWTTokenInfo.getCustomClaimsMap(); + Map claimsMap = new HashMap<>(); + for (Map.Entry claimEntry : claims.entrySet()) { + Claim claim = claimEntry.getValue(); + ClaimValueDTO claimVal = new ClaimValueDTO(claim.getValue(), claim.getType()); + claimsMap.put(claimEntry.getKey(), claimVal); + } + EnforcerConfig enforcerConfig = ConfigHolder.getInstance().getConfig(); + jwtConfigurationDto.populateConfigValues(backendJWTTokenInfo.getEnabled(), + backendJWTTokenInfo.getHeader(), backendJWTTokenInfo.getSigningAlgorithm(), + backendJWTTokenInfo.getEncoding(), enforcerConfig.getJwtConfigurationDto().getPublicCert(), + enforcerConfig.getJwtConfigurationDto().getPrivateKey(), backendJWTTokenInfo.getTokenTTL(), + claimsMap, enforcerConfig.getJwtConfigurationDto().useKid(), + enforcerConfig.getJwtConfigurationDto().getKidValue()); + } + + + + SchemaParser schemaParser = new SchemaParser(); + + byte[] apiDefinition = api.getApiDefinitionFile().toByteArray(); + TypeDefinitionRegistry registry; + + //TODO fix this +// try { +// String scheme = APIDefinitionUtils.ReadGzip(apiDefinition); +// registry = schemaParser.parse(scheme); +// } catch (IOException e) { +// logger.error("Error while parsing the GRPC schema definition of the API: " + name, e); +// throw new RuntimeException(e); +// } +// GraphQLSchema schema = UnExecutableSchemaGenerator.makeUnExecutableSchema(registry); + +// GraphQLSchemaDTO graphQLSchemaDTO = new GraphQLSchemaDTO(schema, registry, +// GraphQLPayloadUtils.parseComplexityDTO(api.getGraphqlComplexityInfoList())); + + String apiLifeCycleState = api.getApiLifeCycleState(); + this.apiConfig = new APIConfig.Builder(name).uuid(api.getId()).vhost(vhost).basePath(basePath).version(version) + .resources(resources).apiType(apiType).apiLifeCycleState(apiLifeCycleState).tier(api.getTier()) + .envType(api.getEnvType()).disableAuthentication(api.getDisableAuthentications()) + .disableScopes(api.getDisableScopes()).trustStore(trustStore).organizationId(api.getOrganizationId()) + .mutualSSL(mutualSSL) + .applicationSecurity(applicationSecurity).jwtConfigurationDto(jwtConfigurationDto) +// .apiDefinition(apiDefinition).environment(api.getEnvironment()) + .environment(api.getEnvironment()) +// .subscriptionValidation(api.getSubscriptionValidation()).graphQLSchemaDTO(graphQLSchemaDTO).build(); + .subscriptionValidation(api.getSubscriptionValidation()).build(); + initFilters(); + logger.info("APIConfig: " + this.apiConfig); + return basePath; + } + + @Override + public ResponseObject process(RequestContext requestContext) { + + ResponseObject responseObject = new ResponseObject(requestContext.getRequestID()); + responseObject.setRequestPath(requestContext.getRequestPath()); + boolean analyticsEnabled = ConfigHolder.getInstance().getConfig().getAnalyticsConfig().isEnabled(); + + Utils.handleCommonHeaders(requestContext); + boolean isExistsMatchedOperations = requestContext.getMatchedResourcePaths() != null && + requestContext.getMatchedResourcePaths().size() > 0; + // This flag is used to apply CORS filter + boolean isOptionCall = requestContext.getRequestMethod().contains(HttpConstants.OPTIONS); + + // handle other not allowed && non option request && not yet handled error + // scenarios. + if ((!isOptionCall && !isExistsMatchedOperations) && !requestContext.getProperties() + .containsKey(APIConstants.MessageFormat.ERROR_CODE)) { + requestContext.getProperties() + .put(APIConstants.MessageFormat.STATUS_CODE, APIConstants.StatusCodes.NOTFOUND.getCode()); + requestContext.getProperties().put(APIConstants.MessageFormat.ERROR_CODE, + APIConstants.StatusCodes.NOTFOUND.getValue()); + requestContext.getProperties().put(APIConstants.MessageFormat.ERROR_MESSAGE, + APIConstants.NOT_FOUND_MESSAGE); + requestContext.getProperties().put(APIConstants.MessageFormat.ERROR_DESCRIPTION, + APIConstants.NOT_FOUND_DESCRIPTION); + } + + if ((isExistsMatchedOperations || isOptionCall) && executeFilterChain(requestContext)) { + EndpointUtils.updateClusterHeaderAndCheckEnv(requestContext); + responseObject.setOrganizationId(requestContext.getMatchedAPI().getOrganizationId()); + responseObject.setRemoveHeaderMap(requestContext.getRemoveHeaders()); + responseObject.setQueryParamsToRemove(requestContext.getQueryParamsToRemove()); + responseObject.setRemoveAllQueryParams(requestContext.isRemoveAllQueryParams()); + responseObject.setQueryParamsToAdd(requestContext.getQueryParamsToAdd()); + responseObject.setQueryParamMap(requestContext.getQueryParameters()); + responseObject.setStatusCode(APIConstants.StatusCodes.OK.getCode()); + if (requestContext.getAddHeaders() != null && requestContext.getAddHeaders().size() > 0) { + responseObject.setHeaderMap(requestContext.getAddHeaders()); + } + if (analyticsEnabled) { + AnalyticsFilter.getInstance().handleSuccessRequest(requestContext); + } + // set metadata for interceptors + responseObject.setMetaDataMap(requestContext.getMetadataMap()); + } else { + // If enforcer stops with a false, it will be passed directly to the client. + responseObject.setDirectResponse(true); + responseObject.setStatusCode(Integer.parseInt( + requestContext.getProperties().get(APIConstants.MessageFormat.STATUS_CODE).toString())); + if (requestContext.getProperties().containsKey(APIConstants.MessageFormat.ERROR_CODE)) { + responseObject.setErrorCode( + requestContext.getProperties().get(APIConstants.MessageFormat.ERROR_CODE).toString()); + } + if (requestContext.getProperties().get(APIConstants.MessageFormat.ERROR_MESSAGE) != null) { + responseObject.setErrorMessage(requestContext.getProperties() + .get(APIConstants.MessageFormat.ERROR_MESSAGE).toString()); + } + if (requestContext.getProperties().get(APIConstants.MessageFormat.ERROR_DESCRIPTION) != null) { + responseObject.setErrorDescription(requestContext.getProperties() + .get(APIConstants.MessageFormat.ERROR_DESCRIPTION).toString()); + } + if (requestContext.getAddHeaders() != null && requestContext.getAddHeaders().size() > 0) { + responseObject.setHeaderMap(requestContext.getAddHeaders()); + } + if (analyticsEnabled && !FilterUtils.isSkippedAnalyticsFaultEvent(responseObject.getErrorCode())) { + AnalyticsFilter.getInstance().handleFailureRequest(requestContext); + responseObject.setMetaDataMap(new HashMap<>(0)); + } + } + + return responseObject; + } + + @Override + public APIConfig getAPIConfig() { + return this.apiConfig; + } + + private void initFilters() { + AuthFilter authFilter = new AuthFilter(); + authFilter.init(apiConfig, null); + this.filters.add(authFilter); + +// GraphQLQueryAnalysisFilter queryAnalysisFilter = new GraphQLQueryAnalysisFilter(); +// queryAnalysisFilter.init(apiConfig, null); +// this.filters.add(queryAnalysisFilter); + + // CORS filter is added as the first filter, and it is not customizable. + CorsFilter corsFilter = new CorsFilter(); + this.filters.add(0, corsFilter); + } +} From e91eb18ba7b6713a9c8ef63054ef2f9fcef37817 Mon Sep 17 00:00:00 2001 From: DinethH Date: Fri, 1 Mar 2024 12:45:22 +0530 Subject: [PATCH 21/52] Request gets matched to resource correctly --- .../src/main/java/org/wso2/apk/enforcer/api/APIFactory.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java index ab20bffc3..ea97c7f68 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java @@ -138,6 +138,12 @@ public byte[] getAPIDefinition(final String basePath, final String version, fina public ResourceConfig getMatchedResource(API api, String matchedResourcePath, String method) { List resourceConfigList = api.getAPIConfig().getResources(); + if (APIConstants.ApiType.GRPC.equals(api.getAPIConfig().getApiType())) { + //TODO check if there's another way to do this, perhaps in the adapter side by setting the method + return resourceConfigList.stream() + .filter(resourceConfig -> resourceConfig.getPath().equals(matchedResourcePath)) + .findFirst().orElse(null); + } return resourceConfigList.stream() .filter(resourceConfig -> resourceConfig.getPath().equals(matchedResourcePath)). filter(resourceConfig -> (method == null) || resourceConfig.getMethod() From 6f1c0b0b8c4bb359c2bd54fec2920aafb180e3df Mon Sep 17 00:00:00 2001 From: DinethH Date: Fri, 1 Mar 2024 17:59:30 +0530 Subject: [PATCH 22/52] enabled http2 backend and unset transport socket matching --- adapter/internal/oasparser/envoyconf/routes_with_clusters.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 9417a80aa..67439767d 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -204,7 +204,6 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte adapterInternalAPI.GetTitle(), apiVersion, "") adapterInternalAPI.Endpoints.HTTP2BackendEnabled = true cluster, address, err := processEndpoints(clusterName, adapterInternalAPI.Endpoints, timeout, basePath) - if err != nil { logger.LoggerOasparser.ErrorC(logging.PrintError(logging.Error2239, logging.MAJOR, "Error while adding grpc endpoints for %s:%v. %v", apiTitle, apiVersion, err.Error())) @@ -218,12 +217,16 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte var clusterName string resourcePath := resource.GetPath() endpoint := resource.GetEndpoints() + //TODO see if I can configure this elsewhere + endpoint.HTTP2BackendEnabled = true basePath := strings.TrimSuffix(endpoint.Endpoints[0].Basepath, "/") existingClusterName := getExistingClusterName(*endpoint, processedEndpoints) if existingClusterName == "" { clusterName = getClusterName(endpoint.EndpointPrefix, organizationID, vHost, adapterInternalAPI.GetTitle(), apiVersion, resource.GetID()) cluster, address, err := processEndpoints(clusterName, endpoint, timeout, basePath) + //TODO see if I can configure this elsewhere + cluster.TransportSocketMatches = nil if err != nil { logger.LoggerOasparser.ErrorC(logging.PrintError(logging.Error2239, logging.MAJOR, "Error while adding resource level endpoints for %s:%v-%v. %v", apiTitle, apiVersion, resourcePath, err.Error())) } else { From 3da8731deccd94d4fed279762fc307698439752e Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 5 Mar 2024 11:36:54 +0530 Subject: [PATCH 23/52] add grpc CRs for testing --- .../tests/resources/tests/grpc-api.yaml | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 test/integration/integration/tests/resources/tests/grpc-api.yaml diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml new file mode 100644 index 000000000..eabb527a9 --- /dev/null +++ b/test/integration/integration/tests/resources/tests/grpc-api.yaml @@ -0,0 +1,130 @@ +# Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: dp.wso2.com/v1alpha2 +kind: API +metadata: + name: dineth-grpc-api + namespace: gateway-integration-test-infra + labels: + api-name: "dineth-grpc-api" + api-version: "v1" + organization: "7505d64a54e061b7acd54ccd58b49dc43500b635" + managed-by: "apk" +spec: + apiName: Dineth GRPC API + apiType: GRPC + apiVersion: v1 + basePath: /dineth-grpc-api/v1 + isDefaultVersion: true + production: + - httpRouteRefs: + - dineth-grpc-route + organization: apk-org + +--- + +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: GRPCRoute +metadata: + name: dineth-grpc-route + namespace: gateway-integration-test-infra + labels: + api-name: "dineth-grpc-api" + api-version: "v1" + organization: "7505d64a54e061b7acd54ccd58b49dc43500b635" + managed-by: "apk" +spec: + hostnames: + - grpc.test.gw.wso2.com + parentRefs: + - group: "gateway.networking.k8s.io" + kind: "Gateway" + name: "default" + sectionName: "httpslistener" + rules: + - matches: + - method: + service: StudentService + method: GetStudent + backendRefs: + - name: dineth-grpc-backend + kind: Backend + port: 6565 + - matches: + - method: + service: StudentService + method: SendStudent + backendRefs: + - name: dineth-grpc-backend + kind: Backend + port: 6565 + - matches: + - method: + service: TeacherService + method: GetTeacher + backendRefs: + - name: dineth-grpc-backend + kind: Backend + port: 6565 + +--- + +#apiVersion: "dp.wso2.com/v1alpha1" +#kind: "Authentication" +#metadata: +# name: dineth-grpc-api-auth +# namespace: gateway-integration-test-infra +# spec: +# default: +# authTypes: +# disabled: true +# targetRef: +# group: "gateway.networking.k8s.io" +# kind: Resource +# namespace: gateway-integration-test-infra +# name: dineth-grpc-api +apiVersion: dp.wso2.com/v1alpha1 +kind: Authentication +metadata: + name: disable-grpc-api-security + namespace: gateway-integration-test-infra +spec: + override: + disabled: true + targetRef: + group: gateway.networking.k8s.io + kind: API + namespace: gateway-integration-test-infra + name: dineth-grpc-api +--- + +apiVersion: dp.wso2.com/v1alpha1 +kind: Backend +metadata: + name: dineth-grpc-backend + namespace: gateway-integration-test-infra + labels: + api-name: "dineth-grpc-api" + api-version: "v1" + organization: "7505d64a54e061b7acd54ccd58b49dc43500b635" + managed-by: "apk" +spec: + services: + - host: grpc-backend + port: 6565 + basePath: "" + protocol: https From 8220d52165988a0e67ba559bec65caeae7e460d7 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 5 Mar 2024 14:51:41 +0530 Subject: [PATCH 24/52] add custom grpc backend deployment to manifest --- .../tests/resources/base/manifests.yaml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/test/integration/integration/tests/resources/base/manifests.yaml b/test/integration/integration/tests/resources/base/manifests.yaml index 5d4ef2460..c902689fd 100644 --- a/test/integration/integration/tests/resources/base/manifests.yaml +++ b/test/integration/integration/tests/resources/base/manifests.yaml @@ -113,6 +113,57 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: grpc-demo-app + namespace: gateway-integration-test-infra +spec: + selector: + app: grpc-demo-app + ports: + - protocol: TCP + port: 6565 + targetPort: 6565 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grpc-demo-app + namespace: gateway-integration-test-infra + labels: + app: grpc-demo-app +spec: + replicas: 1 + selector: + matchLabels: + app: grpc-demo-app + template: + metadata: + labels: + app: grpc-demo-app + spec: + containers: + - name: grpc-demo-app + image: grpc-demo-app:latest + imagePullPolicy: IfNotPresent + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: "100m" # Adjust as needed + memory: "100Mi" # Adjust as needed + ports: + - containerPort: 6565 # Ensuring the container exposes port 6565 for gRPC traffic + +--- +apiVersion: v1 +kind: Service metadata: name: infra-backend-v2 namespace: gateway-integration-test-infra From 0af1bfb62bd46a70beeaf0fb8c5a7230750917de Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 5 Mar 2024 15:22:49 +0530 Subject: [PATCH 25/52] update grpc CRs --- .../tests/resources/tests/grpc-api.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml index eabb527a9..184672b78 100644 --- a/test/integration/integration/tests/resources/tests/grpc-api.yaml +++ b/test/integration/integration/tests/resources/tests/grpc-api.yaml @@ -22,7 +22,7 @@ metadata: labels: api-name: "dineth-grpc-api" api-version: "v1" - organization: "7505d64a54e061b7acd54ccd58b49dc43500b635" + organization: wso2-org managed-by: "apk" spec: apiName: Dineth GRPC API @@ -33,7 +33,7 @@ spec: production: - httpRouteRefs: - dineth-grpc-route - organization: apk-org + organization: wso2-org --- @@ -45,16 +45,16 @@ metadata: labels: api-name: "dineth-grpc-api" api-version: "v1" - organization: "7505d64a54e061b7acd54ccd58b49dc43500b635" + organization: "wso2-org" managed-by: "apk" spec: hostnames: - grpc.test.gw.wso2.com parentRefs: - - group: "gateway.networking.k8s.io" - kind: "Gateway" - name: "default" - sectionName: "httpslistener" + - group: gateway.networking.k8s.io + kind: Gateway + name: default + sectionName: httpslistener rules: - matches: - method: @@ -120,11 +120,11 @@ metadata: labels: api-name: "dineth-grpc-api" api-version: "v1" - organization: "7505d64a54e061b7acd54ccd58b49dc43500b635" + organization: wso2-org managed-by: "apk" spec: services: - - host: grpc-backend + - host: grpc-backend.gateway-integration-test-infra port: 6565 basePath: "" protocol: https From a1e1ff611c4851cf0eeef3e21c9e7096c42698e7 Mon Sep 17 00:00:00 2001 From: DinethH Date: Fri, 8 Mar 2024 14:52:25 +0530 Subject: [PATCH 26/52] fix transport socket not matching --- .../envoyconf/routes_with_clusters.go | 60 +++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 67439767d..acebab5be 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -212,21 +212,73 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte } clusters = append(clusters, cluster) endpoints = append(endpoints, address...) + + //TODO factor out common code (Dineth) + for _, resource := range adapterInternalAPI.GetResources() { + var clusterName string + resourcePath := resource.GetPath() + endpoint := resource.GetEndpoints() + //TODO see if this should be configured elsewhere (Dineth) + endpoint.HTTP2BackendEnabled = true + basePath := strings.TrimSuffix(endpoint.Endpoints[0].Basepath, "/") + existingClusterName := getExistingClusterName(*endpoint, processedEndpoints) + + if existingClusterName == "" { + clusterName = getClusterName(endpoint.EndpointPrefix, organizationID, vHost, adapterInternalAPI.GetTitle(), apiVersion, resource.GetID()) + cluster, address, err := processEndpoints(clusterName, endpoint, timeout, basePath) + //TODO see if this should be configured elsewhere (Dineth) + cluster.TransportSocketMatches = nil + if err != nil { + logger.LoggerOasparser.ErrorC(logging.PrintError(logging.Error2239, logging.MAJOR, "Error while adding resource level endpoints for %s:%v-%v. %v", apiTitle, apiVersion, resourcePath, err.Error())) + } else { + clusters = append(clusters, cluster) + endpoints = append(endpoints, address...) + processedEndpoints[clusterName] = *endpoint + } + } else { + clusterName = existingClusterName + } + // Create resource level interceptor clusters if required + //TODO remove this if no interceptor needed for grpc (Dineth) + clustersI, endpointsI, operationalReqInterceptors, operationalRespInterceptorVal := createInterceptorResourceClusters(adapterInternalAPI, + interceptorCerts, vHost, organizationID, apiRequestInterceptor, apiResponseInterceptor, resource) + clusters = append(clusters, clustersI...) + endpoints = append(endpoints, endpointsI...) + routeParams := genRouteCreateParams(adapterInternalAPI, resource, vHost, basePath, clusterName, *operationalReqInterceptors, *operationalRespInterceptorVal, organizationID, + false, false) + + routeP, err := createRoutes(routeParams) + if err != nil { + logger.LoggerXds.ErrorC(logging.PrintError(logging.Error2231, logging.MAJOR, + "Error while creating routes for GRPC API %s %s for path: %s Error: %s", adapterInternalAPI.GetTitle(), + adapterInternalAPI.GetVersion(), resource.GetPath(), err.Error())) + return nil, nil, nil, fmt.Errorf("error while creating routes. %v", err) + } + routes = append(routes, routeP...) + if adapterInternalAPI.IsDefaultVersion { + defaultRoutes, errDefaultPath := createRoutes(genRouteCreateParams(adapterInternalAPI, resource, vHost, basePath, clusterName, *operationalReqInterceptors, *operationalRespInterceptorVal, organizationID, + false, true)) + if errDefaultPath != nil { + logger.LoggerXds.ErrorC(logging.PrintError(logging.Error2231, logging.MAJOR, "Error while creating routes for GRPC API %s %s for path: %s Error: %s", adapterInternalAPI.GetTitle(), adapterInternalAPI.GetVersion(), removeFirstOccurrence(resource.GetPath(), adapterInternalAPI.GetVersion()), errDefaultPath.Error())) + return nil, nil, nil, fmt.Errorf("error while creating routes. %v", errDefaultPath) + } + routes = append(routes, defaultRoutes...) + } + + } + + return routes, clusters, endpoints, nil } for _, resource := range adapterInternalAPI.GetResources() { var clusterName string resourcePath := resource.GetPath() endpoint := resource.GetEndpoints() - //TODO see if I can configure this elsewhere - endpoint.HTTP2BackendEnabled = true basePath := strings.TrimSuffix(endpoint.Endpoints[0].Basepath, "/") existingClusterName := getExistingClusterName(*endpoint, processedEndpoints) if existingClusterName == "" { clusterName = getClusterName(endpoint.EndpointPrefix, organizationID, vHost, adapterInternalAPI.GetTitle(), apiVersion, resource.GetID()) cluster, address, err := processEndpoints(clusterName, endpoint, timeout, basePath) - //TODO see if I can configure this elsewhere - cluster.TransportSocketMatches = nil if err != nil { logger.LoggerOasparser.ErrorC(logging.PrintError(logging.Error2239, logging.MAJOR, "Error while adding resource level endpoints for %s:%v-%v. %v", apiTitle, apiVersion, resourcePath, err.Error())) } else { From 807edbf480eec84b23b7564344adf1dd8267f37b Mon Sep 17 00:00:00 2001 From: DinethH Date: Sat, 9 Mar 2024 09:35:56 +0530 Subject: [PATCH 27/52] change gRPC CRs --- .../tests/resources/base/manifests.yaml | 22 ++++++------ .../tests/resources/tests/grpc-api.yaml | 34 ++++++++----------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/test/integration/integration/tests/resources/base/manifests.yaml b/test/integration/integration/tests/resources/base/manifests.yaml index c902689fd..08b7b8b63 100644 --- a/test/integration/integration/tests/resources/base/manifests.yaml +++ b/test/integration/integration/tests/resources/base/manifests.yaml @@ -114,11 +114,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: grpc-demo-app + name: dineth-grpc-demo-server namespace: gateway-integration-test-infra spec: selector: - app: grpc-demo-app + app: dineth-grpc-demo-server ports: - protocol: TCP port: 6565 @@ -127,23 +127,23 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: grpc-demo-app + name: dineth-grpc-demo-server namespace: gateway-integration-test-infra labels: - app: grpc-demo-app + app: dineth-grpc-demo-server spec: replicas: 1 selector: matchLabels: - app: grpc-demo-app + app: dineth-grpc-demo-server template: metadata: labels: - app: grpc-demo-app + app: dineth-grpc-demo-server spec: containers: - - name: grpc-demo-app - image: grpc-demo-app:latest + - name: dineth-grpc-demo-server + image: dineth-grpc-demo-server:latest imagePullPolicy: IfNotPresent env: - name: POD_NAME @@ -156,10 +156,10 @@ spec: fieldPath: metadata.namespace resources: requests: - cpu: "100m" # Adjust as needed - memory: "100Mi" # Adjust as needed + cpu: "100m" + memory: "100Mi" ports: - - containerPort: 6565 # Ensuring the container exposes port 6565 for gRPC traffic + - containerPort: 6565 --- apiVersion: v1 diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml index 184672b78..3dc983e24 100644 --- a/test/integration/integration/tests/resources/tests/grpc-api.yaml +++ b/test/integration/integration/tests/resources/tests/grpc-api.yaml @@ -31,7 +31,7 @@ spec: basePath: /dineth-grpc-api/v1 isDefaultVersion: true production: - - httpRouteRefs: + - routeRefs: - dineth-grpc-route organization: wso2-org @@ -58,7 +58,7 @@ spec: rules: - matches: - method: - service: StudentService + service: student.StudentService method: GetStudent backendRefs: - name: dineth-grpc-backend @@ -66,16 +66,24 @@ spec: port: 6565 - matches: - method: - service: StudentService - method: SendStudent + service: student.StudentService + method: SendStudentStream backendRefs: - name: dineth-grpc-backend kind: Backend port: 6565 - matches: - method: - service: TeacherService - method: GetTeacher + service: student.StudentService + method: GetStudentStream + backendRefs: + - name: dineth-grpc-backend + kind: Backend + port: 6565 + - matches: + - method: + service: student.StudentService + method: SendAndGetStudentStream backendRefs: - name: dineth-grpc-backend kind: Backend @@ -83,20 +91,6 @@ spec: --- -#apiVersion: "dp.wso2.com/v1alpha1" -#kind: "Authentication" -#metadata: -# name: dineth-grpc-api-auth -# namespace: gateway-integration-test-infra -# spec: -# default: -# authTypes: -# disabled: true -# targetRef: -# group: "gateway.networking.k8s.io" -# kind: Resource -# namespace: gateway-integration-test-infra -# name: dineth-grpc-api apiVersion: dp.wso2.com/v1alpha1 kind: Authentication metadata: From d3ed16a2234d5b4536978a99b8aebfbcc827522c Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 11 Mar 2024 10:27:41 +0530 Subject: [PATCH 28/52] remove deployment and manifest for grpc backend pod --- .../tests/resources/base/manifests.yaml | 51 ------- .../tests/resources/tests/grpc-api.yaml | 124 ------------------ 2 files changed, 175 deletions(-) delete mode 100644 test/integration/integration/tests/resources/tests/grpc-api.yaml diff --git a/test/integration/integration/tests/resources/base/manifests.yaml b/test/integration/integration/tests/resources/base/manifests.yaml index 08b7b8b63..5d4ef2460 100644 --- a/test/integration/integration/tests/resources/base/manifests.yaml +++ b/test/integration/integration/tests/resources/base/manifests.yaml @@ -113,57 +113,6 @@ spec: --- apiVersion: v1 kind: Service -metadata: - name: dineth-grpc-demo-server - namespace: gateway-integration-test-infra -spec: - selector: - app: dineth-grpc-demo-server - ports: - - protocol: TCP - port: 6565 - targetPort: 6565 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dineth-grpc-demo-server - namespace: gateway-integration-test-infra - labels: - app: dineth-grpc-demo-server -spec: - replicas: 1 - selector: - matchLabels: - app: dineth-grpc-demo-server - template: - metadata: - labels: - app: dineth-grpc-demo-server - spec: - containers: - - name: dineth-grpc-demo-server - image: dineth-grpc-demo-server:latest - imagePullPolicy: IfNotPresent - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: "100m" - memory: "100Mi" - ports: - - containerPort: 6565 - ---- -apiVersion: v1 -kind: Service metadata: name: infra-backend-v2 namespace: gateway-integration-test-infra diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml deleted file mode 100644 index 3dc983e24..000000000 --- a/test/integration/integration/tests/resources/tests/grpc-api.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. -# -# WSO2 LLC. licenses this file to you under the Apache License, -# Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -apiVersion: dp.wso2.com/v1alpha2 -kind: API -metadata: - name: dineth-grpc-api - namespace: gateway-integration-test-infra - labels: - api-name: "dineth-grpc-api" - api-version: "v1" - organization: wso2-org - managed-by: "apk" -spec: - apiName: Dineth GRPC API - apiType: GRPC - apiVersion: v1 - basePath: /dineth-grpc-api/v1 - isDefaultVersion: true - production: - - routeRefs: - - dineth-grpc-route - organization: wso2-org - ---- - -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: GRPCRoute -metadata: - name: dineth-grpc-route - namespace: gateway-integration-test-infra - labels: - api-name: "dineth-grpc-api" - api-version: "v1" - organization: "wso2-org" - managed-by: "apk" -spec: - hostnames: - - grpc.test.gw.wso2.com - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: default - sectionName: httpslistener - rules: - - matches: - - method: - service: student.StudentService - method: GetStudent - backendRefs: - - name: dineth-grpc-backend - kind: Backend - port: 6565 - - matches: - - method: - service: student.StudentService - method: SendStudentStream - backendRefs: - - name: dineth-grpc-backend - kind: Backend - port: 6565 - - matches: - - method: - service: student.StudentService - method: GetStudentStream - backendRefs: - - name: dineth-grpc-backend - kind: Backend - port: 6565 - - matches: - - method: - service: student.StudentService - method: SendAndGetStudentStream - backendRefs: - - name: dineth-grpc-backend - kind: Backend - port: 6565 - ---- - -apiVersion: dp.wso2.com/v1alpha1 -kind: Authentication -metadata: - name: disable-grpc-api-security - namespace: gateway-integration-test-infra -spec: - override: - disabled: true - targetRef: - group: gateway.networking.k8s.io - kind: API - namespace: gateway-integration-test-infra - name: dineth-grpc-api ---- - -apiVersion: dp.wso2.com/v1alpha1 -kind: Backend -metadata: - name: dineth-grpc-backend - namespace: gateway-integration-test-infra - labels: - api-name: "dineth-grpc-api" - api-version: "v1" - organization: wso2-org - managed-by: "apk" -spec: - services: - - host: grpc-backend.gateway-integration-test-infra - port: 6565 - basePath: "" - protocol: https From 0ff06d687d053260f387c670ad10095bfa40c254 Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 11 Mar 2024 15:23:33 +0530 Subject: [PATCH 29/52] add CRs for grpc api --- .../tests/resources/base/manifests.yaml | 47 +++++++ .../tests/resources/tests/grpc-api.yaml | 124 ++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 test/integration/integration/tests/resources/tests/grpc-api.yaml diff --git a/test/integration/integration/tests/resources/base/manifests.yaml b/test/integration/integration/tests/resources/base/manifests.yaml index 5d4ef2460..fb369f030 100644 --- a/test/integration/integration/tests/resources/base/manifests.yaml +++ b/test/integration/integration/tests/resources/base/manifests.yaml @@ -113,6 +113,53 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: dineth-grpc-demo-server + namespace: gateway-integration-test-infra +spec: + selector: + app: dineth-grpc-demo-server + ports: + - protocol: TCP + port: 6565 + targetPort: 6565 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dineth-grpc-demo-server + namespace: gateway-integration-test-infra + labels: + app: dineth-grpc-demo-server +spec: + replicas: 1 + selector: + matchLabels: + app: dineth-grpc-demo-server + template: + metadata: + labels: + app: dineth-grpc-demo-server + spec: + containers: + - name: dineth-grpc-demo-server + image: ddh13/dineth-grpc-demo-server:1.0.0 + imagePullPolicy: Always + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: v1 +kind: Service metadata: name: infra-backend-v2 namespace: gateway-integration-test-infra diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml new file mode 100644 index 000000000..3eba502fa --- /dev/null +++ b/test/integration/integration/tests/resources/tests/grpc-api.yaml @@ -0,0 +1,124 @@ +# Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: dp.wso2.com/v1alpha2 +kind: API +metadata: + name: dineth-grpc-api + namespace: gateway-integration-test-infra + labels: + api-name: "dineth-grpc-api" + api-version: "v1" + organization: wso2-org + managed-by: "apk" +spec: + apiName: Dineth GRPC API + apiType: GRPC + apiVersion: v1 + basePath: /dineth-grpc-api/v1 + isDefaultVersion: false + production: + - routeRefs: + - dineth-grpc-route + organization: wso2-org + +--- + +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: GRPCRoute +metadata: + name: dineth-grpc-route + namespace: gateway-integration-test-infra + labels: + api-name: "dineth-grpc-api" + api-version: "v1" + organization: "wso2-org" + managed-by: "apk" +spec: + hostnames: + - grpc.test.gw.wso2.com + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: default + sectionName: httpslistener + rules: + - matches: + - method: + service: student.StudentService + method: GetStudent + backendRefs: + - name: dineth-grpc-backend + kind: Backend + port: 6565 + - matches: + - method: + service: student.StudentService + method: SendStudentStream + backendRefs: + - name: dineth-grpc-backend + kind: Backend + port: 6565 + - matches: + - method: + service: student.StudentService + method: GetStudentStream + backendRefs: + - name: dineth-grpc-backend + kind: Backend + port: 6565 + - matches: + - method: + service: student.StudentService + method: SendAndGetStudentStream + backendRefs: + - name: dineth-grpc-backend + kind: Backend + port: 6565 + +--- + +apiVersion: dp.wso2.com/v1alpha1 +kind: Authentication +metadata: + name: disable-grpc-api-security + namespace: gateway-integration-test-infra +spec: + override: + disabled: true + targetRef: + group: gateway.networking.k8s.io + kind: API + namespace: gateway-integration-test-infra + name: dineth-grpc-api +--- + +apiVersion: dp.wso2.com/v1alpha1 +kind: Backend +metadata: + name: dineth-grpc-backend + namespace: gateway-integration-test-infra + labels: + api-name: "dineth-grpc-api" + api-version: "v1" + organization: wso2-org + managed-by: "apk" +spec: + services: + - host: grpc-backend.gateway-integration-test-infra + port: 6565 + basePath: "" + protocol: https From 96b3f6aaf6efcda9f6d402516f6225d4a706ce1f Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 11 Mar 2024 15:40:15 +0530 Subject: [PATCH 30/52] add empty test for grpc api --- .../integration/integration/tests/grpc-api.go | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 test/integration/integration/tests/grpc-api.go diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go new file mode 100644 index 000000000..69738db0f --- /dev/null +++ b/test/integration/integration/tests/grpc-api.go @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tests + +import ( + "testing" + + "github.com/wso2/apk/test/integration/integration/utils/http" + "github.com/wso2/apk/test/integration/integration/utils/suite" +) + +func init() { + IntegrationTests = append(IntegrationTests, GRPCAPI) +} + +// DisableAPISecurity test +var GRPCAPI = suite.IntegrationTest{ + ShortName: "GRPCAPI", + Description: "Tests gRPC API", + Manifests: []string{"tests/grpc-api.yaml"}, + Test: func(t *testing.T, suite *suite.IntegrationTestSuite) { + gwAddr := "grpc.test.gw.wso2.com:9095" + //token := http.GetTestToken(t) + + testCases := []http.ExpectedResponse{ + //{ + // Request: http.Request{ + // Host: "gql.test.gw.wso2.com", + // Path: "/gql/v1", + // Method: "POST", + // Headers: map[string]string{ + // "Content-Type": "application/json", + // }, + // Body: `{"query":"query{\n human(id:1000){\n id\n name\n }\n}","variables":{}}`, + // }, + // ExpectedRequest: &http.ExpectedRequest{ + // Request: http.Request{ + // Method: ""}, + // }, + // Response: http.Response{StatusCode: 200}, + //}, + + } + for i := range testCases { + tc := testCases[i] + t.Run(tc.GetTestCaseName(i), func(t *testing.T) { + t.Parallel() + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, tc) + }) + } + }, +} From 4ce242b2f06add25d402de3ba3fe1a80e6805ca4 Mon Sep 17 00:00:00 2001 From: DinethH Date: Fri, 15 Mar 2024 01:22:42 +0530 Subject: [PATCH 31/52] works with postman client --- .../envoyconf/routes_with_clusters.go | 28 ++++--------------- .../oasparser/model/adapter_internal_api.go | 7 ++--- 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index acebab5be..040882a29 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -1117,28 +1117,6 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error routes = append(routes, route) } } - } else if apiType == "GRPC" { - //TODO this is only a temporary measure, gets triggered by grpc api - logger.LoggerOasparser.Debugf("Creating routes for resource : %s that has no policies", resourcePath) - // No policies defined for the resource. Therefore, create one route for all operations. - methodRegex := strings.Join(resourceMethods, "|") - if !strings.Contains(methodRegex, "OPTIONS") { - methodRegex = methodRegex + "|OPTIONS" - } - match := generateRouteMatch(routePath) - //match.Headers = generateHTTPMethodMatcher(methodRegex, clusterName) - action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria) - rewritePath := generateRoutePathForReWrite(basePath, resourcePath, pathMatchType) - action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, resourcePath, pathMatchType) - - //TODO think of a better way to do this - //remove xws2basepaath from resourcepath - suffix := strings.TrimPrefix(resourcePath, xWso2Basepath) - action.Route.RegexRewrite.Substitution = suffix - // general headers to add and remove are included in this methods - route := generateRouteConfig(xWso2Basepath, match, action, nil, decorator, perRouteFilterConfigs, nil, nil, nil, nil) - - routes = append(routes, route) } else { logger.LoggerOasparser.Debugf("Creating routes for resource : %s that has no policies", resourcePath) // No policies defined for the resource. Therefore, create one route for all operations. @@ -1152,6 +1130,12 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error rewritePath := generateRoutePathForReWrite(basePath, resourcePath, pathMatchType) action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, resourcePath, pathMatchType) + if apiType == "GRPC" { + match.Headers = nil + newRoutePath := "/" + strings.TrimPrefix(resourcePath, basePath+".") + action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, newRoutePath, pathMatchType) + } + route := generateRouteConfig(xWso2Basepath, match, action, nil, decorator, perRouteFilterConfigs, nil, nil, nil, nil) // general headers to add and remove are included in this methods routes = append(routes, route) diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 42dc4afaf..6f0ac2f82 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -966,7 +966,6 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap } //We are only supporting one backend for now - //TODO fix this later (Dineth) backend := grpcRoute.Spec.Rules[0].BackendRefs[0] backendName := types.NamespacedName{ Name: string(backend.Name), @@ -1033,7 +1032,6 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap resourceAuthScheme := authScheme resourceRatelimitPolicy := ratelimitPolicy var scopes []string - //TODO remove policies for _, filter := range rule.Filters { if filter.ExtensionRef != nil && filter.ExtensionRef.Kind == constants.KindAuthentication { if ref, found := resourceParams.ResourceAuthSchemes[types.NamespacedName{ @@ -1075,10 +1073,11 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap loggers.LoggerOasparser.Debugf("Calculating auths for API ..., API_UUID = %v", adapterInternalAPI.UUID) apiAuth := getSecurity(resourceAuthScheme) + //TODO Add path match type (Dineth) for _, match := range rule.Matches { - resourcePath := adapterInternalAPI.GetXWso2Basepath() + "/" + *match.Method.Service + "/" + *match.Method.Method + resourcePath := adapterInternalAPI.GetXWso2Basepath() + "." + *match.Method.Service + "/" + *match.Method.Method endPoints = append(endPoints, GetEndpoints(backendName, resourceParams.BackendMapping)...) - resource := &Resource{path: resourcePath, pathMatchType: "PathPrefix", + resource := &Resource{path: resourcePath, pathMatchType: "Exact", methods: []*Operation{{iD: uuid.New().String(), method: string(*match.Method.Type), policies: policies, auth: apiAuth, rateLimitPolicy: parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes: scopes}}, iD: uuid.New().String(), From ece6f36d5feaafbd149668cc9dad2c3865b93ad0 Mon Sep 17 00:00:00 2001 From: DinethH Date: Sat, 16 Mar 2024 11:39:14 +0530 Subject: [PATCH 32/52] changed grpc CR for test --- .../integration/integration/tests/resources/tests/grpc-api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml index 3eba502fa..71f80a74b 100644 --- a/test/integration/integration/tests/resources/tests/grpc-api.yaml +++ b/test/integration/integration/tests/resources/tests/grpc-api.yaml @@ -28,7 +28,7 @@ spec: apiName: Dineth GRPC API apiType: GRPC apiVersion: v1 - basePath: /dineth-grpc-api/v1 + basePath: /dineth.grpc.api.v1 isDefaultVersion: false production: - routeRefs: From dc5a56783cb9711dcdc603f877291ab294b73942 Mon Sep 17 00:00:00 2001 From: DinethH Date: Sat, 16 Mar 2024 12:35:59 +0530 Subject: [PATCH 33/52] move basepath validation for API CRs --- .../apis/dp/v1alpha2/api_webhook.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/common-go-libs/apis/dp/v1alpha2/api_webhook.go b/common-go-libs/apis/dp/v1alpha2/api_webhook.go index 2e3348be3..67faddea5 100644 --- a/common-go-libs/apis/dp/v1alpha2/api_webhook.go +++ b/common-go-libs/apis/dp/v1alpha2/api_webhook.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "io" + "regexp" "strings" gqlparser "github.com/vektah/gqlparser" @@ -103,6 +104,8 @@ func (r *API) validateAPI() error { if r.Spec.BasePath == "" { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("basePath"), "API basePath is required")) + } else if errMsg := validateAPIBasePathRegex(r.Spec.BasePath, r.Spec.APIType); errMsg != "" { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("basePath"), r.Spec.BasePath, errMsg)) } else if errMsg := validateAPIBasePathFormat(r.Spec.BasePath, r.Spec.APIVersion); errMsg != "" { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("basePath"), r.Spec.BasePath, errMsg)) } else if err := r.validateAPIBasePathExistsAndDefaultVersion(); err != nil { @@ -250,6 +253,23 @@ func validateAPIBasePathFormat(basePath string, apiVersion string) string { return "" } +func validateAPIBasePathRegex(basePath, apiType string) string { + var pattern string + if apiType == "GRPC" { + pattern = `^[/][a-zA-Z][a-zA-Z0-9_.]*$` + } else { + pattern = `^[/][a-zA-Z0-9~/_.-]*$` + } + re, err := regexp.Compile(pattern) + if err != nil { + return "Failed to compile basePath regex pattern" + } + if !re.MatchString(basePath) { + return "API basePath is not in a valid format for the specified API type" + } + return "" +} + // getBasePathWithoutVersion returns the basePath without version func getBasePathWithoutVersion(basePath string) string { lastIndex := strings.LastIndex(basePath, "/") From fe60f67cd5d5f9853ea153ab27750fa1c4c5ced8 Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 18 Mar 2024 19:15:20 +0530 Subject: [PATCH 34/52] change test files --- .../integration/integration/tests/grpc-api.go | 160 +++++++++++++++--- .../tests/resources/base/manifests.yaml | 14 +- .../tests/resources/tests/grpc-api.yaml | 45 ++--- test/integration/scripts/run-tests.sh | 1 + test/integration/scripts/setup-hosts.sh | 1 + 5 files changed, 160 insertions(+), 61 deletions(-) diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go index 69738db0f..8dcba85e1 100644 --- a/test/integration/integration/tests/grpc-api.go +++ b/test/integration/integration/tests/grpc-api.go @@ -18,9 +18,14 @@ package tests import ( + "context" + "crypto/tls" + "github.com/wso2/apk/test/integration/integration/utils/generatedcode/student" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "testing" + "time" - "github.com/wso2/apk/test/integration/integration/utils/http" "github.com/wso2/apk/test/integration/integration/utils/suite" ) @@ -28,40 +33,147 @@ func init() { IntegrationTests = append(IntegrationTests, GRPCAPI) } -// DisableAPISecurity test +// GRPCAPI tests gRPC API var GRPCAPI = suite.IntegrationTest{ ShortName: "GRPCAPI", Description: "Tests gRPC API", - Manifests: []string{"tests/grpc-api.yaml"}, + //Manifests: []string{"tests/grpc-api.yaml"}, Test: func(t *testing.T, suite *suite.IntegrationTestSuite) { gwAddr := "grpc.test.gw.wso2.com:9095" //token := http.GetTestToken(t) - testCases := []http.ExpectedResponse{ - //{ - // Request: http.Request{ - // Host: "gql.test.gw.wso2.com", - // Path: "/gql/v1", - // Method: "POST", - // Headers: map[string]string{ - // "Content-Type": "application/json", - // }, - // Body: `{"query":"query{\n human(id:1000){\n id\n name\n }\n}","variables":{}}`, - // }, - // ExpectedRequest: &http.ExpectedRequest{ - // Request: http.Request{ - // Method: ""}, - // }, - // Response: http.Response{StatusCode: 200}, - //}, - + //testCases := []http.ExpectedResponse{ + //{ + // Request: http.Request{ + // Host: "gql.test.gw.wso2.com", + // Path: "/gql/v1", + // Method: "POST", + // Headers: map[string]string{ + // "Content-Type": "application/json", + // }, + // Body: `{"query":"query{\n human(id:1000){\n id\n name\n }\n}","variables":{}}`, + // }, + // ExpectedRequest: &http.ExpectedRequest{ + // Request: http.Request{ + // Method: ""}, + // }, + // Response: http.Response{StatusCode: 200}, + //}, + //} + testCases := []ExpectedResponse{ + { + out: &student.StudentResponse{ + Name: "Dineth", + Age: 10, + }, + err: nil, + }, } for i := range testCases { tc := testCases[i] - t.Run(tc.GetTestCaseName(i), func(t *testing.T) { - t.Parallel() - http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, tc) + //t.Run(tc.GetTestCaseName(i), func(t *testing.T) { + // t.Parallel() + // http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, tc) + //}) + t.Run("Invoke gRPC API", func(t *testing.T) { + out, err := invokeGRPCClientUntilSatisfied(gwAddr, t) + if err != nil { + if tc.err != nil { + t.Errorf("Err -> \nWant: %q\nGot: %q\n", tc.err, err) + } + } else { + if tc.out.Name != out.Name || + tc.out.Age != out.Age { + t.Errorf("Out -> \nWant: %q\nGot : %q", tc.out, out) + } + } + }) } }, } + +func invokeGRPCClient(gwAddr string, t *testing.T) (*student.StudentResponse, error) { + + t.Logf("Starting gRPC client...") + + // Set up TLS credentials for the connection without enforcing server certificate validation. + t.Logf("Setting up TLS credentials without server certificate validation...") + config := &tls.Config{ + InsecureSkipVerify: true, // CAUTION: This disables SSL certificate verification. + } + creds := credentials.NewTLS(config) + + // Dial the server with the TLS credentials and a dial timeout. + dialCtx, dialCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer dialCancel() + t.Logf("Dialing to server at %s with timeout...", gwAddr) + conn, err := grpc.DialContext(dialCtx, gwAddr, grpc.WithTransportCredentials(creds), grpc.WithBlock()) + if err != nil { + t.Fatalf("Failed to connect: %v", err) + } + defer conn.Close() + t.Log("Successfully connected to the server.") + + c := student.NewStudentServiceClient(conn) + + // Prepare the context with a timeout for the request. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + t.Log("Sending request to the server...") + // Create a StudentRequest message + r := &student.StudentRequest{Id: 1234} // Adjust the ID according to your actual implementation + response, err := c.GetStudent(ctx, r) + if err != nil { + t.Logf("Could not fetch student: %v", err) + } + + t.Logf("Received response from server: %v\n", response) + t.Logf("Student Details: %v\n", response) + return response, nil +} + +type ExpectedResponse struct { + out *student.StudentResponse + err error +} + +func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T) (*student.StudentResponse, error) { + var out *student.StudentResponse + var err error + attempt := 0 + maxAttempts := 4 + + for attempt < maxAttempts { + t.Logf("Attempt %d to invoke gRPC client...", attempt+1) + out, err = invokeGRPCClient(gwAddr, t) + + if err != nil { + t.Logf("Error on attempt %d: %v", attempt+1, err) + } else { + // Check if the response is satisfactory. This condition needs to be defined. + // For example, assuming a satisfactory condition is when out.Satisfied is true. + // This is a placeholder condition and should be replaced with your actual success criteria. + if out != nil && isResponseSatisfactory(out) { + t.Logf("Satisfactory response received: %+v", out) + return out, nil + } + } + + if attempt < maxAttempts-1 { + t.Logf("Waiting 20 seconds before next attempt...") + time.Sleep(20 * time.Second) + } + attempt++ + } + + t.Logf("Failed to receive a satisfactory response after %d attempts", maxAttempts) + return out, err // Returning the last response and error, might need adjustment based on requirements. +} + +func isResponseSatisfactory(response *student.StudentResponse) bool { + // Define the condition for a response to be considered satisfactory. + // This is a placeholder function and should contain actual logic to evaluate the response. + return false // Placeholder: assume every response is satisfactory. +} diff --git a/test/integration/integration/tests/resources/base/manifests.yaml b/test/integration/integration/tests/resources/base/manifests.yaml index fb369f030..58883fa98 100644 --- a/test/integration/integration/tests/resources/base/manifests.yaml +++ b/test/integration/integration/tests/resources/base/manifests.yaml @@ -114,11 +114,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: dineth-grpc-demo-server + name: grpc-backend-v1 namespace: gateway-integration-test-infra spec: selector: - app: dineth-grpc-demo-server + app: grpc-backend-v1 ports: - protocol: TCP port: 6565 @@ -127,22 +127,22 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: dineth-grpc-demo-server + name: grpc-backend-v1 namespace: gateway-integration-test-infra labels: - app: dineth-grpc-demo-server + app: grpc-backend-v1 spec: replicas: 1 selector: matchLabels: - app: dineth-grpc-demo-server + app: grpc-backend-v1 template: metadata: labels: - app: dineth-grpc-demo-server + app: grpc-backend-v1 spec: containers: - - name: dineth-grpc-demo-server + - name: grpc-backend-v1 image: ddh13/dineth-grpc-demo-server:1.0.0 imagePullPolicy: Always env: diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml index 71f80a74b..e8585acea 100644 --- a/test/integration/integration/tests/resources/tests/grpc-api.yaml +++ b/test/integration/integration/tests/resources/tests/grpc-api.yaml @@ -17,22 +17,17 @@ apiVersion: dp.wso2.com/v1alpha2 kind: API metadata: - name: dineth-grpc-api + name: grpc-api namespace: gateway-integration-test-infra - labels: - api-name: "dineth-grpc-api" - api-version: "v1" - organization: wso2-org - managed-by: "apk" spec: - apiName: Dineth GRPC API + apiName: GRPC API apiType: GRPC apiVersion: v1 basePath: /dineth.grpc.api.v1 isDefaultVersion: false production: - routeRefs: - - dineth-grpc-route + - grpc-api-grpcroute organization: wso2-org --- @@ -40,28 +35,24 @@ spec: apiVersion: gateway.networking.k8s.io/v1alpha2 kind: GRPCRoute metadata: - name: dineth-grpc-route + name: grpc-api-grpcroute namespace: gateway-integration-test-infra - labels: - api-name: "dineth-grpc-api" - api-version: "v1" - organization: "wso2-org" - managed-by: "apk" spec: - hostnames: - - grpc.test.gw.wso2.com parentRefs: - group: gateway.networking.k8s.io kind: Gateway - name: default + name: wso2-apk-default + namespace: apk-integration-test sectionName: httpslistener + hostnames: + - grpc.test.gw.wso2.com rules: - matches: - method: service: student.StudentService method: GetStudent backendRefs: - - name: dineth-grpc-backend + - name: grpc-backend-v1 kind: Backend port: 6565 - matches: @@ -69,7 +60,7 @@ spec: service: student.StudentService method: SendStudentStream backendRefs: - - name: dineth-grpc-backend + - name: grpc-backend-v1 kind: Backend port: 6565 - matches: @@ -77,7 +68,7 @@ spec: service: student.StudentService method: GetStudentStream backendRefs: - - name: dineth-grpc-backend + - name: grpc-backend-v1 kind: Backend port: 6565 - matches: @@ -85,7 +76,7 @@ spec: service: student.StudentService method: SendAndGetStudentStream backendRefs: - - name: dineth-grpc-backend + - name: grpc-backend-v1 kind: Backend port: 6565 @@ -103,22 +94,16 @@ spec: group: gateway.networking.k8s.io kind: API namespace: gateway-integration-test-infra - name: dineth-grpc-api + name: grpc-api --- - apiVersion: dp.wso2.com/v1alpha1 kind: Backend metadata: - name: dineth-grpc-backend + name: grpc-backend-v1 namespace: gateway-integration-test-infra - labels: - api-name: "dineth-grpc-api" - api-version: "v1" - organization: wso2-org - managed-by: "apk" spec: services: - - host: grpc-backend.gateway-integration-test-infra + - host: grpc-backend-v1.gateway-integration-test-infra port: 6565 basePath: "" protocol: https diff --git a/test/integration/scripts/run-tests.sh b/test/integration/scripts/run-tests.sh index 7a50bd056..ecdd329ac 100644 --- a/test/integration/scripts/run-tests.sh +++ b/test/integration/scripts/run-tests.sh @@ -79,6 +79,7 @@ sudo echo "$IP ratelimit-priority.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP different-endpoint-with-same-route.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP custom-auth-header.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP gql.test.gw.wso2.com" | sudo tee -a /etc/hosts +sudo echo "$IP grpc.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP api-level-jwt.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP resource-level-jwt.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "255.255.255.255 broadcasthost" | sudo tee -a /etc/hosts diff --git a/test/integration/scripts/setup-hosts.sh b/test/integration/scripts/setup-hosts.sh index 7ff4ff8af..ec36022eb 100644 --- a/test/integration/scripts/setup-hosts.sh +++ b/test/integration/scripts/setup-hosts.sh @@ -38,6 +38,7 @@ sudo echo "$IP ratelimit-priority.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP different-endpoint-with-same-route.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP custom-auth-header.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP gql.test.gw.wso2.com" | sudo tee -a /etc/hosts +sudo echo "$IP grpc.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP api-level-jwt.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "$IP resource-level-jwt.test.gw.wso2.com" | sudo tee -a /etc/hosts sudo echo "255.255.255.255 broadcasthost" | sudo tee -a /etc/hosts From 19084f0ee083f45d922bf09dedc9438bde813b25 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 19 Mar 2024 09:31:59 +0530 Subject: [PATCH 35/52] add testcases and helpers --- .../integration/integration/tests/grpc-api.go | 75 ++++++++----------- .../integration/utils/grpcutils/helpers.go | 18 +++++ 2 files changed, 50 insertions(+), 43 deletions(-) create mode 100644 test/integration/integration/utils/grpcutils/helpers.go diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go index 8dcba85e1..5eb2bf734 100644 --- a/test/integration/integration/tests/grpc-api.go +++ b/test/integration/integration/tests/grpc-api.go @@ -21,8 +21,10 @@ import ( "context" "crypto/tls" "github.com/wso2/apk/test/integration/integration/utils/generatedcode/student" + "github.com/wso2/apk/test/integration/integration/utils/grpcutils" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "sigs.k8s.io/gateway-api/conformance/utils/config" "testing" "time" @@ -37,7 +39,7 @@ func init() { var GRPCAPI = suite.IntegrationTest{ ShortName: "GRPCAPI", Description: "Tests gRPC API", - //Manifests: []string{"tests/grpc-api.yaml"}, + Manifests: []string{"tests/grpc-api.yaml"}, Test: func(t *testing.T, suite *suite.IntegrationTestSuite) { gwAddr := "grpc.test.gw.wso2.com:9095" //token := http.GetTestToken(t) @@ -60,13 +62,18 @@ var GRPCAPI = suite.IntegrationTest{ // Response: http.Response{StatusCode: 200}, //}, //} - testCases := []ExpectedResponse{ + + testCases := []grpcutils.GRPCTestCase{ { - out: &student.StudentResponse{ - Name: "Dineth", - Age: 10, + ExpectedResponse: grpcutils.ExpectedResponse{ + Out: &student.StudentResponse{ + Name: "Dineth", + Age: 10, + }, + Err: nil, }, - err: nil, + ActualResponse: &student.StudentResponse{}, + Name: "Get Student Details", }, } for i := range testCases { @@ -76,17 +83,8 @@ var GRPCAPI = suite.IntegrationTest{ // http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, tc) //}) t.Run("Invoke gRPC API", func(t *testing.T) { - out, err := invokeGRPCClientUntilSatisfied(gwAddr, t) - if err != nil { - if tc.err != nil { - t.Errorf("Err -> \nWant: %q\nGot: %q\n", tc.err, err) - } - } else { - if tc.out.Name != out.Name || - tc.out.Age != out.Age { - t.Errorf("Out -> \nWant: %q\nGot : %q", tc.out, out) - } - } + t.Parallel() + invokeGRPCClientUntilSatisfied(gwAddr, t, tc, suite.TimeoutConfig) }) } @@ -97,8 +95,6 @@ func invokeGRPCClient(gwAddr string, t *testing.T) (*student.StudentResponse, er t.Logf("Starting gRPC client...") - // Set up TLS credentials for the connection without enforcing server certificate validation. - t.Logf("Setting up TLS credentials without server certificate validation...") config := &tls.Config{ InsecureSkipVerify: true, // CAUTION: This disables SSL certificate verification. } @@ -110,10 +106,10 @@ func invokeGRPCClient(gwAddr string, t *testing.T) (*student.StudentResponse, er t.Logf("Dialing to server at %s with timeout...", gwAddr) conn, err := grpc.DialContext(dialCtx, gwAddr, grpc.WithTransportCredentials(creds), grpc.WithBlock()) if err != nil { - t.Fatalf("Failed to connect: %v", err) + t.Logf("Failed to connect: %v", err) } defer conn.Close() - t.Log("Successfully connected to the server.") + //t.Log("Successfully connected to the server.") c := student.NewStudentServiceClient(conn) @@ -123,28 +119,23 @@ func invokeGRPCClient(gwAddr string, t *testing.T) (*student.StudentResponse, er t.Log("Sending request to the server...") // Create a StudentRequest message - r := &student.StudentRequest{Id: 1234} // Adjust the ID according to your actual implementation + r := &student.StudentRequest{Id: 1234} response, err := c.GetStudent(ctx, r) if err != nil { t.Logf("Could not fetch student: %v", err) } - t.Logf("Received response from server: %v\n", response) - t.Logf("Student Details: %v\n", response) return response, nil } -type ExpectedResponse struct { - out *student.StudentResponse - err error -} - -func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T) (*student.StudentResponse, error) { +func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase grpcutils.GRPCTestCase, timeout config.TimeoutConfig) { var out *student.StudentResponse var err error attempt := 0 maxAttempts := 4 - + expected := testCase.ExpectedResponse + //timeoutDuration := timeout.RequestTimeout * time.Second + timeoutDuration := 10 * time.Second for attempt < maxAttempts { t.Logf("Attempt %d to invoke gRPC client...", attempt+1) out, err = invokeGRPCClient(gwAddr, t) @@ -152,28 +143,26 @@ func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T) (*student.Stude if err != nil { t.Logf("Error on attempt %d: %v", attempt+1, err) } else { - // Check if the response is satisfactory. This condition needs to be defined. - // For example, assuming a satisfactory condition is when out.Satisfied is true. - // This is a placeholder condition and should be replaced with your actual success criteria. - if out != nil && isResponseSatisfactory(out) { + if out != nil && isResponseSatisfactory(out, expected) { t.Logf("Satisfactory response received: %+v", out) - return out, nil + return } } if attempt < maxAttempts-1 { - t.Logf("Waiting 20 seconds before next attempt...") - time.Sleep(20 * time.Second) + t.Logf("Waiting %s seconds before next attempt...", timeoutDuration) + time.Sleep(timeoutDuration) } attempt++ } t.Logf("Failed to receive a satisfactory response after %d attempts", maxAttempts) - return out, err // Returning the last response and error, might need adjustment based on requirements. + t.Fail() } -func isResponseSatisfactory(response *student.StudentResponse) bool { - // Define the condition for a response to be considered satisfactory. - // This is a placeholder function and should contain actual logic to evaluate the response. - return false // Placeholder: assume every response is satisfactory. +func isResponseSatisfactory(response *student.StudentResponse, expectedResponse grpcutils.ExpectedResponse) bool { + if response.Name == expectedResponse.Out.Name && response.Age == expectedResponse.Out.Age { + return true + } + return false } diff --git a/test/integration/integration/utils/grpcutils/helpers.go b/test/integration/integration/utils/grpcutils/helpers.go new file mode 100644 index 000000000..71ed6d92c --- /dev/null +++ b/test/integration/integration/utils/grpcutils/helpers.go @@ -0,0 +1,18 @@ +package grpcutils + +import "github.com/wso2/apk/test/integration/integration/utils/generatedcode/student" + +type ExpectedResponse struct { + Out *student.StudentResponse + Err error +} +type Request struct { + Host string + Headers map[string]string +} +type GRPCTestCase struct { + Request Request + ExpectedResponse ExpectedResponse + ActualResponse *student.StudentResponse + Name string +} From 16498abdd233a23a11b3045db960360bacb95aef Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 19 Mar 2024 09:32:18 +0530 Subject: [PATCH 36/52] add generated client code --- .../utils/generatedcode/student/student.pb.go | 254 ++++++++++++++ .../generatedcode/student/student_grpc.pb.go | 314 ++++++++++++++++++ 2 files changed, 568 insertions(+) create mode 100644 test/integration/integration/utils/generatedcode/student/student.pb.go create mode 100644 test/integration/integration/utils/generatedcode/student/student_grpc.pb.go diff --git a/test/integration/integration/utils/generatedcode/student/student.pb.go b/test/integration/integration/utils/generatedcode/student/student.pb.go new file mode 100644 index 000000000..c8efeffa3 --- /dev/null +++ b/test/integration/integration/utils/generatedcode/student/student.pb.go @@ -0,0 +1,254 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.12.4 +// source: student.proto + +package student + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type StudentRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int32 `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *StudentRequest) Reset() { + *x = StudentRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_student_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StudentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StudentRequest) ProtoMessage() {} + +func (x *StudentRequest) ProtoReflect() protoreflect.Message { + mi := &file_student_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StudentRequest.ProtoReflect.Descriptor instead. +func (*StudentRequest) Descriptor() ([]byte, []int) { + return file_student_proto_rawDescGZIP(), []int{0} +} + +func (x *StudentRequest) GetId() int32 { + if x != nil { + return x.Id + } + return 0 +} + +type StudentResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Age int32 `protobuf:"varint,2,opt,name=age,proto3" json:"age,omitempty"` +} + +func (x *StudentResponse) Reset() { + *x = StudentResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_student_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StudentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StudentResponse) ProtoMessage() {} + +func (x *StudentResponse) ProtoReflect() protoreflect.Message { + mi := &file_student_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StudentResponse.ProtoReflect.Descriptor instead. +func (*StudentResponse) Descriptor() ([]byte, []int) { + return file_student_proto_rawDescGZIP(), []int{1} +} + +func (x *StudentResponse) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *StudentResponse) GetAge() int32 { + if x != nil { + return x.Age + } + return 0 +} + +var File_student_proto protoreflect.FileDescriptor + +var file_student_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x1a, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x22, 0x20, 0x0a, 0x0e, 0x53, + 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x22, 0x37, 0x0a, + 0x0f, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x03, 0x61, 0x67, 0x65, 0x32, 0xd6, 0x03, 0x0a, 0x0e, 0x53, 0x74, 0x75, 0x64, 0x65, + 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x67, 0x0a, 0x0a, 0x47, 0x65, 0x74, + 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x12, 0x2a, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x74, 0x75, + 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, + 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x2a, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x74, 0x75, 0x64, + 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, + 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x30, 0x01, 0x12, 0x70, 0x0a, 0x11, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x75, 0x64, 0x65, + 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x2a, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, + 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x74, + 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, + 0x74, 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x78, 0x0a, 0x17, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x6e, 0x64, + 0x47, 0x65, 0x74, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x12, 0x2a, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, + 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x64, + 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, + 0x30, 0x5a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, + 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, + 0x65, 0x73, 0x2f, 0x73, 0x63, 0x68, 0x6f, 0x6f, 0x6c, 0x2f, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, + 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_student_proto_rawDescOnce sync.Once + file_student_proto_rawDescData = file_student_proto_rawDesc +) + +func file_student_proto_rawDescGZIP() []byte { + file_student_proto_rawDescOnce.Do(func() { + file_student_proto_rawDescData = protoimpl.X.CompressGZIP(file_student_proto_rawDescData) + }) + return file_student_proto_rawDescData +} + +var file_student_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_student_proto_goTypes = []interface{}{ + (*StudentRequest)(nil), // 0: dineth.grpc.api.v1.student.StudentRequest + (*StudentResponse)(nil), // 1: dineth.grpc.api.v1.student.StudentResponse +} +var file_student_proto_depIdxs = []int32{ + 0, // 0: dineth.grpc.api.v1.student.StudentService.GetStudent:input_type -> dineth.grpc.api.v1.student.StudentRequest + 0, // 1: dineth.grpc.api.v1.student.StudentService.GetStudentStream:input_type -> dineth.grpc.api.v1.student.StudentRequest + 0, // 2: dineth.grpc.api.v1.student.StudentService.SendStudentStream:input_type -> dineth.grpc.api.v1.student.StudentRequest + 0, // 3: dineth.grpc.api.v1.student.StudentService.SendAndGetStudentStream:input_type -> dineth.grpc.api.v1.student.StudentRequest + 1, // 4: dineth.grpc.api.v1.student.StudentService.GetStudent:output_type -> dineth.grpc.api.v1.student.StudentResponse + 1, // 5: dineth.grpc.api.v1.student.StudentService.GetStudentStream:output_type -> dineth.grpc.api.v1.student.StudentResponse + 1, // 6: dineth.grpc.api.v1.student.StudentService.SendStudentStream:output_type -> dineth.grpc.api.v1.student.StudentResponse + 1, // 7: dineth.grpc.api.v1.student.StudentService.SendAndGetStudentStream:output_type -> dineth.grpc.api.v1.student.StudentResponse + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_student_proto_init() } +func file_student_proto_init() { + if File_student_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_student_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StudentRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_student_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StudentResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_student_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_student_proto_goTypes, + DependencyIndexes: file_student_proto_depIdxs, + MessageInfos: file_student_proto_msgTypes, + }.Build() + File_student_proto = out.File + file_student_proto_rawDesc = nil + file_student_proto_goTypes = nil + file_student_proto_depIdxs = nil +} diff --git a/test/integration/integration/utils/generatedcode/student/student_grpc.pb.go b/test/integration/integration/utils/generatedcode/student/student_grpc.pb.go new file mode 100644 index 000000000..be4b29e7d --- /dev/null +++ b/test/integration/integration/utils/generatedcode/student/student_grpc.pb.go @@ -0,0 +1,314 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.12.4 +// source: student.proto + +package student + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + StudentService_GetStudent_FullMethodName = "/dineth.grpc.api.v1.student.StudentService/GetStudent" + StudentService_GetStudentStream_FullMethodName = "/dineth.grpc.api.v1.student.StudentService/GetStudentStream" + StudentService_SendStudentStream_FullMethodName = "/dineth.grpc.api.v1.student.StudentService/SendStudentStream" + StudentService_SendAndGetStudentStream_FullMethodName = "/dineth.grpc.api.v1.student.StudentService/SendAndGetStudentStream" +) + +// StudentServiceClient is the client API for StudentService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type StudentServiceClient interface { + GetStudent(ctx context.Context, in *StudentRequest, opts ...grpc.CallOption) (*StudentResponse, error) + GetStudentStream(ctx context.Context, in *StudentRequest, opts ...grpc.CallOption) (StudentService_GetStudentStreamClient, error) + SendStudentStream(ctx context.Context, opts ...grpc.CallOption) (StudentService_SendStudentStreamClient, error) + SendAndGetStudentStream(ctx context.Context, opts ...grpc.CallOption) (StudentService_SendAndGetStudentStreamClient, error) +} + +type studentServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewStudentServiceClient(cc grpc.ClientConnInterface) StudentServiceClient { + return &studentServiceClient{cc} +} + +func (c *studentServiceClient) GetStudent(ctx context.Context, in *StudentRequest, opts ...grpc.CallOption) (*StudentResponse, error) { + out := new(StudentResponse) + err := c.cc.Invoke(ctx, StudentService_GetStudent_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *studentServiceClient) GetStudentStream(ctx context.Context, in *StudentRequest, opts ...grpc.CallOption) (StudentService_GetStudentStreamClient, error) { + stream, err := c.cc.NewStream(ctx, &StudentService_ServiceDesc.Streams[0], StudentService_GetStudentStream_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &studentServiceGetStudentStreamClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type StudentService_GetStudentStreamClient interface { + Recv() (*StudentResponse, error) + grpc.ClientStream +} + +type studentServiceGetStudentStreamClient struct { + grpc.ClientStream +} + +func (x *studentServiceGetStudentStreamClient) Recv() (*StudentResponse, error) { + m := new(StudentResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *studentServiceClient) SendStudentStream(ctx context.Context, opts ...grpc.CallOption) (StudentService_SendStudentStreamClient, error) { + stream, err := c.cc.NewStream(ctx, &StudentService_ServiceDesc.Streams[1], StudentService_SendStudentStream_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &studentServiceSendStudentStreamClient{stream} + return x, nil +} + +type StudentService_SendStudentStreamClient interface { + Send(*StudentRequest) error + CloseAndRecv() (*StudentResponse, error) + grpc.ClientStream +} + +type studentServiceSendStudentStreamClient struct { + grpc.ClientStream +} + +func (x *studentServiceSendStudentStreamClient) Send(m *StudentRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *studentServiceSendStudentStreamClient) CloseAndRecv() (*StudentResponse, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(StudentResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *studentServiceClient) SendAndGetStudentStream(ctx context.Context, opts ...grpc.CallOption) (StudentService_SendAndGetStudentStreamClient, error) { + stream, err := c.cc.NewStream(ctx, &StudentService_ServiceDesc.Streams[2], StudentService_SendAndGetStudentStream_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &studentServiceSendAndGetStudentStreamClient{stream} + return x, nil +} + +type StudentService_SendAndGetStudentStreamClient interface { + Send(*StudentRequest) error + Recv() (*StudentResponse, error) + grpc.ClientStream +} + +type studentServiceSendAndGetStudentStreamClient struct { + grpc.ClientStream +} + +func (x *studentServiceSendAndGetStudentStreamClient) Send(m *StudentRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *studentServiceSendAndGetStudentStreamClient) Recv() (*StudentResponse, error) { + m := new(StudentResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// StudentServiceServer is the server API for StudentService service. +// All implementations must embed UnimplementedStudentServiceServer +// for forward compatibility +type StudentServiceServer interface { + GetStudent(context.Context, *StudentRequest) (*StudentResponse, error) + GetStudentStream(*StudentRequest, StudentService_GetStudentStreamServer) error + SendStudentStream(StudentService_SendStudentStreamServer) error + SendAndGetStudentStream(StudentService_SendAndGetStudentStreamServer) error + mustEmbedUnimplementedStudentServiceServer() +} + +// UnimplementedStudentServiceServer must be embedded to have forward compatible implementations. +type UnimplementedStudentServiceServer struct { +} + +func (UnimplementedStudentServiceServer) GetStudent(context.Context, *StudentRequest) (*StudentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetStudent not implemented") +} +func (UnimplementedStudentServiceServer) GetStudentStream(*StudentRequest, StudentService_GetStudentStreamServer) error { + return status.Errorf(codes.Unimplemented, "method GetStudentStream not implemented") +} +func (UnimplementedStudentServiceServer) SendStudentStream(StudentService_SendStudentStreamServer) error { + return status.Errorf(codes.Unimplemented, "method SendStudentStream not implemented") +} +func (UnimplementedStudentServiceServer) SendAndGetStudentStream(StudentService_SendAndGetStudentStreamServer) error { + return status.Errorf(codes.Unimplemented, "method SendAndGetStudentStream not implemented") +} +func (UnimplementedStudentServiceServer) mustEmbedUnimplementedStudentServiceServer() {} + +// UnsafeStudentServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to StudentServiceServer will +// result in compilation errors. +type UnsafeStudentServiceServer interface { + mustEmbedUnimplementedStudentServiceServer() +} + +func RegisterStudentServiceServer(s grpc.ServiceRegistrar, srv StudentServiceServer) { + s.RegisterService(&StudentService_ServiceDesc, srv) +} + +func _StudentService_GetStudent_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StudentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StudentServiceServer).GetStudent(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StudentService_GetStudent_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StudentServiceServer).GetStudent(ctx, req.(*StudentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StudentService_GetStudentStream_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(StudentRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(StudentServiceServer).GetStudentStream(m, &studentServiceGetStudentStreamServer{stream}) +} + +type StudentService_GetStudentStreamServer interface { + Send(*StudentResponse) error + grpc.ServerStream +} + +type studentServiceGetStudentStreamServer struct { + grpc.ServerStream +} + +func (x *studentServiceGetStudentStreamServer) Send(m *StudentResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _StudentService_SendStudentStream_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(StudentServiceServer).SendStudentStream(&studentServiceSendStudentStreamServer{stream}) +} + +type StudentService_SendStudentStreamServer interface { + SendAndClose(*StudentResponse) error + Recv() (*StudentRequest, error) + grpc.ServerStream +} + +type studentServiceSendStudentStreamServer struct { + grpc.ServerStream +} + +func (x *studentServiceSendStudentStreamServer) SendAndClose(m *StudentResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *studentServiceSendStudentStreamServer) Recv() (*StudentRequest, error) { + m := new(StudentRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _StudentService_SendAndGetStudentStream_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(StudentServiceServer).SendAndGetStudentStream(&studentServiceSendAndGetStudentStreamServer{stream}) +} + +type StudentService_SendAndGetStudentStreamServer interface { + Send(*StudentResponse) error + Recv() (*StudentRequest, error) + grpc.ServerStream +} + +type studentServiceSendAndGetStudentStreamServer struct { + grpc.ServerStream +} + +func (x *studentServiceSendAndGetStudentStreamServer) Send(m *StudentResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *studentServiceSendAndGetStudentStreamServer) Recv() (*StudentRequest, error) { + m := new(StudentRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// StudentService_ServiceDesc is the grpc.ServiceDesc for StudentService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var StudentService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "dineth.grpc.api.v1.student.StudentService", + HandlerType: (*StudentServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetStudent", + Handler: _StudentService_GetStudent_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "GetStudentStream", + Handler: _StudentService_GetStudentStream_Handler, + ServerStreams: true, + }, + { + StreamName: "SendStudentStream", + Handler: _StudentService_SendStudentStream_Handler, + ClientStreams: true, + }, + { + StreamName: "SendAndGetStudentStream", + Handler: _StudentService_SendAndGetStudentStream_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "student.proto", +} From dfe02b478e68cc387b1546d2e0ddb60e71ace808 Mon Sep 17 00:00:00 2001 From: DinethH Date: Thu, 21 Mar 2024 09:25:41 +0530 Subject: [PATCH 37/52] change test case timeout --- test/integration/integration/tests/grpc-api.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go index 5eb2bf734..49c1ce4d3 100644 --- a/test/integration/integration/tests/grpc-api.go +++ b/test/integration/integration/tests/grpc-api.go @@ -91,7 +91,7 @@ var GRPCAPI = suite.IntegrationTest{ }, } -func invokeGRPCClient(gwAddr string, t *testing.T) (*student.StudentResponse, error) { +func invokeGRPCClient(gwAddr string, t *testing.T, timeout config.TimeoutConfig) (*student.StudentResponse, error) { t.Logf("Starting gRPC client...") @@ -129,16 +129,18 @@ func invokeGRPCClient(gwAddr string, t *testing.T) (*student.StudentResponse, er } func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase grpcutils.GRPCTestCase, timeout config.TimeoutConfig) { + //(delay to allow CRs to be applied) + time.Sleep(5 * time.Second) var out *student.StudentResponse var err error attempt := 0 maxAttempts := 4 expected := testCase.ExpectedResponse //timeoutDuration := timeout.RequestTimeout * time.Second - timeoutDuration := 10 * time.Second + timeoutDuration := 50 * time.Second for attempt < maxAttempts { t.Logf("Attempt %d to invoke gRPC client...", attempt+1) - out, err = invokeGRPCClient(gwAddr, t) + out, err = invokeGRPCClient(gwAddr, t, timeout) if err != nil { t.Logf("Error on attempt %d: %v", attempt+1, err) From 6bb830dbe92da5ec47e523c3054629fa3a387995 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 26 Mar 2024 11:26:55 +0530 Subject: [PATCH 38/52] changed request timeout --- .../integration/integration/tests/grpc-api.go | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go index 49c1ce4d3..37c01f8e3 100644 --- a/test/integration/integration/tests/grpc-api.go +++ b/test/integration/integration/tests/grpc-api.go @@ -42,26 +42,6 @@ var GRPCAPI = suite.IntegrationTest{ Manifests: []string{"tests/grpc-api.yaml"}, Test: func(t *testing.T, suite *suite.IntegrationTestSuite) { gwAddr := "grpc.test.gw.wso2.com:9095" - //token := http.GetTestToken(t) - - //testCases := []http.ExpectedResponse{ - //{ - // Request: http.Request{ - // Host: "gql.test.gw.wso2.com", - // Path: "/gql/v1", - // Method: "POST", - // Headers: map[string]string{ - // "Content-Type": "application/json", - // }, - // Body: `{"query":"query{\n human(id:1000){\n id\n name\n }\n}","variables":{}}`, - // }, - // ExpectedRequest: &http.ExpectedRequest{ - // Request: http.Request{ - // Method: ""}, - // }, - // Response: http.Response{StatusCode: 200}, - //}, - //} testCases := []grpcutils.GRPCTestCase{ { @@ -78,10 +58,6 @@ var GRPCAPI = suite.IntegrationTest{ } for i := range testCases { tc := testCases[i] - //t.Run(tc.GetTestCaseName(i), func(t *testing.T) { - // t.Parallel() - // http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, tc) - //}) t.Run("Invoke gRPC API", func(t *testing.T) { t.Parallel() invokeGRPCClientUntilSatisfied(gwAddr, t, tc, suite.TimeoutConfig) @@ -96,12 +72,12 @@ func invokeGRPCClient(gwAddr string, t *testing.T, timeout config.TimeoutConfig) t.Logf("Starting gRPC client...") config := &tls.Config{ - InsecureSkipVerify: true, // CAUTION: This disables SSL certificate verification. + InsecureSkipVerify: true, } creds := credentials.NewTLS(config) // Dial the server with the TLS credentials and a dial timeout. - dialCtx, dialCancel := context.WithTimeout(context.Background(), 10*time.Second) + dialCtx, dialCancel := context.WithTimeout(context.Background(), 60*time.Second) defer dialCancel() t.Logf("Dialing to server at %s with timeout...", gwAddr) conn, err := grpc.DialContext(dialCtx, gwAddr, grpc.WithTransportCredentials(creds), grpc.WithBlock()) @@ -114,7 +90,7 @@ func invokeGRPCClient(gwAddr string, t *testing.T, timeout config.TimeoutConfig) c := student.NewStudentServiceClient(conn) // Prepare the context with a timeout for the request. - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), timeout.RequestTimeout) defer cancel() t.Log("Sending request to the server...") @@ -123,8 +99,9 @@ func invokeGRPCClient(gwAddr string, t *testing.T, timeout config.TimeoutConfig) response, err := c.GetStudent(ctx, r) if err != nil { t.Logf("Could not fetch student: %v", err) + t.Logf("Error: %v\n", response) } - t.Logf("Received response from server: %v\n", response) + return response, nil } @@ -136,7 +113,6 @@ func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase grpcut attempt := 0 maxAttempts := 4 expected := testCase.ExpectedResponse - //timeoutDuration := timeout.RequestTimeout * time.Second timeoutDuration := 50 * time.Second for attempt < maxAttempts { t.Logf("Attempt %d to invoke gRPC client...", attempt+1) From cc05715ef11d8bbc739f9b723d5c49990bfda6b3 Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 15 Apr 2024 14:27:02 +0530 Subject: [PATCH 39/52] fix after release --- .../operator/synchronizer/grpc_api.go | 87 ++++++++++--------- .../operator/synchronizer/synchronizer.go | 14 +-- 2 files changed, 48 insertions(+), 53 deletions(-) diff --git a/adapter/internal/operator/synchronizer/grpc_api.go b/adapter/internal/operator/synchronizer/grpc_api.go index 3c0671ed8..aadf86098 100644 --- a/adapter/internal/operator/synchronizer/grpc_api.go +++ b/adapter/internal/operator/synchronizer/grpc_api.go @@ -18,7 +18,6 @@ package synchronizer import ( "errors" - "fmt" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/wso2/apk/adapter/config" @@ -32,8 +31,36 @@ import ( gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) +// extract APIDetails from the GQLRoute +func updateInternalMapsFromGRPCRoute(apiState APIState, grpcRoute *GRPCRouteState, envType string) (*model.AdapterInternalAPI, map[string]struct{}, error) { + adapterInternalAPI, err := generateGRPCAdapterInternalAPI(apiState, grpcRoute, envType) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2632, logging.MAJOR, "Error generating AdapterInternalAPI for GRPCRoute: %v. %v", grpcRoute.GRPCRouteCombined.Name, err)) + return nil, nil, err + } + + vHosts := getVhostsForGRPCAPI(grpcRoute.GRPCRouteCombined) + labels := getLabelsForGRPCAPI(grpcRoute.GRPCRouteCombined) + listeners, relativeSectionNames := getListenersForGRPCAPI(grpcRoute.GRPCRouteCombined, adapterInternalAPI.UUID) + // We dont have a use case where a perticular API's two different grpc routes refer to two different gateway. Hence get the first listener name for the list for processing. + if len(listeners) == 0 || len(relativeSectionNames) == 0 { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2633, logging.MINOR, "Failed to find a matching listener for grpc route: %v. ", + grpcRoute.GRPCRouteCombined.Name)) + return nil, nil, errors.New("failed to find matching listener name for the provided grpc route") + } + listenerName := listeners[0] + sectionName := relativeSectionNames[0] + + if len(listeners) > 0 { + if err := xds.PopulateInternalMaps(adapterInternalAPI, labels, vHosts, sectionName, listenerName); err != nil { + return nil, nil, err + } + } + return adapterInternalAPI, labels, nil +} + // generateGRPCAdapterInternalAPI this will populate a AdapterInternalAPI representation for an GRPCRoute -func generateGRPCAdapterInternalAPI(apiState APIState, grpcRoute *GRPCRouteState, envType string) (*model.AdapterInternalAPI, map[string]struct{}, error) { +func generateGRPCAdapterInternalAPI(apiState APIState, grpcRoute *GRPCRouteState, envType string) (*model.AdapterInternalAPI, error) { var adapterInternalAPI model.AdapterInternalAPI adapterInternalAPI.SetIsDefaultVersion(apiState.APIDefinition.Spec.IsDefaultVersion) adapterInternalAPI.SetInfoAPICR(*apiState.APIDefinition) @@ -48,6 +75,7 @@ func generateGRPCAdapterInternalAPI(apiState APIState, grpcRoute *GRPCRouteState environment = conf.Adapter.Environment } adapterInternalAPI.SetEnvironment(environment) + adapterInternalAPI.SetXWso2RequestBodyPass(true) resourceParams := model.ResourceParams{ AuthSchemes: apiState.Authentications, @@ -63,60 +91,38 @@ func generateGRPCAdapterInternalAPI(apiState APIState, grpcRoute *GRPCRouteState } if err := adapterInternalAPI.SetInfoGRPCRouteCR(grpcRoute.GRPCRouteCombined, resourceParams); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2631, logging.MAJOR, "Error setting GRPCRoute CR info to adapterInternalAPI. %v", err)) - return nil, nil, err + return nil, err } - if err := adapterInternalAPI.Validate(); err != nil { - loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2632, logging.MAJOR, "Error validating adapterInternalAPI intermediate representation. %v", err)) - return nil, nil, err - } - vHosts := getVhostsForGRPCAPI(grpcRoute.GRPCRouteCombined) - labels := getLabelsForGRPCAPI(grpcRoute.GRPCRouteCombined) - listeners, relativeSectionNames := getListenersForGRPCAPI(grpcRoute.GRPCRouteCombined, adapterInternalAPI.UUID) - // We don't have a use case where a perticular API's two different grpc routes refer to two different gateway. Hence get the first listener name for the list for processing. - if len(listeners) == 0 || len(relativeSectionNames) == 0 { - loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2633, logging.MINOR, "Failed to find a matching listener for grpc route: %v. ", - grpcRoute.GRPCRouteCombined.Name)) - return nil, nil, errors.New("failed to find matching listener name for the provided grpc route") + if apiState.MutualSSL != nil && apiState.MutualSSL.Required != "" && !adapterInternalAPI.GetDisableAuthentications() { + adapterInternalAPI.SetDisableMtls(apiState.MutualSSL.Disabled) + adapterInternalAPI.SetMutualSSL(apiState.MutualSSL.Required) + adapterInternalAPI.SetClientCerts(apiState.APIDefinition.Name, apiState.MutualSSL.ClientCertificates) + } else { + adapterInternalAPI.SetDisableMtls(true) } - updatedLabelsMap := make(map[string]struct{}) - listenerName := listeners[0] - sectionName := relativeSectionNames[0] - if len(listeners) != 0 { - updatedLabels, err := xds.UpdateAPICache(vHosts, labels, listenerName, sectionName, adapterInternalAPI) - if err != nil { - loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2633, logging.MAJOR, "Error updating the API : %s:%s in vhosts: %s, API_UUID: %v. %v", - adapterInternalAPI.GetTitle(), adapterInternalAPI.GetVersion(), vHosts, adapterInternalAPI.UUID, err)) - return nil, nil, err - } - for newLabel := range updatedLabels { - updatedLabelsMap[newLabel] = struct{}{} - } - } + return &adapterInternalAPI, nil - return &adapterInternalAPI, updatedLabelsMap, nil } // getVhostForAPI returns the vHosts related to an API. -func getVhostsForGRPCAPI(grpcRoute *gwapiv1a2.GRPCRoute) []string { - var vHosts []string +func getVhostsForGRPCAPI(grpcRoute *gwapiv1a2.GRPCRoute) map[string]struct{} { + vHosts := make(map[string]struct{}) for _, hostName := range grpcRoute.Spec.Hostnames { - vHosts = append(vHosts, string(hostName)) + vHosts[string(hostName)] = struct{}{} } - fmt.Println("vhosts size: ", len(vHosts)) return vHosts } // getLabelsForAPI returns the labels related to an API. -func getLabelsForGRPCAPI(grpcRoute *gwapiv1a2.GRPCRoute) []string { - var labels []string - var err error +func getLabelsForGRPCAPI(grpcRoute *gwapiv1a2.GRPCRoute) map[string]struct{} { + labels := make(map[string]struct{}) for _, parentRef := range grpcRoute.Spec.ParentRefs { - err = xds.SanitizeGateway(string(parentRef.Name), false) + err := xds.SanitizeGateway(string(parentRef.Name), false) if err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2653, logging.CRITICAL, "Gateway Label is invalid: %s", string(parentRef.Name))) } else { - labels = append(labels, string(parentRef.Name)) + labels[string(parentRef.Name)] = struct{}{} } } return labels @@ -156,9 +162,8 @@ func getListenersForGRPCAPI(grpcRoute *gwapiv1a2.GRPCRoute, apiUUID string) ([]s func deleteGRPCAPIFromEnv(grpcRoute *gwapiv1a2.GRPCRoute, apiState APIState) error { labels := getLabelsForGRPCAPI(grpcRoute) - org := apiState.APIDefinition.Spec.Organization uuid := string(apiState.APIDefinition.ObjectMeta.UID) - return xds.DeleteAPICREvent(labels, uuid, org) + return xds.DeleteAPI(uuid, labels) } // undeployGRPCAPIInGateway undeploys the related API in CREATE and UPDATE events. diff --git a/adapter/internal/operator/synchronizer/synchronizer.go b/adapter/internal/operator/synchronizer/synchronizer.go index 34aeddfc1..0ce9bbea7 100644 --- a/adapter/internal/operator/synchronizer/synchronizer.go +++ b/adapter/internal/operator/synchronizer/synchronizer.go @@ -204,18 +204,8 @@ func deployMultipleAPIsInGateway(event *APIEvent, successChannel *chan SuccessEv } } if apiState.APIDefinition.Spec.APIType == "GRPC" { - if apiState.ProdGRPCRoute == nil { - var adapterInternalAPI model.AdapterInternalAPI - adapterInternalAPI.SetInfoAPICR(*apiState.APIDefinition) - xds.RemoveAPICacheForEnv(adapterInternalAPI, constants.Production) - } - if apiState.SandGRPCRoute == nil { - var adapterInternalAPI model.AdapterInternalAPI - adapterInternalAPI.SetInfoAPICR(*apiState.APIDefinition) - xds.RemoveAPICacheForEnv(adapterInternalAPI, constants.Sandbox) - } if apiState.ProdGRPCRoute != nil { - _, updatedLabels, err := generateGRPCAdapterInternalAPI(apiState, apiState.ProdGRPCRoute, constants.Production) + _, updatedLabels, err := updateInternalMapsFromGRPCRoute(apiState, apiState.ProdGRPCRoute, constants.Production) if err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2665, logging.CRITICAL, "Error deploying prod grpcRoute of API : %v in Organization %v from environments %v. Error: %v", @@ -228,7 +218,7 @@ func deployMultipleAPIsInGateway(event *APIEvent, successChannel *chan SuccessEv } } if apiState.SandGRPCRoute != nil { - _, updatedLabels, err := generateGRPCAdapterInternalAPI(apiState, apiState.SandGRPCRoute, constants.Sandbox) + _, updatedLabels, err := updateInternalMapsFromGRPCRoute(apiState, apiState.SandGRPCRoute, constants.Sandbox) if err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2665, logging.CRITICAL, "Error deploying sand grpcRoute of API : %v in Organization %v from environments %v. Error: %v", From 0170a21e893b0da5e9b9be7357aed47d57a94495 Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 15 Apr 2024 14:36:44 +0530 Subject: [PATCH 40/52] changed method to post --- adapter/internal/oasparser/model/adapter_internal_api.go | 2 +- .../org/wso2/apk/enforcer/commons/model/ResourceConfig.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 6f0ac2f82..486f28826 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -1078,7 +1078,7 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap resourcePath := adapterInternalAPI.GetXWso2Basepath() + "." + *match.Method.Service + "/" + *match.Method.Method endPoints = append(endPoints, GetEndpoints(backendName, resourceParams.BackendMapping)...) resource := &Resource{path: resourcePath, pathMatchType: "Exact", - methods: []*Operation{{iD: uuid.New().String(), method: string(*match.Method.Type), policies: policies, + methods: []*Operation{{iD: uuid.New().String(), method: "post", policies: policies, auth: apiAuth, rateLimitPolicy: parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes: scopes}}, iD: uuid.New().String(), } diff --git a/gateway/enforcer/org.wso2.apk.enforcer.commons/src/main/java/org/wso2/apk/enforcer/commons/model/ResourceConfig.java b/gateway/enforcer/org.wso2.apk.enforcer.commons/src/main/java/org/wso2/apk/enforcer/commons/model/ResourceConfig.java index 44428a76b..2d938a80e 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer.commons/src/main/java/org/wso2/apk/enforcer/commons/model/ResourceConfig.java +++ b/gateway/enforcer/org.wso2.apk.enforcer.commons/src/main/java/org/wso2/apk/enforcer/commons/model/ResourceConfig.java @@ -41,7 +41,7 @@ public class ResourceConfig { */ public enum HttpMethods { GET("get"), POST("post"), PUT("put"), DELETE("delete"), HEAD("head"), - PATCH("patch"), OPTIONS("options"), QUERY("query"), MUTATION("mutation"), EXACT("Exact"), + PATCH("patch"), OPTIONS("options"), QUERY("query"), MUTATION("mutation"), SUBSCRIPTION("subscription"); private String value; From 67fc8e32e06f35be024b869f7ef674527bad8c10 Mon Sep 17 00:00:00 2001 From: DinethH Date: Mon, 15 Apr 2024 14:43:50 +0530 Subject: [PATCH 41/52] fixed auth --- .../oasparser/model/adapter_internal_api.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 486f28826..09e120b79 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -1100,7 +1100,21 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap if authScheme.Spec.Override != nil && authScheme.Spec.Override.Disabled != nil { adapterInternalAPI.disableAuthentications = *authScheme.Spec.Override.Disabled } + authSpec := utils.SelectPolicy(&authScheme.Spec.Override, &authScheme.Spec.Default, nil, nil) + if authSpec != nil && authSpec.AuthTypes != nil && authSpec.AuthTypes.Oauth2.Required != "" { + adapterInternalAPI.SetXWSO2ApplicationSecurity(authSpec.AuthTypes.Oauth2.Required == "mandatory") + } else { + adapterInternalAPI.SetXWSO2ApplicationSecurity(true) + } adapterInternalAPI.disableScopes = disableScopes + // Check whether the API has a backend JWT token + if apiPolicy != nil && apiPolicy.Spec.Override != nil && apiPolicy.Spec.Override.BackendJWTPolicy != nil { + backendJWTPolicy := resourceParams.BackendJWTMapping[types.NamespacedName{ + Name: apiPolicy.Spec.Override.BackendJWTPolicy.Name, + Namespace: grpcRoute.Namespace, + }.String()].Spec + adapterInternalAPI.backendJWTTokenInfo = parseBackendJWTTokenToInternal(backendJWTPolicy) + } return nil } From d0a8a48ddedd12ced34fa49bd88b901e8f08bbbb Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 16 Apr 2024 10:08:30 +0530 Subject: [PATCH 42/52] change 2023 to 2024 --- .../integration/integration/tests/resources/tests/grpc-api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml index e8585acea..8c908830c 100644 --- a/test/integration/integration/tests/resources/tests/grpc-api.yaml +++ b/test/integration/integration/tests/resources/tests/grpc-api.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. # # WSO2 LLC. licenses this file to you under the Apache License, # Version 2.0 (the "License"); you may not use this file except From c03b37efc6c182e1211e1706022f2845907c2e68 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 16 Apr 2024 10:24:17 +0530 Subject: [PATCH 43/52] removed TODOs --- adapter/internal/oasparser/envoyconf/routes_with_clusters.go | 2 -- .../src/main/java/org/wso2/apk/enforcer/api/APIFactory.java | 1 - 2 files changed, 3 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 040882a29..19fc922c5 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -213,12 +213,10 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte clusters = append(clusters, cluster) endpoints = append(endpoints, address...) - //TODO factor out common code (Dineth) for _, resource := range adapterInternalAPI.GetResources() { var clusterName string resourcePath := resource.GetPath() endpoint := resource.GetEndpoints() - //TODO see if this should be configured elsewhere (Dineth) endpoint.HTTP2BackendEnabled = true basePath := strings.TrimSuffix(endpoint.Endpoints[0].Basepath, "/") existingClusterName := getExistingClusterName(*endpoint, processedEndpoints) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java index ea97c7f68..0b21aaf78 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/APIFactory.java @@ -139,7 +139,6 @@ public byte[] getAPIDefinition(final String basePath, final String version, fina public ResourceConfig getMatchedResource(API api, String matchedResourcePath, String method) { List resourceConfigList = api.getAPIConfig().getResources(); if (APIConstants.ApiType.GRPC.equals(api.getAPIConfig().getApiType())) { - //TODO check if there's another way to do this, perhaps in the adapter side by setting the method return resourceConfigList.stream() .filter(resourceConfig -> resourceConfig.getPath().equals(matchedResourcePath)) .findFirst().orElse(null); From cbdc119c9a95b22b50e837791d6688d7cd799b21 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 16 Apr 2024 11:23:06 +0530 Subject: [PATCH 44/52] change https to http in testcase --- adapter/internal/oasparser/envoyconf/routes_with_clusters.go | 2 -- .../integration/integration/tests/resources/tests/grpc-api.yaml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 19fc922c5..19b8ae1dd 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -224,8 +224,6 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte if existingClusterName == "" { clusterName = getClusterName(endpoint.EndpointPrefix, organizationID, vHost, adapterInternalAPI.GetTitle(), apiVersion, resource.GetID()) cluster, address, err := processEndpoints(clusterName, endpoint, timeout, basePath) - //TODO see if this should be configured elsewhere (Dineth) - cluster.TransportSocketMatches = nil if err != nil { logger.LoggerOasparser.ErrorC(logging.PrintError(logging.Error2239, logging.MAJOR, "Error while adding resource level endpoints for %s:%v-%v. %v", apiTitle, apiVersion, resourcePath, err.Error())) } else { diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml index 8c908830c..9209fb93a 100644 --- a/test/integration/integration/tests/resources/tests/grpc-api.yaml +++ b/test/integration/integration/tests/resources/tests/grpc-api.yaml @@ -106,4 +106,4 @@ spec: - host: grpc-backend-v1.gateway-integration-test-infra port: 6565 basePath: "" - protocol: https + protocol: http From 3e666720824afc28487a4e376d0750abe64c7021 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 16 Apr 2024 14:35:09 +0530 Subject: [PATCH 45/52] generalize testcase --- .../integration/integration/tests/grpc-api.go | 13 +++---------- .../integration/utils/grpcutils/helpers.go | 13 +++++++++---- .../integration/utils/grpcutils/student.go | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 test/integration/integration/utils/grpcutils/student.go diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go index 37c01f8e3..8f5ef2f38 100644 --- a/test/integration/integration/tests/grpc-api.go +++ b/test/integration/integration/tests/grpc-api.go @@ -60,7 +60,7 @@ var GRPCAPI = suite.IntegrationTest{ tc := testCases[i] t.Run("Invoke gRPC API", func(t *testing.T) { t.Parallel() - invokeGRPCClientUntilSatisfied(gwAddr, t, tc, suite.TimeoutConfig) + invokeGRPCClientUntilSatisfied(gwAddr, t, tc, suite.TimeoutConfig, grpcutils.StudentResponseSatisfier{}) }) } @@ -105,7 +105,7 @@ func invokeGRPCClient(gwAddr string, t *testing.T, timeout config.TimeoutConfig) return response, nil } -func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase grpcutils.GRPCTestCase, timeout config.TimeoutConfig) { +func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase grpcutils.GRPCTestCase, timeout config.TimeoutConfig, satisfier grpcutils.ResponseSatisfier) { //(delay to allow CRs to be applied) time.Sleep(5 * time.Second) var out *student.StudentResponse @@ -121,7 +121,7 @@ func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase grpcut if err != nil { t.Logf("Error on attempt %d: %v", attempt+1, err) } else { - if out != nil && isResponseSatisfactory(out, expected) { + if satisfier.IsSatisfactory(out, expected) { t.Logf("Satisfactory response received: %+v", out) return } @@ -137,10 +137,3 @@ func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase grpcut t.Logf("Failed to receive a satisfactory response after %d attempts", maxAttempts) t.Fail() } - -func isResponseSatisfactory(response *student.StudentResponse, expectedResponse grpcutils.ExpectedResponse) bool { - if response.Name == expectedResponse.Out.Name && response.Age == expectedResponse.Out.Age { - return true - } - return false -} diff --git a/test/integration/integration/utils/grpcutils/helpers.go b/test/integration/integration/utils/grpcutils/helpers.go index 71ed6d92c..a803f7952 100644 --- a/test/integration/integration/utils/grpcutils/helpers.go +++ b/test/integration/integration/utils/grpcutils/helpers.go @@ -2,17 +2,22 @@ package grpcutils import "github.com/wso2/apk/test/integration/integration/utils/generatedcode/student" -type ExpectedResponse struct { - Out *student.StudentResponse - Err error -} type Request struct { Host string Headers map[string]string } + +type ExpectedResponse struct { + Out *student.StudentResponse + Err error +} + type GRPCTestCase struct { Request Request ExpectedResponse ExpectedResponse ActualResponse *student.StudentResponse Name string } +type ResponseSatisfier interface { + IsSatisfactory(response interface{}, expectedResponse ExpectedResponse) bool +} diff --git a/test/integration/integration/utils/grpcutils/student.go b/test/integration/integration/utils/grpcutils/student.go new file mode 100644 index 000000000..e2b379979 --- /dev/null +++ b/test/integration/integration/utils/grpcutils/student.go @@ -0,0 +1,19 @@ +package grpcutils + +import "github.com/wso2/apk/test/integration/integration/utils/generatedcode/student" + +type StudentResponseSatisfier struct{} + +// IsSatisfactory checks if the given response is satisfactory based on the expected response. +func (srs StudentResponseSatisfier) IsSatisfactory(response interface{}, expectedResponse ExpectedResponse) bool { + // Type assert the response to *student.StudentResponse + resp, ok := response.(*student.StudentResponse) + if !ok { + return false // or panic, or handle the error according to your error handling policy + } + + if resp.Name == expectedResponse.Out.Name && resp.Age == expectedResponse.Out.Age { + return true + } + return false +} From 760761b5aa9554a7624b61f2f90ae347a0216e14 Mon Sep 17 00:00:00 2001 From: DinethH Date: Tue, 16 Apr 2024 15:24:47 +0530 Subject: [PATCH 46/52] generalize testcase a bit more generalize testcase a bit more generalize testcase a bit more generalize testcase a bit more generalize testcase a bit more generalize testcase a bit more generalize testcase a bit more --- .../integration/integration/tests/grpc-api.go | 86 +------------------ .../utils/grpc-code/student/student.go | 47 ++++++++++ .../student/student.pb.go | 0 .../student/student_grpc.pb.go | 0 .../integration/utils/grpcutils/helpers.go | 75 +++++++++++++++- .../integration/utils/grpcutils/student.go | 19 ---- 6 files changed, 122 insertions(+), 105 deletions(-) create mode 100644 test/integration/integration/utils/grpc-code/student/student.go rename test/integration/integration/utils/{generatedcode => grpc-code}/student/student.pb.go (100%) rename test/integration/integration/utils/{generatedcode => grpc-code}/student/student_grpc.pb.go (100%) delete mode 100644 test/integration/integration/utils/grpcutils/student.go diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go index 8f5ef2f38..354cbd1c8 100644 --- a/test/integration/integration/tests/grpc-api.go +++ b/test/integration/integration/tests/grpc-api.go @@ -18,17 +18,10 @@ package tests import ( - "context" - "crypto/tls" - "github.com/wso2/apk/test/integration/integration/utils/generatedcode/student" + "github.com/wso2/apk/test/integration/integration/utils/grpc-code/student" "github.com/wso2/apk/test/integration/integration/utils/grpcutils" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "sigs.k8s.io/gateway-api/conformance/utils/config" - "testing" - "time" - "github.com/wso2/apk/test/integration/integration/utils/suite" + "testing" ) func init() { @@ -54,86 +47,15 @@ var GRPCAPI = suite.IntegrationTest{ }, ActualResponse: &student.StudentResponse{}, Name: "Get Student Details", + Method: student.GetStudent, }, } for i := range testCases { tc := testCases[i] t.Run("Invoke gRPC API", func(t *testing.T) { t.Parallel() - invokeGRPCClientUntilSatisfied(gwAddr, t, tc, suite.TimeoutConfig, grpcutils.StudentResponseSatisfier{}) - + grpcutils.InvokeGRPCClientUntilSatisfied(gwAddr, t, tc, student.StudentResponseSatisfier{}, tc.Method) }) } }, } - -func invokeGRPCClient(gwAddr string, t *testing.T, timeout config.TimeoutConfig) (*student.StudentResponse, error) { - - t.Logf("Starting gRPC client...") - - config := &tls.Config{ - InsecureSkipVerify: true, - } - creds := credentials.NewTLS(config) - - // Dial the server with the TLS credentials and a dial timeout. - dialCtx, dialCancel := context.WithTimeout(context.Background(), 60*time.Second) - defer dialCancel() - t.Logf("Dialing to server at %s with timeout...", gwAddr) - conn, err := grpc.DialContext(dialCtx, gwAddr, grpc.WithTransportCredentials(creds), grpc.WithBlock()) - if err != nil { - t.Logf("Failed to connect: %v", err) - } - defer conn.Close() - //t.Log("Successfully connected to the server.") - - c := student.NewStudentServiceClient(conn) - - // Prepare the context with a timeout for the request. - ctx, cancel := context.WithTimeout(context.Background(), timeout.RequestTimeout) - defer cancel() - - t.Log("Sending request to the server...") - // Create a StudentRequest message - r := &student.StudentRequest{Id: 1234} - response, err := c.GetStudent(ctx, r) - if err != nil { - t.Logf("Could not fetch student: %v", err) - t.Logf("Error: %v\n", response) - } - - return response, nil -} - -func invokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase grpcutils.GRPCTestCase, timeout config.TimeoutConfig, satisfier grpcutils.ResponseSatisfier) { - //(delay to allow CRs to be applied) - time.Sleep(5 * time.Second) - var out *student.StudentResponse - var err error - attempt := 0 - maxAttempts := 4 - expected := testCase.ExpectedResponse - timeoutDuration := 50 * time.Second - for attempt < maxAttempts { - t.Logf("Attempt %d to invoke gRPC client...", attempt+1) - out, err = invokeGRPCClient(gwAddr, t, timeout) - - if err != nil { - t.Logf("Error on attempt %d: %v", attempt+1, err) - } else { - if satisfier.IsSatisfactory(out, expected) { - t.Logf("Satisfactory response received: %+v", out) - return - } - } - - if attempt < maxAttempts-1 { - t.Logf("Waiting %s seconds before next attempt...", timeoutDuration) - time.Sleep(timeoutDuration) - } - attempt++ - } - - t.Logf("Failed to receive a satisfactory response after %d attempts", maxAttempts) - t.Fail() -} diff --git a/test/integration/integration/utils/grpc-code/student/student.go b/test/integration/integration/utils/grpc-code/student/student.go new file mode 100644 index 000000000..52d4c3e6f --- /dev/null +++ b/test/integration/integration/utils/grpc-code/student/student.go @@ -0,0 +1,47 @@ +package student + +import ( + "context" + "github.com/wso2/apk/test/integration/integration/utils/grpcutils" + "google.golang.org/grpc" + "log" + "time" +) + +type StudentResponseSatisfier struct{} + +// IsSatisfactory checks if the given response is satisfactory based on the expected response. +func (srs StudentResponseSatisfier) IsSatisfactory(response any, expectedResponse grpcutils.ExpectedResponse) bool { + // Type assert the response to *student.StudentResponse + resp, respOk := response.(*StudentResponse) + if !respOk { + log.Println("Failed to assert response as *student.StudentResponse") + return false + } + // Type assert the expected output to *student.StudentResponse + expectedResp, expOk := expectedResponse.Out.(*StudentResponse) + if !expOk { + log.Println("Failed to assert expectedResponse.Out as *student.StudentResponse") + return false + } + + // Compare the actual response with the expected response + if resp.Name == expectedResp.Name && resp.Age == expectedResp.Age { + log.Println("Response is satisfactory") + return true + } else { + log.Println("Response does not match the expected output") + return false + } +} +func GetStudent(conn *grpc.ClientConn) (any, error) { + c := NewStudentServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + r := &StudentRequest{Id: 1234} + response, err := c.GetStudent(ctx, r) + if err != nil { + return nil, err + } + return response, nil +} diff --git a/test/integration/integration/utils/generatedcode/student/student.pb.go b/test/integration/integration/utils/grpc-code/student/student.pb.go similarity index 100% rename from test/integration/integration/utils/generatedcode/student/student.pb.go rename to test/integration/integration/utils/grpc-code/student/student.pb.go diff --git a/test/integration/integration/utils/generatedcode/student/student_grpc.pb.go b/test/integration/integration/utils/grpc-code/student/student_grpc.pb.go similarity index 100% rename from test/integration/integration/utils/generatedcode/student/student_grpc.pb.go rename to test/integration/integration/utils/grpc-code/student/student_grpc.pb.go diff --git a/test/integration/integration/utils/grpcutils/helpers.go b/test/integration/integration/utils/grpcutils/helpers.go index a803f7952..197bcfac0 100644 --- a/test/integration/integration/utils/grpcutils/helpers.go +++ b/test/integration/integration/utils/grpcutils/helpers.go @@ -1,23 +1,90 @@ package grpcutils -import "github.com/wso2/apk/test/integration/integration/utils/generatedcode/student" +import ( + "crypto/tls" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "testing" + "time" +) type Request struct { Host string Headers map[string]string } - +type ClientCreator[T any] func(conn *grpc.ClientConn) T type ExpectedResponse struct { - Out *student.StudentResponse + Out any Err error } type GRPCTestCase struct { Request Request ExpectedResponse ExpectedResponse - ActualResponse *student.StudentResponse + ActualResponse any Name string + Method func(conn *grpc.ClientConn) (any, error) } type ResponseSatisfier interface { IsSatisfactory(response interface{}, expectedResponse ExpectedResponse) bool } + +func DialGRPCServer(gwAddr string, t *testing.T) (*grpc.ClientConn, error) { + // Set up a connection to the server. + t.Logf("Dialing gRPC server at %s...", gwAddr) + creds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) + conn, err := grpc.Dial(gwAddr, grpc.WithTransportCredentials(creds)) + if err != nil { + t.Fatalf("Could not connect to the server: %v", err) + } + return conn, nil +} +func InvokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase GRPCTestCase, satisfier ResponseSatisfier, fn ExecuteClientCall) { + //(delay to allow CRs to be applied) + time.Sleep(5 * time.Second) + + var out any + var err error + attempt := 0 + maxAttempts := 4 + expected := testCase.ExpectedResponse + timeoutDuration := 50 * time.Second + for attempt < maxAttempts { + t.Logf("Attempt %d to invoke gRPC client...", attempt+1) + out, err = InvokeGRPCClient(gwAddr, t, fn) + + if err != nil { + t.Logf("Error on attempt %d: %v", attempt+1, err) + } else { + if satisfier.IsSatisfactory(out, expected) { + t.Logf("Satisfactory response received: %+v", out) + return + } + } + + if attempt < maxAttempts-1 { + t.Logf("Waiting %s seconds before next attempt...", timeoutDuration) + time.Sleep(timeoutDuration) + } + attempt++ + } + + t.Logf("Failed to receive a satisfactory response after %d attempts", maxAttempts) + t.Fail() +} + +type ExecuteClientCall func(conn *grpc.ClientConn) (any, error) + +func InvokeGRPCClient(gwAddr string, t *testing.T, fn ExecuteClientCall) (any, error) { + + conn, err := DialGRPCServer(gwAddr, t) + if err != nil { + t.Fatalf("Could not connect to the server: %v", err) + } + + response, err := fn(conn) + if err != nil { + return nil, err + } + return response, nil +} diff --git a/test/integration/integration/utils/grpcutils/student.go b/test/integration/integration/utils/grpcutils/student.go deleted file mode 100644 index e2b379979..000000000 --- a/test/integration/integration/utils/grpcutils/student.go +++ /dev/null @@ -1,19 +0,0 @@ -package grpcutils - -import "github.com/wso2/apk/test/integration/integration/utils/generatedcode/student" - -type StudentResponseSatisfier struct{} - -// IsSatisfactory checks if the given response is satisfactory based on the expected response. -func (srs StudentResponseSatisfier) IsSatisfactory(response interface{}, expectedResponse ExpectedResponse) bool { - // Type assert the response to *student.StudentResponse - resp, ok := response.(*student.StudentResponse) - if !ok { - return false // or panic, or handle the error according to your error handling policy - } - - if resp.Name == expectedResponse.Out.Name && resp.Age == expectedResponse.Out.Age { - return true - } - return false -} From 271ff0f05121bef4fadfeb06e1a1e42c6ec53b60 Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 17 Apr 2024 10:06:44 +0530 Subject: [PATCH 47/52] add testcase for grpc api default version --- .../integration/integration/tests/grpc-api.go | 12 + .../tests/resources/tests/grpc-api.yaml | 2 +- .../utils/grpc-code/student/student.go | 1 - .../student_default_version/student2.go | 46 +++ .../student_default_version.pb.go | 251 ++++++++++++++ .../student_default_version_grpc.pb.go | 314 ++++++++++++++++++ .../integration/utils/grpcutils/helpers.go | 1 - 7 files changed, 624 insertions(+), 3 deletions(-) create mode 100644 test/integration/integration/utils/grpc-code/student_default_version/student2.go create mode 100644 test/integration/integration/utils/grpc-code/student_default_version/student_default_version.pb.go create mode 100644 test/integration/integration/utils/grpc-code/student_default_version/student_default_version_grpc.pb.go diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go index 354cbd1c8..761e5fcab 100644 --- a/test/integration/integration/tests/grpc-api.go +++ b/test/integration/integration/tests/grpc-api.go @@ -49,6 +49,18 @@ var GRPCAPI = suite.IntegrationTest{ Name: "Get Student Details", Method: student.GetStudent, }, + //{ + // ExpectedResponse: grpcutils.ExpectedResponse{ + // Out: &student_default_version.StudentResponse{ + // Name: "Dineth", + // Age: 10, + // }, + // Err: nil, + // }, + // ActualResponse: &student_default_version.StudentResponse{}, + // Name: "Get Student Details (Default API Version)", + // Method: student_default_version.GetStudent, + //}, } for i := range testCases { tc := testCases[i] diff --git a/test/integration/integration/tests/resources/tests/grpc-api.yaml b/test/integration/integration/tests/resources/tests/grpc-api.yaml index 9209fb93a..6329c8777 100644 --- a/test/integration/integration/tests/resources/tests/grpc-api.yaml +++ b/test/integration/integration/tests/resources/tests/grpc-api.yaml @@ -24,7 +24,7 @@ spec: apiType: GRPC apiVersion: v1 basePath: /dineth.grpc.api.v1 - isDefaultVersion: false + isDefaultVersion: true production: - routeRefs: - grpc-api-grpcroute diff --git a/test/integration/integration/utils/grpc-code/student/student.go b/test/integration/integration/utils/grpc-code/student/student.go index 52d4c3e6f..36143eb56 100644 --- a/test/integration/integration/utils/grpc-code/student/student.go +++ b/test/integration/integration/utils/grpc-code/student/student.go @@ -27,7 +27,6 @@ func (srs StudentResponseSatisfier) IsSatisfactory(response any, expectedRespons // Compare the actual response with the expected response if resp.Name == expectedResp.Name && resp.Age == expectedResp.Age { - log.Println("Response is satisfactory") return true } else { log.Println("Response does not match the expected output") diff --git a/test/integration/integration/utils/grpc-code/student_default_version/student2.go b/test/integration/integration/utils/grpc-code/student_default_version/student2.go new file mode 100644 index 000000000..939bb3b90 --- /dev/null +++ b/test/integration/integration/utils/grpc-code/student_default_version/student2.go @@ -0,0 +1,46 @@ +package student_default_version + +import ( + "context" + "github.com/wso2/apk/test/integration/integration/utils/grpcutils" + "google.golang.org/grpc" + "log" + "time" +) + +type StudentResponseSatisfier struct{} + +// IsSatisfactory checks if the given response is satisfactory based on the expected response. +func (srs StudentResponseSatisfier) IsSatisfactory(response any, expectedResponse grpcutils.ExpectedResponse) bool { + // Type assert the response to *student.StudentResponse + resp, respOk := response.(*StudentResponse) + if !respOk { + log.Println("Failed to assert response as *student.StudentResponse") + return false + } + // Type assert the expected output to *student.StudentResponse + expectedResp, expOk := expectedResponse.Out.(*StudentResponse) + if !expOk { + log.Println("Failed to assert expectedResponse.Out as *student.StudentResponse") + return false + } + + // Compare the actual response with the expected response + if resp.Name == expectedResp.Name && resp.Age == expectedResp.Age { + return true + } else { + log.Println("Response does not match the expected output") + return false + } +} +func GetStudent(conn *grpc.ClientConn) (any, error) { + c := NewStudentServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + r := &StudentRequest{Id: 1234} + response, err := c.GetStudent(ctx, r) + if err != nil { + return nil, err + } + return response, nil +} diff --git a/test/integration/integration/utils/grpc-code/student_default_version/student_default_version.pb.go b/test/integration/integration/utils/grpc-code/student_default_version/student_default_version.pb.go new file mode 100644 index 000000000..a006069e8 --- /dev/null +++ b/test/integration/integration/utils/grpc-code/student_default_version/student_default_version.pb.go @@ -0,0 +1,251 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.12.4 +// source: student_default_version.proto + +package student_default_version + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type StudentRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int32 `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *StudentRequest) Reset() { + *x = StudentRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_student_default_version_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StudentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StudentRequest) ProtoMessage() {} + +func (x *StudentRequest) ProtoReflect() protoreflect.Message { + mi := &file_student_default_version_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StudentRequest.ProtoReflect.Descriptor instead. +func (*StudentRequest) Descriptor() ([]byte, []int) { + return file_student_default_version_proto_rawDescGZIP(), []int{0} +} + +func (x *StudentRequest) GetId() int32 { + if x != nil { + return x.Id + } + return 0 +} + +type StudentResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Age int32 `protobuf:"varint,2,opt,name=age,proto3" json:"age,omitempty"` +} + +func (x *StudentResponse) Reset() { + *x = StudentResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_student_default_version_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StudentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StudentResponse) ProtoMessage() {} + +func (x *StudentResponse) ProtoReflect() protoreflect.Message { + mi := &file_student_default_version_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StudentResponse.ProtoReflect.Descriptor instead. +func (*StudentResponse) Descriptor() ([]byte, []int) { + return file_student_default_version_proto_rawDescGZIP(), []int{1} +} + +func (x *StudentResponse) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *StudentResponse) GetAge() int32 { + if x != nil { + return x.Age + } + return 0 +} + +var File_student_default_version_proto protoreflect.FileDescriptor + +var file_student_default_version_proto_rawDesc = []byte{ + 0x0a, 0x1d, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x17, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x22, 0x20, 0x0a, 0x0e, 0x53, 0x74, 0x75, 0x64, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x22, 0x37, 0x0a, 0x0f, 0x53, 0x74, + 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, + 0x61, 0x67, 0x65, 0x32, 0xbe, 0x03, 0x0a, 0x0e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x61, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, 0x74, 0x75, + 0x64, 0x65, 0x6e, 0x74, 0x12, 0x27, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, + 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, + 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x69, 0x0a, 0x10, 0x47, 0x65, 0x74, + 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x27, 0x2e, + 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, + 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x30, 0x01, 0x12, 0x6a, 0x0a, 0x11, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x75, 0x64, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x27, 0x2e, 0x64, 0x69, 0x6e, 0x65, + 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x74, 0x75, 0x64, + 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, 0x75, + 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, + 0x12, 0x72, 0x0a, 0x17, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x6e, 0x64, 0x47, 0x65, 0x74, 0x53, 0x74, + 0x75, 0x64, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x27, 0x2e, 0x64, 0x69, + 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x74, + 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x64, 0x69, 0x6e, 0x65, 0x74, 0x68, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x2e, 0x53, + 0x74, 0x75, 0x64, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x28, 0x01, 0x30, 0x01, 0x42, 0x0f, 0x0a, 0x0b, 0x6f, 0x72, 0x67, 0x2e, 0x65, 0x78, 0x61, 0x6d, + 0x70, 0x6c, 0x65, 0x50, 0x01, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_student_default_version_proto_rawDescOnce sync.Once + file_student_default_version_proto_rawDescData = file_student_default_version_proto_rawDesc +) + +func file_student_default_version_proto_rawDescGZIP() []byte { + file_student_default_version_proto_rawDescOnce.Do(func() { + file_student_default_version_proto_rawDescData = protoimpl.X.CompressGZIP(file_student_default_version_proto_rawDescData) + }) + return file_student_default_version_proto_rawDescData +} + +var file_student_default_version_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_student_default_version_proto_goTypes = []interface{}{ + (*StudentRequest)(nil), // 0: dineth.grpc.api.student.StudentRequest + (*StudentResponse)(nil), // 1: dineth.grpc.api.student.StudentResponse +} +var file_student_default_version_proto_depIdxs = []int32{ + 0, // 0: dineth.grpc.api.student.StudentService.GetStudent:input_type -> dineth.grpc.api.student.StudentRequest + 0, // 1: dineth.grpc.api.student.StudentService.GetStudentStream:input_type -> dineth.grpc.api.student.StudentRequest + 0, // 2: dineth.grpc.api.student.StudentService.SendStudentStream:input_type -> dineth.grpc.api.student.StudentRequest + 0, // 3: dineth.grpc.api.student.StudentService.SendAndGetStudentStream:input_type -> dineth.grpc.api.student.StudentRequest + 1, // 4: dineth.grpc.api.student.StudentService.GetStudent:output_type -> dineth.grpc.api.student.StudentResponse + 1, // 5: dineth.grpc.api.student.StudentService.GetStudentStream:output_type -> dineth.grpc.api.student.StudentResponse + 1, // 6: dineth.grpc.api.student.StudentService.SendStudentStream:output_type -> dineth.grpc.api.student.StudentResponse + 1, // 7: dineth.grpc.api.student.StudentService.SendAndGetStudentStream:output_type -> dineth.grpc.api.student.StudentResponse + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_student_default_version_proto_init() } +func file_student_default_version_proto_init() { + if File_student_default_version_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_student_default_version_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StudentRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_student_default_version_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StudentResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_student_default_version_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_student_default_version_proto_goTypes, + DependencyIndexes: file_student_default_version_proto_depIdxs, + MessageInfos: file_student_default_version_proto_msgTypes, + }.Build() + File_student_default_version_proto = out.File + file_student_default_version_proto_rawDesc = nil + file_student_default_version_proto_goTypes = nil + file_student_default_version_proto_depIdxs = nil +} diff --git a/test/integration/integration/utils/grpc-code/student_default_version/student_default_version_grpc.pb.go b/test/integration/integration/utils/grpc-code/student_default_version/student_default_version_grpc.pb.go new file mode 100644 index 000000000..b3fe9b74b --- /dev/null +++ b/test/integration/integration/utils/grpc-code/student_default_version/student_default_version_grpc.pb.go @@ -0,0 +1,314 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.12.4 +// source: student_default_version.proto + +package student_default_version + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + StudentService_GetStudent_FullMethodName = "/dineth.grpc.api.student.StudentService/GetStudent" + StudentService_GetStudentStream_FullMethodName = "/dineth.grpc.api.student.StudentService/GetStudentStream" + StudentService_SendStudentStream_FullMethodName = "/dineth.grpc.api.student.StudentService/SendStudentStream" + StudentService_SendAndGetStudentStream_FullMethodName = "/dineth.grpc.api.student.StudentService/SendAndGetStudentStream" +) + +// StudentServiceClient is the client API for StudentService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type StudentServiceClient interface { + GetStudent(ctx context.Context, in *StudentRequest, opts ...grpc.CallOption) (*StudentResponse, error) + GetStudentStream(ctx context.Context, in *StudentRequest, opts ...grpc.CallOption) (StudentService_GetStudentStreamClient, error) + SendStudentStream(ctx context.Context, opts ...grpc.CallOption) (StudentService_SendStudentStreamClient, error) + SendAndGetStudentStream(ctx context.Context, opts ...grpc.CallOption) (StudentService_SendAndGetStudentStreamClient, error) +} + +type studentServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewStudentServiceClient(cc grpc.ClientConnInterface) StudentServiceClient { + return &studentServiceClient{cc} +} + +func (c *studentServiceClient) GetStudent(ctx context.Context, in *StudentRequest, opts ...grpc.CallOption) (*StudentResponse, error) { + out := new(StudentResponse) + err := c.cc.Invoke(ctx, StudentService_GetStudent_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *studentServiceClient) GetStudentStream(ctx context.Context, in *StudentRequest, opts ...grpc.CallOption) (StudentService_GetStudentStreamClient, error) { + stream, err := c.cc.NewStream(ctx, &StudentService_ServiceDesc.Streams[0], StudentService_GetStudentStream_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &studentServiceGetStudentStreamClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type StudentService_GetStudentStreamClient interface { + Recv() (*StudentResponse, error) + grpc.ClientStream +} + +type studentServiceGetStudentStreamClient struct { + grpc.ClientStream +} + +func (x *studentServiceGetStudentStreamClient) Recv() (*StudentResponse, error) { + m := new(StudentResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *studentServiceClient) SendStudentStream(ctx context.Context, opts ...grpc.CallOption) (StudentService_SendStudentStreamClient, error) { + stream, err := c.cc.NewStream(ctx, &StudentService_ServiceDesc.Streams[1], StudentService_SendStudentStream_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &studentServiceSendStudentStreamClient{stream} + return x, nil +} + +type StudentService_SendStudentStreamClient interface { + Send(*StudentRequest) error + CloseAndRecv() (*StudentResponse, error) + grpc.ClientStream +} + +type studentServiceSendStudentStreamClient struct { + grpc.ClientStream +} + +func (x *studentServiceSendStudentStreamClient) Send(m *StudentRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *studentServiceSendStudentStreamClient) CloseAndRecv() (*StudentResponse, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(StudentResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *studentServiceClient) SendAndGetStudentStream(ctx context.Context, opts ...grpc.CallOption) (StudentService_SendAndGetStudentStreamClient, error) { + stream, err := c.cc.NewStream(ctx, &StudentService_ServiceDesc.Streams[2], StudentService_SendAndGetStudentStream_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &studentServiceSendAndGetStudentStreamClient{stream} + return x, nil +} + +type StudentService_SendAndGetStudentStreamClient interface { + Send(*StudentRequest) error + Recv() (*StudentResponse, error) + grpc.ClientStream +} + +type studentServiceSendAndGetStudentStreamClient struct { + grpc.ClientStream +} + +func (x *studentServiceSendAndGetStudentStreamClient) Send(m *StudentRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *studentServiceSendAndGetStudentStreamClient) Recv() (*StudentResponse, error) { + m := new(StudentResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// StudentServiceServer is the server API for StudentService service. +// All implementations must embed UnimplementedStudentServiceServer +// for forward compatibility +type StudentServiceServer interface { + GetStudent(context.Context, *StudentRequest) (*StudentResponse, error) + GetStudentStream(*StudentRequest, StudentService_GetStudentStreamServer) error + SendStudentStream(StudentService_SendStudentStreamServer) error + SendAndGetStudentStream(StudentService_SendAndGetStudentStreamServer) error + mustEmbedUnimplementedStudentServiceServer() +} + +// UnimplementedStudentServiceServer must be embedded to have forward compatible implementations. +type UnimplementedStudentServiceServer struct { +} + +func (UnimplementedStudentServiceServer) GetStudent(context.Context, *StudentRequest) (*StudentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetStudent not implemented") +} +func (UnimplementedStudentServiceServer) GetStudentStream(*StudentRequest, StudentService_GetStudentStreamServer) error { + return status.Errorf(codes.Unimplemented, "method GetStudentStream not implemented") +} +func (UnimplementedStudentServiceServer) SendStudentStream(StudentService_SendStudentStreamServer) error { + return status.Errorf(codes.Unimplemented, "method SendStudentStream not implemented") +} +func (UnimplementedStudentServiceServer) SendAndGetStudentStream(StudentService_SendAndGetStudentStreamServer) error { + return status.Errorf(codes.Unimplemented, "method SendAndGetStudentStream not implemented") +} +func (UnimplementedStudentServiceServer) mustEmbedUnimplementedStudentServiceServer() {} + +// UnsafeStudentServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to StudentServiceServer will +// result in compilation errors. +type UnsafeStudentServiceServer interface { + mustEmbedUnimplementedStudentServiceServer() +} + +func RegisterStudentServiceServer(s grpc.ServiceRegistrar, srv StudentServiceServer) { + s.RegisterService(&StudentService_ServiceDesc, srv) +} + +func _StudentService_GetStudent_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StudentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StudentServiceServer).GetStudent(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StudentService_GetStudent_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StudentServiceServer).GetStudent(ctx, req.(*StudentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StudentService_GetStudentStream_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(StudentRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(StudentServiceServer).GetStudentStream(m, &studentServiceGetStudentStreamServer{stream}) +} + +type StudentService_GetStudentStreamServer interface { + Send(*StudentResponse) error + grpc.ServerStream +} + +type studentServiceGetStudentStreamServer struct { + grpc.ServerStream +} + +func (x *studentServiceGetStudentStreamServer) Send(m *StudentResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _StudentService_SendStudentStream_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(StudentServiceServer).SendStudentStream(&studentServiceSendStudentStreamServer{stream}) +} + +type StudentService_SendStudentStreamServer interface { + SendAndClose(*StudentResponse) error + Recv() (*StudentRequest, error) + grpc.ServerStream +} + +type studentServiceSendStudentStreamServer struct { + grpc.ServerStream +} + +func (x *studentServiceSendStudentStreamServer) SendAndClose(m *StudentResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *studentServiceSendStudentStreamServer) Recv() (*StudentRequest, error) { + m := new(StudentRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _StudentService_SendAndGetStudentStream_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(StudentServiceServer).SendAndGetStudentStream(&studentServiceSendAndGetStudentStreamServer{stream}) +} + +type StudentService_SendAndGetStudentStreamServer interface { + Send(*StudentResponse) error + Recv() (*StudentRequest, error) + grpc.ServerStream +} + +type studentServiceSendAndGetStudentStreamServer struct { + grpc.ServerStream +} + +func (x *studentServiceSendAndGetStudentStreamServer) Send(m *StudentResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *studentServiceSendAndGetStudentStreamServer) Recv() (*StudentRequest, error) { + m := new(StudentRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// StudentService_ServiceDesc is the grpc.ServiceDesc for StudentService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var StudentService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "dineth.grpc.api.student.StudentService", + HandlerType: (*StudentServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetStudent", + Handler: _StudentService_GetStudent_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "GetStudentStream", + Handler: _StudentService_GetStudentStream_Handler, + ServerStreams: true, + }, + { + StreamName: "SendStudentStream", + Handler: _StudentService_SendStudentStream_Handler, + ClientStreams: true, + }, + { + StreamName: "SendAndGetStudentStream", + Handler: _StudentService_SendAndGetStudentStream_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "student_default_version.proto", +} diff --git a/test/integration/integration/utils/grpcutils/helpers.go b/test/integration/integration/utils/grpcutils/helpers.go index 197bcfac0..d2158aea5 100644 --- a/test/integration/integration/utils/grpcutils/helpers.go +++ b/test/integration/integration/utils/grpcutils/helpers.go @@ -57,7 +57,6 @@ func InvokeGRPCClientUntilSatisfied(gwAddr string, t *testing.T, testCase GRPCTe t.Logf("Error on attempt %d: %v", attempt+1, err) } else { if satisfier.IsSatisfactory(out, expected) { - t.Logf("Satisfactory response received: %+v", out) return } } From 1d7aea95b74b75c80edab60c72ad1ba4a847ab53 Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 17 Apr 2024 13:11:41 +0530 Subject: [PATCH 48/52] add default version capability for grpc api --- .../envoyconf/routes_with_clusters.go | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 19b8ae1dd..172917eb6 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -928,8 +928,14 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error decorator *routev3.Decorator ) if params.createDefaultPath { - xWso2Basepath = removeFirstOccurrence(xWso2Basepath, "/"+version) - resourcePath = removeFirstOccurrence(resource.GetPath(), "/"+version) + //check if basepath is separated from version by a . or / + if strings.Contains(basePath, "."+version) { + xWso2Basepath = removeFirstOccurrence(basePath, "."+version) + resourcePath = removeFirstOccurrence(resource.GetPath(), "."+version) + } else { + xWso2Basepath = removeFirstOccurrence(xWso2Basepath, "/"+version) + resourcePath = removeFirstOccurrence(resource.GetPath(), "/"+version) + } } if pathMatchType != gwapiv1.PathMatchExact { @@ -1129,6 +1135,10 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error if apiType == "GRPC" { match.Headers = nil newRoutePath := "/" + strings.TrimPrefix(resourcePath, basePath+".") + if newRoutePath == "/"+resourcePath { + temp := removeFirstOccurrence(basePath, "."+version) + newRoutePath = "/" + strings.TrimPrefix(resourcePath, temp+".") + } action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, newRoutePath, pathMatchType) } @@ -1283,8 +1293,14 @@ func CreateAPIDefinitionEndpoint(adapterInternalAPI *model.AdapterInternalAPI, v matchPath := basePath + endpoint if isDefaultversion { - basePathWithoutVersion := removeLastOccurrence(basePath, "/"+version) - matchPath = basePathWithoutVersion + endpoint + if adapterInternalAPI.GetAPIType() == "GRPC" { + basePathWithoutVersion := removeLastOccurrence(basePath, "."+version) + matchPath = basePathWithoutVersion + "/" + vHost + endpoint + } else { + basePathWithoutVersion := removeLastOccurrence(basePath, "/"+version) + matchPath = basePathWithoutVersion + endpoint + + } } matchPath = strings.Replace(matchPath, basePath, regexp.QuoteMeta(basePath), 1) From f5e918529f202891ef9b4f158a3abf738add5445 Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 17 Apr 2024 13:21:50 +0530 Subject: [PATCH 49/52] modify test for grpc default version testing --- .../integration/integration/tests/grpc-api.go | 29 ++++++++++--------- .../integration/utils/grpcutils/helpers.go | 1 + 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go index 761e5fcab..054bda3f3 100644 --- a/test/integration/integration/tests/grpc-api.go +++ b/test/integration/integration/tests/grpc-api.go @@ -19,6 +19,7 @@ package tests import ( "github.com/wso2/apk/test/integration/integration/utils/grpc-code/student" + "github.com/wso2/apk/test/integration/integration/utils/grpc-code/student_default_version" "github.com/wso2/apk/test/integration/integration/utils/grpcutils" "github.com/wso2/apk/test/integration/integration/utils/suite" "testing" @@ -48,25 +49,27 @@ var GRPCAPI = suite.IntegrationTest{ ActualResponse: &student.StudentResponse{}, Name: "Get Student Details", Method: student.GetStudent, + Satisfier: student.StudentResponseSatisfier{}, + }, + { + ExpectedResponse: grpcutils.ExpectedResponse{ + Out: &student_default_version.StudentResponse{ + Name: "Dineth", + Age: 10, + }, + Err: nil, + }, + ActualResponse: &student_default_version.StudentResponse{}, + Name: "Get Student Details (Default API Version)", + Method: student_default_version.GetStudent, + Satisfier: student_default_version.StudentResponseSatisfier{}, }, - //{ - // ExpectedResponse: grpcutils.ExpectedResponse{ - // Out: &student_default_version.StudentResponse{ - // Name: "Dineth", - // Age: 10, - // }, - // Err: nil, - // }, - // ActualResponse: &student_default_version.StudentResponse{}, - // Name: "Get Student Details (Default API Version)", - // Method: student_default_version.GetStudent, - //}, } for i := range testCases { tc := testCases[i] t.Run("Invoke gRPC API", func(t *testing.T) { t.Parallel() - grpcutils.InvokeGRPCClientUntilSatisfied(gwAddr, t, tc, student.StudentResponseSatisfier{}, tc.Method) + grpcutils.InvokeGRPCClientUntilSatisfied(gwAddr, t, tc, tc.Satisfier, tc.Method) }) } }, diff --git a/test/integration/integration/utils/grpcutils/helpers.go b/test/integration/integration/utils/grpcutils/helpers.go index d2158aea5..66d339215 100644 --- a/test/integration/integration/utils/grpcutils/helpers.go +++ b/test/integration/integration/utils/grpcutils/helpers.go @@ -24,6 +24,7 @@ type GRPCTestCase struct { ActualResponse any Name string Method func(conn *grpc.ClientConn) (any, error) + Satisfier ResponseSatisfier } type ResponseSatisfier interface { IsSatisfactory(response interface{}, expectedResponse ExpectedResponse) bool From 290c65f22356386ad308edf91fec86af583e44a0 Mon Sep 17 00:00:00 2001 From: DinethH Date: Wed, 24 Apr 2024 21:34:39 +0530 Subject: [PATCH 50/52] resolved comments --- .../operator/synchronizer/grpc_api.go | 2 +- .../org/wso2/apk/enforcer/api/GRPCAPI.java | 31 ++++++------------- .../integration/integration/tests/grpc-api.go | 2 +- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/adapter/internal/operator/synchronizer/grpc_api.go b/adapter/internal/operator/synchronizer/grpc_api.go index aadf86098..26c0bd92c 100644 --- a/adapter/internal/operator/synchronizer/grpc_api.go +++ b/adapter/internal/operator/synchronizer/grpc_api.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/GRPCAPI.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/GRPCAPI.java index ca68c9a59..0ec44ae71 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/GRPCAPI.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/api/GRPCAPI.java @@ -8,14 +8,21 @@ import org.wso2.apk.enforcer.commons.Filter; import org.wso2.apk.enforcer.commons.dto.ClaimValueDTO; import org.wso2.apk.enforcer.commons.dto.JWTConfigurationDto; -import org.wso2.apk.enforcer.commons.model.*; +import org.wso2.apk.enforcer.commons.model.APIConfig; +import org.wso2.apk.enforcer.commons.model.ResourceConfig; +import org.wso2.apk.enforcer.commons.model.EndpointSecurity; +import org.wso2.apk.enforcer.commons.model.RequestContext; import org.wso2.apk.enforcer.commons.model.EndpointCluster; import org.wso2.apk.enforcer.config.ConfigHolder; import org.wso2.apk.enforcer.config.EnforcerConfig; import org.wso2.apk.enforcer.constants.APIConstants; import org.wso2.apk.enforcer.constants.HttpConstants; import org.wso2.apk.enforcer.cors.CorsFilter; -import org.wso2.apk.enforcer.discovery.api.*; +import org.wso2.apk.enforcer.discovery.api.Api; +import org.wso2.apk.enforcer.discovery.api.Resource; +import org.wso2.apk.enforcer.discovery.api.Operation; +import org.wso2.apk.enforcer.discovery.api.BackendJWTTokenInfo; +import org.wso2.apk.enforcer.discovery.api.Claim; import org.wso2.apk.enforcer.security.AuthFilter; import org.wso2.apk.enforcer.security.mtls.MtlsUtils; import org.wso2.apk.enforcer.server.swagger.APIDefinitionUtils; @@ -94,24 +101,11 @@ public String init(Api api) { } - SchemaParser schemaParser = new SchemaParser(); byte[] apiDefinition = api.getApiDefinitionFile().toByteArray(); TypeDefinitionRegistry registry; - //TODO fix this -// try { -// String scheme = APIDefinitionUtils.ReadGzip(apiDefinition); -// registry = schemaParser.parse(scheme); -// } catch (IOException e) { -// logger.error("Error while parsing the GRPC schema definition of the API: " + name, e); -// throw new RuntimeException(e); -// } -// GraphQLSchema schema = UnExecutableSchemaGenerator.makeUnExecutableSchema(registry); - -// GraphQLSchemaDTO graphQLSchemaDTO = new GraphQLSchemaDTO(schema, registry, -// GraphQLPayloadUtils.parseComplexityDTO(api.getGraphqlComplexityInfoList())); String apiLifeCycleState = api.getApiLifeCycleState(); this.apiConfig = new APIConfig.Builder(name).uuid(api.getId()).vhost(vhost).basePath(basePath).version(version) @@ -120,9 +114,8 @@ public String init(Api api) { .disableScopes(api.getDisableScopes()).trustStore(trustStore).organizationId(api.getOrganizationId()) .mutualSSL(mutualSSL) .applicationSecurity(applicationSecurity).jwtConfigurationDto(jwtConfigurationDto) -// .apiDefinition(apiDefinition).environment(api.getEnvironment()) + .apiDefinition(apiDefinition).environment(api.getEnvironment()) .environment(api.getEnvironment()) -// .subscriptionValidation(api.getSubscriptionValidation()).graphQLSchemaDTO(graphQLSchemaDTO).build(); .subscriptionValidation(api.getSubscriptionValidation()).build(); initFilters(); logger.info("APIConfig: " + this.apiConfig); @@ -212,10 +205,6 @@ private void initFilters() { authFilter.init(apiConfig, null); this.filters.add(authFilter); -// GraphQLQueryAnalysisFilter queryAnalysisFilter = new GraphQLQueryAnalysisFilter(); -// queryAnalysisFilter.init(apiConfig, null); -// this.filters.add(queryAnalysisFilter); - // CORS filter is added as the first filter, and it is not customizable. CorsFilter corsFilter = new CorsFilter(); this.filters.add(0, corsFilter); diff --git a/test/integration/integration/tests/grpc-api.go b/test/integration/integration/tests/grpc-api.go index 054bda3f3..43632c1f9 100644 --- a/test/integration/integration/tests/grpc-api.go +++ b/test/integration/integration/tests/grpc-api.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 43899079b60ab13e9894486785d8c82f82993135 Mon Sep 17 00:00:00 2001 From: DinethH Date: Thu, 25 Apr 2024 12:53:45 +0530 Subject: [PATCH 51/52] reconciles grpc scopes --- .../operator/controllers/dp/api_controller.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index ce7846ffb..ed541a472 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -90,6 +90,7 @@ const ( serviceHTTPRouteIndex = "serviceHTTPRouteIndex" httprouteScopeIndex = "httprouteScopeIndex" gqlRouteScopeIndex = "gqlRouteScopeIndex" + grpcRouteScopeIndex = "grpcRouteScopeIndex" configMapBackend = "configMapBackend" configMapAPIDefinition = "configMapAPIDefinition" secretBackend = "secretBackend" @@ -1660,6 +1661,22 @@ func (apiReconciler *APIReconciler) getAPIsForScope(ctx context.Context, obj k8c requests = append(requests, apiReconciler.getAPIForGQLRoute(ctx, &httpRoute)...) } + grpcRouteList := &gwapiv1a2.GRPCRouteList{} + if err := apiReconciler.client.List(ctx, grpcRouteList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(grpcRouteScopeIndex, utils.NamespacedName(scope).String()), + }); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2625, logging.CRITICAL, "Unable to find associated GRPCRoutes: %s", utils.NamespacedName(scope).String())) + return []reconcile.Request{} + } + + if len(grpcRouteList.Items) == 0 { + loggers.LoggerAPKOperator.Debugf("GRPCRoutes for scope not found: %s", utils.NamespacedName(scope).String()) + } + for item := range grpcRouteList.Items { + grpcRoute := grpcRouteList.Items[item] + requests = append(requests, apiReconciler.getAPIForGRPCRoute(ctx, &grpcRoute)...) + } + return requests } From 60fcea5895055d7d2aa7bfafe1c28a99ef4ed193 Mon Sep 17 00:00:00 2001 From: DinethH Date: Fri, 26 Apr 2024 11:53:19 +0530 Subject: [PATCH 52/52] remove todo --- adapter/internal/oasparser/envoyconf/routes_with_clusters.go | 1 - adapter/internal/oasparser/model/adapter_internal_api.go | 1 - 2 files changed, 2 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 172917eb6..dc2b983c3 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -235,7 +235,6 @@ func CreateRoutesWithClusters(adapterInternalAPI *model.AdapterInternalAPI, inte clusterName = existingClusterName } // Create resource level interceptor clusters if required - //TODO remove this if no interceptor needed for grpc (Dineth) clustersI, endpointsI, operationalReqInterceptors, operationalRespInterceptorVal := createInterceptorResourceClusters(adapterInternalAPI, interceptorCerts, vHost, organizationID, apiRequestInterceptor, apiResponseInterceptor, resource) clusters = append(clusters, clustersI...) diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 09e120b79..0e698a741 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -1073,7 +1073,6 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoGRPCRouteCR(grpcRoute *gwap loggers.LoggerOasparser.Debugf("Calculating auths for API ..., API_UUID = %v", adapterInternalAPI.UUID) apiAuth := getSecurity(resourceAuthScheme) - //TODO Add path match type (Dineth) for _, match := range rule.Matches { resourcePath := adapterInternalAPI.GetXWso2Basepath() + "." + *match.Method.Service + "/" + *match.Method.Method endPoints = append(endPoints, GetEndpoints(backendName, resourceParams.BackendMapping)...)