From f884ed33e6166ad7ac29d50e689ed73959f42fdd Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Fri, 29 Mar 2024 16:41:38 +0530 Subject: [PATCH 1/5] drop duplicate reconciles --- .../internal/controlplane/eventPublisher.go | 9 +- .../operator/controllers/dp/api_controller.go | 158 ++++++++++++++++-- 2 files changed, 149 insertions(+), 18 deletions(-) diff --git a/adapter/internal/controlplane/eventPublisher.go b/adapter/internal/controlplane/eventPublisher.go index a9c380ad2..4ea143331 100644 --- a/adapter/internal/controlplane/eventPublisher.go +++ b/adapter/internal/controlplane/eventPublisher.go @@ -84,17 +84,11 @@ type API struct { BasePath string `json:"basePath"` Organization string `json:"organization"` SystemAPI bool `json:"systemAPI"` - APIProperties []Property `json:"apiProperties,omitempty"` + APIProperties map[string]string `json:"apiProperties,omitempty"` Environment string `json:"environment,omitempty"` RevisionID string `json:"revisionID"` } -// Property holds key value pair of APIProperties -type Property struct { - Name string `json:"name,omitempty"` - Value string `json:"value,omitempty"` -} - // init reads the configuration and starts the worker to send data. func init() { configOnce.Do(func() { @@ -191,6 +185,7 @@ func sendData() { // AddToEventQueue adds the api event to queue func AddToEventQueue(data APICPEvent) { + loggers.LoggerAPK.Infof("Event added to queue : %+v", data) eventQueue <- data } diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 591a79db2..232a4b394 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -26,7 +26,12 @@ import ( "fmt" "io/ioutil" "sync" - + "reflect" + "strings" + "sort" + "strconv" + "crypto/sha256" + "encoding/hex" "github.com/wso2/apk/adapter/config" "github.com/wso2/apk/adapter/internal/controlplane" "github.com/wso2/apk/adapter/internal/discovery/xds" @@ -102,11 +107,13 @@ type APIReconciler struct { statusUpdater *status.UpdateHandler mgr manager.Manager apiPropagationEnabled bool + apiHashes map[string]string } // NewAPIController creates a new API controller instance. API Controllers watches for dpv1alpha2.API and gwapiv1b1.HTTPRoute. func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.OperatorDataStore, statusUpdater *status.UpdateHandler, ch *chan *synchronizer.APIEvent, successChannel *chan synchronizer.SuccessEvent) error { + apiHash := make(map[string]string) apiReconciler := &APIReconciler{ client: mgr.GetClient(), ods: operatorDataStore, @@ -114,6 +121,7 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera successChannel: successChannel, statusUpdater: statusUpdater, mgr: mgr, + apiHashes: apiHash, } ctx := context.Background() c, err := controller.New(constants.APIController, mgr, controller.Options{Reconciler: apiReconciler}) @@ -248,6 +256,8 @@ func (apiReconciler *APIReconciler) Reconcile(ctx context.Context, req ctrl.Requ var apiCR dpv1alpha2.API if err := apiReconciler.client.Get(ctx, req.NamespacedName, &apiCR); err != nil { apiState, found := apiReconciler.ods.GetCachedAPI(req.NamespacedName) + // remove the hash from the api hashes map if presents + delete(apiReconciler.apiHashes, req.NamespacedName.String()) if found && k8error.IsNotFound(err) { if apiReconciler.apiPropagationEnabled { // Convert api state to api cp data @@ -428,13 +438,19 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 } loggers.LoggerAPKOperator.Debugf("Child references are retrieved successfully for API CR %s", apiRef.String()) - + apiNamespacedName := utils.NamespacedName(apiState.APIDefinition).String() if !api.Status.DeploymentStatus.Accepted { if apiReconciler.apiPropagationEnabled { - // Publish the api data to CP - apiCpData := apiReconciler.convertAPIStateToAPICp(ctx, *apiState) - apiCpData.Event = controlplane.EventTypeCreate - controlplane.AddToEventQueue(apiCpData) + value, ok := apiReconciler.apiHashes[apiNamespacedName] + apiHash := apiReconciler.getAPIHash(apiState) + if !ok || value != apiHash { + loggers.LoggerAPKOperator.Infof("API hash changed sending the API to agent") + apiReconciler.apiHashes[apiNamespacedName] = apiHash + // Publish the api data to CP + apiCpData := apiReconciler.convertAPIStateToAPICp(ctx, *apiState) + apiCpData.Event = controlplane.EventTypeCreate + controlplane.AddToEventQueue(apiCpData) + } } apiReconciler.ods.AddAPIState(apiRef, apiState) @@ -443,10 +459,17 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 } else if cachedAPI, events, updated := apiReconciler.ods.UpdateAPIState(apiRef, apiState); updated { if apiReconciler.apiPropagationEnabled { - // Publish the api data to CP - apiCpData := apiReconciler.convertAPIStateToAPICp(ctx, *apiState) - apiCpData.Event = controlplane.EventTypeUpdate - controlplane.AddToEventQueue(apiCpData) + value, ok := apiReconciler.apiHashes[apiNamespacedName] + apiHash := apiReconciler.getAPIHash(apiState) + if !ok || value != apiHash { + loggers.LoggerAPK.Infof("Ok: %+v, value: %+v nam: %+v", ok, value, apiNamespacedName) + loggers.LoggerAPKOperator.Infof("API hash changed sending the API to agent") + apiReconciler.apiHashes[apiNamespacedName] = apiHash + // Publish the api data to CP + apiCpData := apiReconciler.convertAPIStateToAPICp(ctx, *apiState) + apiCpData.Event = controlplane.EventTypeUpdate + controlplane.AddToEventQueue(apiCpData) + } } apiReconciler.traverseAPIStateAndUpdateOwnerReferences(ctx, *apiState) loggers.LoggerAPKOperator.Infof("API CR %s with API UUID : %v is updated on %v", apiRef.String(), @@ -2157,7 +2180,10 @@ func (apiReconciler *APIReconciler) convertAPIStateToAPICp(ctx context.Context, if !revisionIDExists { revisionID = "0" } - + properties := make(map[string]string) + for _, val := range spec.APIProperties { + properties[val.Name] = val.Value + } api := controlplane.API{ APIName: spec.APIName, APIVersion: spec.APIVersion, @@ -2170,6 +2196,7 @@ func (apiReconciler *APIReconciler) convertAPIStateToAPICp(ctx context.Context, Definition: apiDef, APIUUID: apiUUID, RevisionID: revisionID, + APIProperties: properties, } apiCPEvent.API = api apiCPEvent.CRName = apiState.APIDefinition.ObjectMeta.Name @@ -2177,3 +2204,112 @@ func (apiReconciler *APIReconciler) convertAPIStateToAPICp(ctx context.Context, return apiCPEvent } + +func (apiReconciler *APIReconciler) getAPIHash(apiState *synchronizer.APIState) string { + + getUniqueId := func(obj interface{}, fields ...string) string { + loggers.LoggerAPK.Infof("Type of obj: %T", obj) + var sb strings.Builder + objValue := reflect.ValueOf(obj) + for _, field := range fields { + fieldNames := strings.Split(field, ".") + name1 := fieldNames[0] + name2 := fieldNames[1] + if objValue.IsValid() && objValue.Elem().FieldByName(name1).IsValid() { + if (objValue.Elem().FieldByName(name1).FieldByName(name2).IsValid()) { + v := objValue.Elem().FieldByName(name1).FieldByName(name2) + switch v.Kind() { + case reflect.String: + sb.WriteString(v.String()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + sb.WriteString(strconv.FormatInt(v.Int(), 10)) + } + } + } + + // if objValue.IsValid() { + // loggers.LoggerAPK.Infof("valid") + // // Print fields if objValue is a struct + // if objValue.Elem().Kind() == reflect.Struct { + // for i := 0; i < objValue.Elem().NumField(); i++ { + // fieldName := objValue.Elem().Type().Field(i).Name + // loggers.LoggerAPK.Infof("Field: %s", fieldName) + // field1 := objValue.Elem().Field(i) + // if field1.Kind() == reflect.Struct { + // for j := 0; j < field1.NumField(); j++ { + // childFieldName := field1.Type().Field(j).Name + // loggers.LoggerAPK.Infof(" Child Field: %s", childFieldName) + // } + // } + // } + + // } + // if objValue.IsValid() && objValue.Elem().FieldByName(field).IsValid() { + // loggers.LoggerAPK.Infof("valid valid") + // } + // } + } + return sb.String() + } + + uniqueIds := make([]string, 0) + uniqueIds = append(uniqueIds, getUniqueId(apiState.APIDefinition, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + for _, auth := range apiState.Authentications { + uniqueIds = append(uniqueIds, getUniqueId(auth, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + for _, arl := range apiState.RateLimitPolicies { + uniqueIds = append(uniqueIds, getUniqueId(arl, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + for _, ra := range apiState.ResourceAuthentications { + uniqueIds = append(uniqueIds, getUniqueId(ra, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + for _, rrl := range apiState.ResourceRateLimitPolicies { + uniqueIds = append(uniqueIds, getUniqueId(rrl, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + for _, ral := range apiState.ResourceAPIPolicies { + uniqueIds = append(uniqueIds, getUniqueId(ral, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + for _, ap := range apiState.APIPolicies { + uniqueIds = append(uniqueIds, getUniqueId(ap, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + for _, ism := range apiState.InterceptorServiceMapping { + uniqueIds = append(uniqueIds, getUniqueId(ism, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + for _, bjm := range apiState.BackendJWTMapping { + uniqueIds = append(uniqueIds, getUniqueId(bjm, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + if apiState.ProdHTTPRoute!= nil { + for _, phr := range apiState.ProdHTTPRoute.HTTPRoutePartitions { + uniqueIds = append(uniqueIds, getUniqueId(phr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + } + if apiState.SandHTTPRoute!= nil { + for _, shr := range apiState.SandHTTPRoute.HTTPRoutePartitions { + uniqueIds = append(uniqueIds, getUniqueId(shr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + } + if apiState.ProdGQLRoute!= nil { + for _, pgqr := range apiState.ProdGQLRoute.GQLRoutePartitions { + uniqueIds = append(uniqueIds, getUniqueId(pgqr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + } + if apiState.SandGQLRoute!= nil { + for _, sgqr := range apiState.SandGQLRoute.GQLRoutePartitions { + uniqueIds = append(uniqueIds, getUniqueId(sgqr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + } + } + + + sort.Strings(uniqueIds) + joinedUniqueIds := strings.Join(uniqueIds, "") + mutualSSLUniqueId := "" + if (apiState.MutualSSL != nil) { + mutualSSLUniqueId += strconv.FormatBool(apiState.MutualSSL.Disabled) + apiState.MutualSSL.Required + strings.Join(apiState.MutualSSL.ClientCertificates, "") + } + joinedUniqueIds = joinedUniqueIds + strconv.FormatBool(apiState.SubscriptionValidation) + mutualSSLUniqueId + loggers.LoggerAPK.Infof("Prepared unique string: %s", joinedUniqueIds) + hash := sha256.Sum256([]byte(joinedUniqueIds)) + hashedString := hex.EncodeToString(hash[:]) + loggers.LoggerAPK.Infof("Prepared hash: %s", hashedString) + return hashedString +} From 3611528ba0d739a9cc5ed011d56226fc3fb6393b Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Sat, 30 Mar 2024 18:40:09 +0530 Subject: [PATCH 2/5] Improve hash logic --- .../internal/controlplane/eventPublisher.go | 3 + .../operator/controllers/dp/api_controller.go | 142 +++++++++++++----- 2 files changed, 107 insertions(+), 38 deletions(-) diff --git a/adapter/internal/controlplane/eventPublisher.go b/adapter/internal/controlplane/eventPublisher.go index 4ea143331..08fb7fc66 100644 --- a/adapter/internal/controlplane/eventPublisher.go +++ b/adapter/internal/controlplane/eventPublisher.go @@ -87,6 +87,9 @@ type API struct { APIProperties map[string]string `json:"apiProperties,omitempty"` Environment string `json:"environment,omitempty"` RevisionID string `json:"revisionID"` + SandEndpoint string `json:"sandEndpoint"` + ProdEndpoint string `json:"prodEndpoint"` + EndpointProtocol string `json:"endpointProtocol"` } // init reads the configuration and starts the worker to send data. diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 232a4b394..774cb12d4 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -439,11 +439,13 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 loggers.LoggerAPKOperator.Debugf("Child references are retrieved successfully for API CR %s", apiRef.String()) apiNamespacedName := utils.NamespacedName(apiState.APIDefinition).String() + loggers.LoggerAPK.Infof("label section of the api : %+v", apiState.APIDefinition.ObjectMeta.Labels) + storedHash, hashFound := apiState.APIDefinition.ObjectMeta.Labels["apiHash"] if !api.Status.DeploymentStatus.Accepted { if apiReconciler.apiPropagationEnabled { - value, ok := apiReconciler.apiHashes[apiNamespacedName] apiHash := apiReconciler.getAPIHash(apiState) - if !ok || value != apiHash { + if !hashFound || storedHash != apiHash { + apiReconciler.patchAPIHash(ctx, apiHash, apiState.APIDefinition.ObjectMeta.Name, apiState.APIDefinition.ObjectMeta.Namespace) loggers.LoggerAPKOperator.Infof("API hash changed sending the API to agent") apiReconciler.apiHashes[apiNamespacedName] = apiHash // Publish the api data to CP @@ -459,10 +461,11 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 } else if cachedAPI, events, updated := apiReconciler.ods.UpdateAPIState(apiRef, apiState); updated { if apiReconciler.apiPropagationEnabled { - value, ok := apiReconciler.apiHashes[apiNamespacedName] + apiHash := apiReconciler.getAPIHash(apiState) - if !ok || value != apiHash { - loggers.LoggerAPK.Infof("Ok: %+v, value: %+v nam: %+v", ok, value, apiNamespacedName) + if !hashFound || storedHash != apiHash { + apiReconciler.patchAPIHash(ctx, apiHash, apiState.APIDefinition.ObjectMeta.Name, apiState.APIDefinition.ObjectMeta.Namespace) + loggers.LoggerAPK.Infof("Ok: %+v, value: %+v nam: %+v", hashFound, storedHash, apiNamespacedName) loggers.LoggerAPKOperator.Infof("API hash changed sending the API to agent") apiReconciler.apiHashes[apiNamespacedName] = apiHash // Publish the api data to CP @@ -2061,13 +2064,13 @@ func (apiReconciler *APIReconciler) handleStatus() { } } -func (apiReconciler *APIReconciler) handleLabels(ctx context.Context) { - type patchStringValue struct { - Op string `json:"op"` - Path string `json:"path"` - Value string `json:"value"` - } +type patchStringValue struct { + Op string `json:"op"` + Path string `json:"path"` + Value string `json:"value"` +} +func (apiReconciler *APIReconciler) handleLabels(ctx context.Context) { loggers.LoggerAPKOperator.Info("A thread assigned to handle label updates to API CR.") for labelUpdate := range *controlplane.GetLabelQueue() { loggers.LoggerAPKOperator.Infof("Starting to process label update for API %s/%s. Labels: %+v", labelUpdate.Namespace, labelUpdate.Name, labelUpdate.Labels) @@ -2091,10 +2094,63 @@ func (apiReconciler *APIReconciler) handleLabels(ctx context.Context) { err := apiReconciler.client.Patch(ctx, &apiCR, k8client.RawPatch(types.JSONPatchType, payloadBytes)) if err != nil { loggers.LoggerAPKOperator.Errorf("Failed to patch api %s/%s with patch: %+v, error: %+v", labelUpdate.Name, labelUpdate.Namespace, patchOps, err) + // Patch did not work it could be due to labels field does not exists. Lets try to update the CR with labels field. + var apiCR dpv1alpha2.API + if err := apiReconciler.client.Get(ctx, types.NamespacedName{Namespace: labelUpdate.Namespace, Name: labelUpdate.Name}, &apiCR); err == nil { + if apiCR.ObjectMeta.Labels == nil { + apiCR.ObjectMeta.Labels = map[string]string{} + } + for key, value := range labelUpdate.Labels { + apiCR.ObjectMeta.Labels[key] = value + } + crUpdateError := apiReconciler.client.Update(ctx, &apiCR) + if crUpdateError != nil { + loggers.LoggerAPKOperator.Errorf("Error while updating the API CR for api labels. Error: %+v", crUpdateError) + } + } else { + loggers.LoggerAPKOperator.Errorf("Error while loading api: %s/%s, Error: %v", labelUpdate.Name, labelUpdate.Namespace, err) + } } } } + + +func (apiReconciler *APIReconciler) patchAPIHash(ctx context.Context, hash string, name string, namespace string) { + apiCR := dpv1alpha2.API{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + hashKey := "apiHash" + patchOps := []patchStringValue{} + patchOps = append(patchOps, patchStringValue{ + Op: "replace", + Path: fmt.Sprintf("/metadata/labels/%s", hashKey), + Value: hash, + }) + payloadBytes, _ := json.Marshal(patchOps) + err := apiReconciler.client.Patch(ctx, &apiCR, k8client.RawPatch(types.JSONPatchType, payloadBytes)) + if err != nil { + loggers.LoggerAPKOperator.Errorf("Failed to patch api %s/%s with patch: %+v, error: %+v", name, namespace, patchOps, err) + // Patch did not work it could be due to labels field does not exists. Lets try to update the CR with labels field. + var apiCR dpv1alpha2.API + if err := apiReconciler.client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &apiCR); err == nil { + if apiCR.ObjectMeta.Labels == nil { + apiCR.ObjectMeta.Labels = map[string]string{} + } + apiCR.ObjectMeta.Labels["apiHash"] = hash + crUpdateError := apiReconciler.client.Update(ctx, &apiCR) + if crUpdateError != nil { + loggers.LoggerAPKOperator.Errorf("Error while updating the API CR for api hash. Error: %+v", crUpdateError) + } + } else { + loggers.LoggerAPKOperator.Errorf("Error while loading api: %s/%s, Error: %v", name, namespace, err) + } + } +} + func (apiReconciler *APIReconciler) handleOwnerReference(ctx context.Context, obj k8client.Object, apiRequests *[]reconcile.Request) { apis := []dpv1alpha2.API{} for _, req := range *apiRequests { @@ -2184,6 +2240,7 @@ func (apiReconciler *APIReconciler) convertAPIStateToAPICp(ctx context.Context, for _, val := range spec.APIProperties { properties[val.Name] = val.Value } + prodEndpoint, sandEndpoint, endpointProtocol := findProdSandEndpoints(&apiState); api := controlplane.API{ APIName: spec.APIName, APIVersion: spec.APIVersion, @@ -2197,6 +2254,9 @@ func (apiReconciler *APIReconciler) convertAPIStateToAPICp(ctx context.Context, APIUUID: apiUUID, RevisionID: revisionID, APIProperties: properties, + ProdEndpoint: prodEndpoint, + SandEndpoint: sandEndpoint, + EndpointProtocol: endpointProtocol, } apiCPEvent.API = api apiCPEvent.CRName = apiState.APIDefinition.ObjectMeta.Name @@ -2211,13 +2271,16 @@ func (apiReconciler *APIReconciler) getAPIHash(apiState *synchronizer.APIState) loggers.LoggerAPK.Infof("Type of obj: %T", obj) var sb strings.Builder objValue := reflect.ValueOf(obj) + if objValue.Kind() == reflect.Ptr { + objValue = objValue.Elem() + } for _, field := range fields { fieldNames := strings.Split(field, ".") name1 := fieldNames[0] name2 := fieldNames[1] - if objValue.IsValid() && objValue.Elem().FieldByName(name1).IsValid() { - if (objValue.Elem().FieldByName(name1).FieldByName(name2).IsValid()) { - v := objValue.Elem().FieldByName(name1).FieldByName(name2) + if objValue.IsValid() && objValue.FieldByName(name1).IsValid() { + if (objValue.FieldByName(name1).FieldByName(name2).IsValid()) { + v := objValue.FieldByName(name1).FieldByName(name2) switch v.Kind() { case reflect.String: sb.WriteString(v.String()) @@ -2226,28 +2289,6 @@ func (apiReconciler *APIReconciler) getAPIHash(apiState *synchronizer.APIState) } } } - - // if objValue.IsValid() { - // loggers.LoggerAPK.Infof("valid") - // // Print fields if objValue is a struct - // if objValue.Elem().Kind() == reflect.Struct { - // for i := 0; i < objValue.Elem().NumField(); i++ { - // fieldName := objValue.Elem().Type().Field(i).Name - // loggers.LoggerAPK.Infof("Field: %s", fieldName) - // field1 := objValue.Elem().Field(i) - // if field1.Kind() == reflect.Struct { - // for j := 0; j < field1.NumField(); j++ { - // childFieldName := field1.Type().Field(j).Name - // loggers.LoggerAPK.Infof(" Child Field: %s", childFieldName) - // } - // } - // } - - // } - // if objValue.IsValid() && objValue.Elem().FieldByName(field).IsValid() { - // loggers.LoggerAPK.Infof("valid valid") - // } - // } } return sb.String() } @@ -2310,6 +2351,31 @@ func (apiReconciler *APIReconciler) getAPIHash(apiState *synchronizer.APIState) loggers.LoggerAPK.Infof("Prepared unique string: %s", joinedUniqueIds) hash := sha256.Sum256([]byte(joinedUniqueIds)) hashedString := hex.EncodeToString(hash[:]) - loggers.LoggerAPK.Infof("Prepared hash: %s", hashedString) - return hashedString + truncatedHash := hashedString[:62] + loggers.LoggerAPK.Infof("Prepared hash: %s, truncated hash: %s", hashedString, truncatedHash) + return truncatedHash +} + +func findProdSandEndpoints(apiState *synchronizer.APIState) (string, string, string) { + prodEndpoint := "" + sandEndpoint := "" + endpointProtocol := "" + if apiState.ProdHTTPRoute != nil { + for _, backend := range apiState.ProdHTTPRoute.BackendMapping { + if (len(backend.Backend.Spec.Services) > 0) { + sandEndpoint = fmt.Sprintf("%s:%d", backend.Backend.Spec.Services[0].Host, backend.Backend.Spec.Services[0].Port) + endpointProtocol = string(backend.Backend.Spec.Protocol) + } + } + } + if apiState.SandHTTPRoute != nil { + for _, backend := range apiState.SandHTTPRoute.BackendMapping { + if (len(backend.Backend.Spec.Services) > 0) { + prodEndpoint = fmt.Sprintf("%s:%d", backend.Backend.Spec.Services[0].Host, backend.Backend.Spec.Services[0].Port) + endpointProtocol = string(backend.Backend.Spec.Protocol) + } + } + } + loggers.LoggerAPK.Infof("Returning endpoints: %s, %s", prodEndpoint, sandEndpoint) + return prodEndpoint, sandEndpoint, endpointProtocol } From 0427f3599677a80f6804499f9d1255ae0f3f4de3 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Sat, 30 Mar 2024 18:45:27 +0530 Subject: [PATCH 3/5] Skip system api Dp to CP --- adapter/internal/operator/controllers/dp/api_controller.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 774cb12d4..31504d0fa 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -442,7 +442,7 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 loggers.LoggerAPK.Infof("label section of the api : %+v", apiState.APIDefinition.ObjectMeta.Labels) storedHash, hashFound := apiState.APIDefinition.ObjectMeta.Labels["apiHash"] if !api.Status.DeploymentStatus.Accepted { - if apiReconciler.apiPropagationEnabled { + if apiReconciler.apiPropagationEnabled && !apiState.APIDefinition.Spec.SystemAPI { apiHash := apiReconciler.getAPIHash(apiState) if !hashFound || storedHash != apiHash { apiReconciler.patchAPIHash(ctx, apiHash, apiState.APIDefinition.ObjectMeta.Name, apiState.APIDefinition.ObjectMeta.Namespace) @@ -460,8 +460,7 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 return &synchronizer.APIEvent{EventType: constants.Create, Events: []synchronizer.APIState{*apiState}, UpdatedEvents: []string{}}, nil } else if cachedAPI, events, updated := apiReconciler.ods.UpdateAPIState(apiRef, apiState); updated { - if apiReconciler.apiPropagationEnabled { - + if apiReconciler.apiPropagationEnabled && !apiState.APIDefinition.Spec.SystemAPI{ apiHash := apiReconciler.getAPIHash(apiState) if !hashFound || storedHash != apiHash { apiReconciler.patchAPIHash(ctx, apiHash, apiState.APIDefinition.ObjectMeta.Name, apiState.APIDefinition.ObjectMeta.Namespace) From 33c3027879f3011a82d4d690301e14775eaf9c1a Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Sun, 31 Mar 2024 09:35:15 +0530 Subject: [PATCH 4/5] Stop deploying the api until all the resouce policies added to the maps --- .../operator/controllers/dp/api_controller.go | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 31504d0fa..7030c1334 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -437,6 +437,12 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 } } + // Validate resource level extension refs resolved + extRefValErr := apiReconciler.validateHttpRouteExtRefs(apiState) + if extRefValErr != nil { + return nil, extRefValErr + } + loggers.LoggerAPKOperator.Debugf("Child references are retrieved successfully for API CR %s", apiRef.String()) apiNamespacedName := utils.NamespacedName(apiState.APIDefinition).String() loggers.LoggerAPK.Infof("label section of the api : %+v", apiState.APIDefinition.ObjectMeta.Labels) @@ -454,7 +460,6 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 controlplane.AddToEventQueue(apiCpData) } } - apiReconciler.ods.AddAPIState(apiRef, apiState) apiReconciler.traverseAPIStateAndUpdateOwnerReferences(ctx, *apiState) return &synchronizer.APIEvent{EventType: constants.Create, Events: []synchronizer.APIState{*apiState}, UpdatedEvents: []string{}}, nil @@ -504,6 +509,7 @@ func (apiReconciler *APIReconciler) resolveHTTPRouteRefs(ctx context.Context, ht } httpRouteState.BackendMapping = apiReconciler.getResolvedBackendsMapping(ctx, httpRouteState, interceptorServiceMapping, api) httpRouteState.Scopes, err = apiReconciler.getScopesForHTTPRoute(ctx, httpRouteState.HTTPRouteCombined, api) + return httpRouteState, err } @@ -2264,6 +2270,53 @@ func (apiReconciler *APIReconciler) convertAPIStateToAPICp(ctx context.Context, } +func (apiReconciler *APIReconciler) validateHttpRouteExtRefs(apiState *synchronizer.APIState) error { + extRefs := []*gwapiv1b1.LocalObjectReference{} + if apiState.ProdHTTPRoute != nil { + for _, httpRoute := range apiState.ProdHTTPRoute.HTTPRoutePartitions { + for _, rule := range httpRoute.Spec.Rules { + for _, filter := range rule.Filters { + extRefs = append(extRefs, filter.ExtensionRef) + } + } + } + } + if apiState.SandHTTPRoute != nil { + for _, httpRoute := range apiState.SandHTTPRoute.HTTPRoutePartitions { + for _, rule := range httpRoute.Spec.Rules { + for _, filter := range rule.Filters { + extRefs = append(extRefs, filter.ExtensionRef) + } + } + } + } + for _, extRef := range extRefs { + if extRef != nil { + extKind := string(extRef.Kind) + key := types.NamespacedName{Namespace: string(apiState.APIDefinition.Namespace), Name: string(extRef.Name)}.String() + if (extKind == "APIPolicy") { + _, found := apiState.ResourceAPIPolicies[key] + if !found { + return fmt.Errorf("apipolicy not added to the ResourceAPIPolicies map yet. Key: %s", key) + } + } + if (extKind == "RateLimitPolicy") { + _, found := apiState.ResourceRateLimitPolicies[key] + if !found { + return fmt.Errorf("ratelimitPolicy not added to the ResourceRateLimitPolicies map yet. Key: %s", key) + } + } + if (extKind == "Authentication") { + _, found := apiState.ResourceAuthentications[key] + if !found { + return fmt.Errorf("authentication not added to the resourse Authentication map yet. Key: %s", key) + } + } + } + } + return nil +} + func (apiReconciler *APIReconciler) getAPIHash(apiState *synchronizer.APIState) string { getUniqueId := func(obj interface{}, fields ...string) string { From 2636a744e824b0800c6ec8a51f7e6daada182be5 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Sun, 31 Mar 2024 12:07:16 +0530 Subject: [PATCH 5/5] Add cors support --- .../internal/controlplane/eventPublisher.go | 57 ++-- .../operator/controllers/dp/api_controller.go | 243 ++++++++++++------ .../operator/synchronizer/data_store.go | 2 +- 3 files changed, 205 insertions(+), 97 deletions(-) diff --git a/adapter/internal/controlplane/eventPublisher.go b/adapter/internal/controlplane/eventPublisher.go index 08fb7fc66..91a5b38ae 100644 --- a/adapter/internal/controlplane/eventPublisher.go +++ b/adapter/internal/controlplane/eventPublisher.go @@ -75,21 +75,35 @@ type APICPEvent struct { // API holds the data that needs to be sent to agent type API struct { - APIUUID string `json:"apiUUID"` - APIName string `json:"apiName"` - APIVersion string `json:"apiVersion"` - IsDefaultVersion bool `json:"isDefaultVersion"` - Definition string `json:"definition"` - APIType string `json:"apiType"` - BasePath string `json:"basePath"` - Organization string `json:"organization"` - SystemAPI bool `json:"systemAPI"` + APIUUID string `json:"apiUUID"` + APIName string `json:"apiName"` + APIVersion string `json:"apiVersion"` + IsDefaultVersion bool `json:"isDefaultVersion"` + Definition string `json:"definition"` + APIType string `json:"apiType"` + BasePath string `json:"basePath"` + Organization string `json:"organization"` + SystemAPI bool `json:"systemAPI"` APIProperties map[string]string `json:"apiProperties,omitempty"` - Environment string `json:"environment,omitempty"` - RevisionID string `json:"revisionID"` - SandEndpoint string `json:"sandEndpoint"` - ProdEndpoint string `json:"prodEndpoint"` - EndpointProtocol string `json:"endpointProtocol"` + Environment string `json:"environment,omitempty"` + RevisionID string `json:"revisionID"` + SandEndpoint string `json:"sandEndpoint"` + ProdEndpoint string `json:"prodEndpoint"` + EndpointProtocol string `json:"endpointProtocol"` + CORSPolicy *CORSPolicy `json:"cORSPolicy,omitempty"` + Vhost string `json:"vhost"` + SecurityScheme []string `json:"securityScheme"` + AuthHeader string `json:"authHeader"` +} + +// CORSPolicy hold cors configs +type CORSPolicy struct { + AccessControlAllowCredentials bool `json:"accessControlAllowCredentials,omitempty"` + AccessControlAllowHeaders []string `json:"accessControlAllowHeaders,omitempty"` + AccessControlAllowOrigins []string `json:"accessControlAllowOrigins,omitempty"` + AccessControlExposeHeaders []string `json:"accessControlExposeHeaders,omitempty"` + AccessControlMaxAge *int `json:"accessControlMaxAge,omitempty"` + AccessControlAllowMethods []string `json:"accessControlAllowMethods,omitempty"` } // init reads the configuration and starts the worker to send data. @@ -152,7 +166,7 @@ func sendData() { continue } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusServiceUnavailable { body, _ := ioutil.ReadAll(resp.Body) loggers.LoggerAPK.Errorf("Error: Unexpected status code: %d, received message: %s, retrying after %d seconds", resp.StatusCode, string(body), retryInterval) // Sleep for some time before retrying @@ -171,15 +185,18 @@ func sendData() { } // Assuming the response contains an ID field, you can extract it like this: id, ok := responseMap["id"].(string) - if !ok { - loggers.LoggerAPK.Errorf("Id field not present in response body. encoded body: %+v", responseMap) - break + revisionID, revisionOk := responseMap["revisionID"].(string) + if !ok || !revisionOk { + loggers.LoggerAPK.Errorf("Id or/both revision Id field not present in response body. encoded body: %+v", responseMap) + id = "" + revisionID = "" + // break } loggers.LoggerAPK.Infof("Adding label update to API %s/%s, Lebels: apiUUID: %s", event.CRNamespace, event.CRName, id) labelsQueue <- APICRLabelsUpdate{ Namespace: event.CRNamespace, Name: event.CRName, - Labels: map[string]string{"apiUUID": id}, + Labels: map[string]string{"apiUUID": id, "revisionID": revisionID}, } break } @@ -188,7 +205,7 @@ func sendData() { // AddToEventQueue adds the api event to queue func AddToEventQueue(data APICPEvent) { - loggers.LoggerAPK.Infof("Event added to queue : %+v", data) + loggers.LoggerAPK.Debugf("Event added to CP Event queue : %+v", data) eventQueue <- data } diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 7030c1334..9c1e70af1 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -21,17 +21,18 @@ import ( "bytes" "compress/gzip" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" "io/ioutil" - "sync" "reflect" - "strings" - "sort" + "sort" "strconv" - "crypto/sha256" - "encoding/hex" + "strings" + "sync" + "github.com/wso2/apk/adapter/config" "github.com/wso2/apk/adapter/internal/controlplane" "github.com/wso2/apk/adapter/internal/discovery/xds" @@ -59,6 +60,7 @@ import ( k8client "sigs.k8s.io/controller-runtime/pkg/client" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" + "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -100,20 +102,18 @@ var ( // APIReconciler reconciles a API object type APIReconciler struct { - client k8client.Client - ods *synchronizer.OperatorDataStore - ch *chan *synchronizer.APIEvent - successChannel *chan synchronizer.SuccessEvent - statusUpdater *status.UpdateHandler - mgr manager.Manager + client k8client.Client + ods *synchronizer.OperatorDataStore + ch *chan *synchronizer.APIEvent + successChannel *chan synchronizer.SuccessEvent + statusUpdater *status.UpdateHandler + mgr manager.Manager apiPropagationEnabled bool - apiHashes map[string]string } // NewAPIController creates a new API controller instance. API Controllers watches for dpv1alpha2.API and gwapiv1b1.HTTPRoute. func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.OperatorDataStore, statusUpdater *status.UpdateHandler, ch *chan *synchronizer.APIEvent, successChannel *chan synchronizer.SuccessEvent) error { - apiHash := make(map[string]string) apiReconciler := &APIReconciler{ client: mgr.GetClient(), ods: operatorDataStore, @@ -121,7 +121,6 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera successChannel: successChannel, statusUpdater: statusUpdater, mgr: mgr, - apiHashes: apiHash, } ctx := context.Background() c, err := controller.New(constants.APIController, mgr, controller.Options{Reconciler: apiReconciler}) @@ -256,8 +255,6 @@ func (apiReconciler *APIReconciler) Reconcile(ctx context.Context, req ctrl.Requ var apiCR dpv1alpha2.API if err := apiReconciler.client.Get(ctx, req.NamespacedName, &apiCR); err != nil { apiState, found := apiReconciler.ods.GetCachedAPI(req.NamespacedName) - // remove the hash from the api hashes map if presents - delete(apiReconciler.apiHashes, req.NamespacedName.String()) if found && k8error.IsNotFound(err) { if apiReconciler.apiPropagationEnabled { // Convert api state to api cp data @@ -438,14 +435,12 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 } // Validate resource level extension refs resolved - extRefValErr := apiReconciler.validateHttpRouteExtRefs(apiState) + extRefValErr := apiReconciler.validateHTTPRouteExtRefs(apiState) if extRefValErr != nil { return nil, extRefValErr } loggers.LoggerAPKOperator.Debugf("Child references are retrieved successfully for API CR %s", apiRef.String()) - apiNamespacedName := utils.NamespacedName(apiState.APIDefinition).String() - loggers.LoggerAPK.Infof("label section of the api : %+v", apiState.APIDefinition.ObjectMeta.Labels) storedHash, hashFound := apiState.APIDefinition.ObjectMeta.Labels["apiHash"] if !api.Status.DeploymentStatus.Accepted { if apiReconciler.apiPropagationEnabled && !apiState.APIDefinition.Spec.SystemAPI { @@ -453,7 +448,6 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 if !hashFound || storedHash != apiHash { apiReconciler.patchAPIHash(ctx, apiHash, apiState.APIDefinition.ObjectMeta.Name, apiState.APIDefinition.ObjectMeta.Namespace) loggers.LoggerAPKOperator.Infof("API hash changed sending the API to agent") - apiReconciler.apiHashes[apiNamespacedName] = apiHash // Publish the api data to CP apiCpData := apiReconciler.convertAPIStateToAPICp(ctx, *apiState) apiCpData.Event = controlplane.EventTypeCreate @@ -465,13 +459,11 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 return &synchronizer.APIEvent{EventType: constants.Create, Events: []synchronizer.APIState{*apiState}, UpdatedEvents: []string{}}, nil } else if cachedAPI, events, updated := apiReconciler.ods.UpdateAPIState(apiRef, apiState); updated { - if apiReconciler.apiPropagationEnabled && !apiState.APIDefinition.Spec.SystemAPI{ + if apiReconciler.apiPropagationEnabled && !apiState.APIDefinition.Spec.SystemAPI { apiHash := apiReconciler.getAPIHash(apiState) if !hashFound || storedHash != apiHash { apiReconciler.patchAPIHash(ctx, apiHash, apiState.APIDefinition.ObjectMeta.Name, apiState.APIDefinition.ObjectMeta.Namespace) - loggers.LoggerAPK.Infof("Ok: %+v, value: %+v nam: %+v", hashFound, storedHash, apiNamespacedName) loggers.LoggerAPKOperator.Infof("API hash changed sending the API to agent") - apiReconciler.apiHashes[apiNamespacedName] = apiHash // Publish the api data to CP apiCpData := apiReconciler.convertAPIStateToAPICp(ctx, *apiState) apiCpData.Event = controlplane.EventTypeUpdate @@ -2119,8 +2111,6 @@ func (apiReconciler *APIReconciler) handleLabels(ctx context.Context) { } } - - func (apiReconciler *APIReconciler) patchAPIHash(ctx context.Context, hash string, name string, namespace string) { apiCR := dpv1alpha2.API{ ObjectMeta: metav1.ObjectMeta{ @@ -2153,7 +2143,7 @@ func (apiReconciler *APIReconciler) patchAPIHash(ctx context.Context, hash strin } else { loggers.LoggerAPKOperator.Errorf("Error while loading api: %s/%s, Error: %v", name, namespace, err) } - } + } } func (apiReconciler *APIReconciler) handleOwnerReference(ctx context.Context, obj k8client.Object, apiRequests *[]reconcile.Request) { @@ -2245,7 +2235,10 @@ func (apiReconciler *APIReconciler) convertAPIStateToAPICp(ctx context.Context, for _, val := range spec.APIProperties { properties[val.Name] = val.Value } - prodEndpoint, sandEndpoint, endpointProtocol := findProdSandEndpoints(&apiState); + prodEndpoint, sandEndpoint, endpointProtocol := findProdSandEndpoints(&apiState) + corsPolicy := pickOneCorsForCP(&apiState) + vhost := getProdVhost(&apiState) + securityScheme, authHeader := prepareSecuritySchemeForCP(&apiState) api := controlplane.API{ APIName: spec.APIName, APIVersion: spec.APIVersion, @@ -2262,6 +2255,10 @@ func (apiReconciler *APIReconciler) convertAPIStateToAPICp(ctx context.Context, ProdEndpoint: prodEndpoint, SandEndpoint: sandEndpoint, EndpointProtocol: endpointProtocol, + CORSPolicy: corsPolicy, + Vhost: vhost, + SecurityScheme: securityScheme, + AuthHeader: authHeader, } apiCPEvent.API = api apiCPEvent.CRName = apiState.APIDefinition.ObjectMeta.Name @@ -2270,7 +2267,7 @@ func (apiReconciler *APIReconciler) convertAPIStateToAPICp(ctx context.Context, } -func (apiReconciler *APIReconciler) validateHttpRouteExtRefs(apiState *synchronizer.APIState) error { +func (apiReconciler *APIReconciler) validateHTTPRouteExtRefs(apiState *synchronizer.APIState) error { extRefs := []*gwapiv1b1.LocalObjectReference{} if apiState.ProdHTTPRoute != nil { for _, httpRoute := range apiState.ProdHTTPRoute.HTTPRoutePartitions { @@ -2294,19 +2291,19 @@ func (apiReconciler *APIReconciler) validateHttpRouteExtRefs(apiState *synchroni if extRef != nil { extKind := string(extRef.Kind) key := types.NamespacedName{Namespace: string(apiState.APIDefinition.Namespace), Name: string(extRef.Name)}.String() - if (extKind == "APIPolicy") { + if extKind == "APIPolicy" { _, found := apiState.ResourceAPIPolicies[key] if !found { return fmt.Errorf("apipolicy not added to the ResourceAPIPolicies map yet. Key: %s", key) } } - if (extKind == "RateLimitPolicy") { + if extKind == "RateLimitPolicy" { _, found := apiState.ResourceRateLimitPolicies[key] if !found { return fmt.Errorf("ratelimitPolicy not added to the ResourceRateLimitPolicies map yet. Key: %s", key) } } - if (extKind == "Authentication") { + if extKind == "Authentication" { _, found := apiState.ResourceAuthentications[key] if !found { return fmt.Errorf("authentication not added to the resourse Authentication map yet. Key: %s", key) @@ -2318,93 +2315,95 @@ func (apiReconciler *APIReconciler) validateHttpRouteExtRefs(apiState *synchroni } func (apiReconciler *APIReconciler) getAPIHash(apiState *synchronizer.APIState) string { - - getUniqueId := func(obj interface{}, fields ...string) string { - loggers.LoggerAPK.Infof("Type of obj: %T", obj) + getUniqueID := func(obj interface{}, fields ...string) string { + defer func() { + if r := recover(); r != nil { + loggers.LoggerAPK.Infof("Error occured while extracting values using reflection. Error: %+v", r) + } + }() var sb strings.Builder - objValue := reflect.ValueOf(obj) + objValue := reflect.ValueOf(obj) if objValue.Kind() == reflect.Ptr { objValue = objValue.Elem() } - for _, field := range fields { + for _, field := range fields { fieldNames := strings.Split(field, ".") name1 := fieldNames[0] name2 := fieldNames[1] if objValue.IsValid() && objValue.FieldByName(name1).IsValid() { - if (objValue.FieldByName(name1).FieldByName(name2).IsValid()) { + if objValue.FieldByName(name1).FieldByName(name2).IsValid() { v := objValue.FieldByName(name1).FieldByName(name2) switch v.Kind() { case reflect.String: - sb.WriteString(v.String()) + sb.WriteString(v.String()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - sb.WriteString(strconv.FormatInt(v.Int(), 10)) + sb.WriteString(strconv.FormatInt(v.Int(), 10)) } - } - } - } - return sb.String() + } + } + } + return sb.String() } - uniqueIds := make([]string, 0) - uniqueIds = append(uniqueIds, getUniqueId(apiState.APIDefinition, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs := make([]string, 0) + uniqueIDs = append(uniqueIDs, getUniqueID(apiState.APIDefinition, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) for _, auth := range apiState.Authentications { - uniqueIds = append(uniqueIds, getUniqueId(auth, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(auth, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } for _, arl := range apiState.RateLimitPolicies { - uniqueIds = append(uniqueIds, getUniqueId(arl, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(arl, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } for _, ra := range apiState.ResourceAuthentications { - uniqueIds = append(uniqueIds, getUniqueId(ra, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(ra, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } for _, rrl := range apiState.ResourceRateLimitPolicies { - uniqueIds = append(uniqueIds, getUniqueId(rrl, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(rrl, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } for _, ral := range apiState.ResourceAPIPolicies { - uniqueIds = append(uniqueIds, getUniqueId(ral, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(ral, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } for _, ap := range apiState.APIPolicies { - uniqueIds = append(uniqueIds, getUniqueId(ap, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(ap, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } for _, ism := range apiState.InterceptorServiceMapping { - uniqueIds = append(uniqueIds, getUniqueId(ism, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(ism, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } for _, bjm := range apiState.BackendJWTMapping { - uniqueIds = append(uniqueIds, getUniqueId(bjm, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(bjm, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } - if apiState.ProdHTTPRoute!= nil { + if apiState.ProdHTTPRoute != nil { for _, phr := range apiState.ProdHTTPRoute.HTTPRoutePartitions { - uniqueIds = append(uniqueIds, getUniqueId(phr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(phr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } } - if apiState.SandHTTPRoute!= nil { + if apiState.SandHTTPRoute != nil { for _, shr := range apiState.SandHTTPRoute.HTTPRoutePartitions { - uniqueIds = append(uniqueIds, getUniqueId(shr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(shr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } } - if apiState.ProdGQLRoute!= nil { + if apiState.ProdGQLRoute != nil { for _, pgqr := range apiState.ProdGQLRoute.GQLRoutePartitions { - uniqueIds = append(uniqueIds, getUniqueId(pgqr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(pgqr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } } - if apiState.SandGQLRoute!= nil { + if apiState.SandGQLRoute != nil { for _, sgqr := range apiState.SandGQLRoute.GQLRoutePartitions { - uniqueIds = append(uniqueIds, getUniqueId(sgqr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) + uniqueIDs = append(uniqueIDs, getUniqueID(sgqr, "ObjectMeta.Name", "ObjectMeta.Namespace", "ObjectMeta.Generation")) } } - - sort.Strings(uniqueIds) - joinedUniqueIds := strings.Join(uniqueIds, "") - mutualSSLUniqueId := "" - if (apiState.MutualSSL != nil) { - mutualSSLUniqueId += strconv.FormatBool(apiState.MutualSSL.Disabled) + apiState.MutualSSL.Required + strings.Join(apiState.MutualSSL.ClientCertificates, "") + sort.Strings(uniqueIDs) + joinedUniqueIDs := strings.Join(uniqueIDs, "") + mutualSSLUniqueID := "" + if apiState.MutualSSL != nil { + mutualSSLUniqueID += strconv.FormatBool(apiState.MutualSSL.Disabled) + apiState.MutualSSL.Required + strings.Join(apiState.MutualSSL.ClientCertificates, "") } - joinedUniqueIds = joinedUniqueIds + strconv.FormatBool(apiState.SubscriptionValidation) + mutualSSLUniqueId - loggers.LoggerAPK.Infof("Prepared unique string: %s", joinedUniqueIds) - hash := sha256.Sum256([]byte(joinedUniqueIds)) + joinedUniqueIDs = joinedUniqueIDs + strconv.FormatBool(apiState.SubscriptionValidation) + mutualSSLUniqueID + hash := sha256.Sum256([]byte(joinedUniqueIDs)) hashedString := hex.EncodeToString(hash[:]) truncatedHash := hashedString[:62] - loggers.LoggerAPK.Infof("Prepared hash: %s, truncated hash: %s", hashedString, truncatedHash) + loggers.LoggerAPK.Debugf("Prepared unique string for api %s/%s: %s, Prepared hash: %s, Truncatd hash to store: %s", apiState.APIDefinition.ObjectMeta.Name, + apiState.APIDefinition.ObjectMeta.Namespace, joinedUniqueIDs, hashedString, truncatedHash) return truncatedHash } @@ -2414,20 +2413,112 @@ func findProdSandEndpoints(apiState *synchronizer.APIState) (string, string, str endpointProtocol := "" if apiState.ProdHTTPRoute != nil { for _, backend := range apiState.ProdHTTPRoute.BackendMapping { - if (len(backend.Backend.Spec.Services) > 0) { - sandEndpoint = fmt.Sprintf("%s:%d", backend.Backend.Spec.Services[0].Host, backend.Backend.Spec.Services[0].Port) + if len(backend.Backend.Spec.Services) > 0 { + sandEndpoint = fmt.Sprintf("%s:%d", backend.Backend.Spec.Services[0].Host, backend.Backend.Spec.Services[0].Port) endpointProtocol = string(backend.Backend.Spec.Protocol) } } } if apiState.SandHTTPRoute != nil { for _, backend := range apiState.SandHTTPRoute.BackendMapping { - if (len(backend.Backend.Spec.Services) > 0) { - prodEndpoint = fmt.Sprintf("%s:%d", backend.Backend.Spec.Services[0].Host, backend.Backend.Spec.Services[0].Port) + if len(backend.Backend.Spec.Services) > 0 { + prodEndpoint = fmt.Sprintf("%s:%d", backend.Backend.Spec.Services[0].Host, backend.Backend.Spec.Services[0].Port) endpointProtocol = string(backend.Backend.Spec.Protocol) } } } - loggers.LoggerAPK.Infof("Returning endpoints: %s, %s", prodEndpoint, sandEndpoint) return prodEndpoint, sandEndpoint, endpointProtocol } + +func pickOneCorsForCP(apiState *synchronizer.APIState) *controlplane.CORSPolicy { + apiPolicies := []v1alpha2.APIPolicy{} + for _, apiPolicy := range apiState.APIPolicies { + apiPolicies = append(apiPolicies, apiPolicy) + } + for _, apiPolicy := range apiState.ResourceAPIPolicies { + apiPolicies = append(apiPolicies, apiPolicy) + } + for _, apiPolicy := range apiPolicies { + corsPolicy := v1alpha2.CORSPolicy{} + found := false + if apiPolicy.Spec.Override != nil && apiPolicy.Spec.Override.CORSPolicy != nil { + corsPolicy = *apiPolicy.Spec.Override.CORSPolicy + found = true + } else if apiPolicy.Spec.Default != nil && apiPolicy.Spec.Default.CORSPolicy != nil { + corsPolicy = *apiPolicy.Spec.Default.CORSPolicy + found = true + } + if found { + modifiedCors := controlplane.CORSPolicy{} + modifiedCors.AccessControlAllowCredentials = corsPolicy.AccessControlAllowCredentials + modifiedCors.AccessControlAllowHeaders = corsPolicy.AccessControlAllowHeaders + modifiedCors.AccessControlAllowOrigins = corsPolicy.AccessControlAllowOrigins + modifiedCors.AccessControlExposeHeaders = corsPolicy.AccessControlExposeHeaders + modifiedCors.AccessControlMaxAge = corsPolicy.AccessControlMaxAge + modifiedCors.AccessControlAllowMethods = corsPolicy.AccessControlAllowMethods + return &modifiedCors + } + } + return nil +} + +func getProdVhost(apiState *synchronizer.APIState) string { + if apiState.ProdHTTPRoute != nil { + for _, httpRoute := range apiState.ProdHTTPRoute.HTTPRoutePartitions { + if len(httpRoute.Spec.Hostnames) > 0 { + return string(httpRoute.Spec.Hostnames[0]) + } + } + } + return "default.gw.wso2.com" +} + +func prepareSecuritySchemeForCP(apiState *synchronizer.APIState) ([]string, string) { + var pickedAuth *v1alpha2.Authentication + authHeader := "Authorization" + for _, auth := range apiState.Authentications { + pickedAuth = &auth + break + } + if pickedAuth != nil { + var authSpec *v1alpha2.AuthSpec + if pickedAuth.Spec.Override != nil { + authSpec = pickedAuth.Spec.Override + } else { + authSpec = pickedAuth.Spec.Default + } + if authSpec != nil { + if authSpec.AuthTypes != nil { + authSchemes := []string{} + isAuthMandatory := false + isMTLSMandatory := false + if authSpec.AuthTypes.Oauth2.Required == "mandatory" { + isAuthMandatory = true + } + if !authSpec.AuthTypes.Oauth2.Disabled { + authSchemes = append(authSchemes, "oauth2") + if authSpec.AuthTypes.Oauth2.Header != "" { + authHeader = authSpec.AuthTypes.Oauth2.Header + } + } + if authSpec.AuthTypes.MutualSSL.Required == "mandatory" { + isMTLSMandatory = true + } + if !authSpec.AuthTypes.MutualSSL.Disabled { + authSchemes = append(authSchemes, "mutualssl") + } + if len(authSpec.AuthTypes.APIKey) > 0 { + authSchemes = append(authSchemes, "api_key") + } + if isAuthMandatory { + authSchemes = append(authSchemes, "oauth_basic_auth_api_key_mandatory") + } + if isMTLSMandatory { + authSchemes = append(authSchemes, "mutualssl_mandatory") + } + return authSchemes, authHeader + } + } + } + return []string{"oauth2", "oauth_basic_auth_api_key_mandatory"}, authHeader +} diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 80265707d..3dd1797c6 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -326,7 +326,7 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced } } } - + cachedAPI.APIDefinition = apiState.APIDefinition return *cachedAPI, events, updated }