Skip to content

Commit

Permalink
Add subscription based changes
Browse files Browse the repository at this point in the history
  • Loading branch information
Krishanx92 committed Sep 24, 2024
1 parent ad3a9c9 commit 8bc2ff1
Show file tree
Hide file tree
Showing 10 changed files with 197 additions and 34 deletions.
4 changes: 2 additions & 2 deletions apim-apk-agent/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ module github.com/wso2/product-apim-tooling/apim-apk-agent

go 1.22

toolchain go1.22.6
toolchain go1.22.7

require (
github.com/gin-gonic/gin v1.9.1
github.com/google/uuid v1.6.0
github.com/pelletier/go-toml v1.9.5
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
github.com/wso2/apk/common-go-libs v0.0.0-20240919082014-14a8b44a534b
github.com/wso2/apk/common-go-libs v0.0.0-20240920041902-85449a1c0150
google.golang.org/grpc v1.62.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/yaml.v2 v2.4.0
Expand Down
4 changes: 2 additions & 2 deletions apim-apk-agent/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,8 @@ github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqf
github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74=
github.com/wso2/apk/adapter v0.0.0-20240408123538-86a74d977eee h1:g0ivVkzybfcEkB0vBGTAXTUuMZpsF3zOTVtAgmW851s=
github.com/wso2/apk/adapter v0.0.0-20240408123538-86a74d977eee/go.mod h1:xYS5auF/YxnyRykw7NBSn/YR2FHD4hTeyav4Nhec8d0=
github.com/wso2/apk/common-go-libs v0.0.0-20240919082014-14a8b44a534b h1:no/3KBj0Lr1M8+z5+rFG2nPp7lvpXona9glRRZODCOQ=
github.com/wso2/apk/common-go-libs v0.0.0-20240919082014-14a8b44a534b/go.mod h1:SbZVA1jeiVG9dqk9fGcY/bB0JgEaQgtXqFAlxAfN0Lk=
github.com/wso2/apk/common-go-libs v0.0.0-20240920041902-85449a1c0150 h1:X3OezAh2UOxmQIRxsAua87nNqmoIGXx1yfQIvc4a+G4=
github.com/wso2/apk/common-go-libs v0.0.0-20240920041902-85449a1c0150/go.mod h1:SbZVA1jeiVG9dqk9fGcY/bB0JgEaQgtXqFAlxAfN0Lk=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
Expand Down
1 change: 1 addition & 0 deletions apim-apk-agent/internal/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ func Run(conf *config.Config) {
utilruntime.Must(gwapiv1.AddToScheme(scheme))
utilruntime.Must(dpv1alpha1.AddToScheme(scheme))
utilruntime.Must(dpv1alpha2.AddToScheme(scheme))
utilruntime.Must(dpv1alpha3.AddToScheme(scheme))
utilruntime.Must(cpv1alpha2.AddToScheme(scheme))
utilruntime.Must(cpv1alpha2.AddToScheme(scheme))
utilruntime.Must(dpv1alpha3.AddToScheme(scheme))
Expand Down
1 change: 1 addition & 0 deletions apim-apk-agent/internal/eventhub/marshaller.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ func MarshalSubscription(subscriptionInternal *types.Subscription) managementser
UUID: subscriptionInternal.SubscriptionUUID,
Organization: subscriptionInternal.ApplicationOrganization,
SubscribedAPI: &managementserver.SubscribedAPI{Name: subscriptionInternal.APIName, Version: subscriptionInternal.APIVersion},
RateLimit: subscriptionInternal.PolicyID,
TimeStamp: subscriptionInternal.TimeStamp,
}
return sub
Expand Down
32 changes: 32 additions & 0 deletions apim-apk-agent/internal/k8sClient/k8s_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
"sigs.k8s.io/gateway-api/apis/v1alpha2"
gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1alpha2"
)

// DeployAPICR applies the given API struct to the Kubernetes cluster.
Expand Down Expand Up @@ -397,6 +398,37 @@ func UpdateRateLimitPolicyCR(policy eventhubTypes.RateLimitPolicy, k8sClient cli
}
}

// DeploySubscriptionRateLimitPolicyCR applies the given RateLimitPolicies struct to the Kubernetes cluster.
func DeploySubscriptionRateLimitPolicyCR(policy eventhubTypes.SubscriptionPolicy, k8sClient client.Client) {
conf, _ := config.ReadConfigs()

crRateLimitPolicies := dpv1alpha3.RateLimitPolicy{
ObjectMeta: metav1.ObjectMeta{Name: policy.Name,
Namespace: conf.DataPlane.Namespace,
},
Spec: dpv1alpha3.RateLimitPolicySpec{
Override: &dpv1alpha3.RateLimitAPIPolicy{
Subscription: &dpv1alpha3.SubscriptionRateLimitPolicy{
StopOnQuotaReach: policy.StopOnQuotaReach,
Organization: policy.TenantDomain,
RequestCount: &dpv1alpha3.RequestCount{
RequestsPerUnit: uint32(policy.DefaultLimit.RequestCount.RequestCount),
Unit: policy.DefaultLimit.RequestCount.TimeUnit,
},
},
},
TargetRef: gwapiv1b1.PolicyTargetReference{Group: constants.GatewayGroup, Kind: "Subscription", Name: "default"},
},
}

if err := k8sClient.Create(context.Background(), &crRateLimitPolicies); err != nil {
loggers.LoggerK8sClient.Error("Unable to create RateLimitPolicies CR: " + err.Error())
} else {
loggers.LoggerK8sClient.Info("RateLimitPolicies CR created: " + crRateLimitPolicies.Name)
}

}

// DeployBackendCR applies the given Backends struct to the Kubernetes cluster.
func DeployBackendCR(backends *dpv1alpha2.Backend, k8sClient client.Client) {
crBackends := &dpv1alpha2.Backend{}
Expand Down
37 changes: 21 additions & 16 deletions apim-apk-agent/internal/messaging/notification_listener.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,6 @@ func processNotificationEvent(conf *config.Config, notification *msg.EventNotifi
"Hence dropping the event", err)
return err
}
logger.LoggerMessaging.Debugf("\n\n[%s]", decodedByte)
AgentMode := conf.Agent.Mode
eventType = notification.Event.PayloadData.EventType
if strings.Contains(eventType, apiLifeCycleChange) {
Expand Down Expand Up @@ -370,7 +369,6 @@ func marshalAppAttributes(attributes interface{}) map[string]string {
func handleSubscriptionEvents(data []byte, eventType string) {
var subscriptionEvent msg.SubscriptionEvent
subEventErr := json.Unmarshal([]byte(string(data)), &subscriptionEvent)
logger.LoggerAgent.Info("Subscription Event Received")
if subEventErr != nil {
logger.LoggerMessaging.Errorf("Error occurred while unmarshalling Subscription event data %v", subEventErr)
return
Expand All @@ -389,19 +387,19 @@ func handleSubscriptionEvents(data []byte, eventType string) {
SubStatus: subscriptionEvent.SubscriptionState,
Organization: subscriptionEvent.TenantDomain,
SubscribedApi: &event.SubscribedAPI{Name: subscriptionEvent.APIName, Version: subscriptionEvent.APIVersion},
RatelimitTier: subscriptionEvent.PolicyID,
}

applicationMapping := event.ApplicationMapping{Uuid: utils.GetUniqueIDOfApplicationMapping(subscriptionEvent.ApplicationUUID, subscriptionEvent.SubscriptionUUID), ApplicationRef: subscriptionEvent.ApplicationUUID, SubscriptionRef: subscriptionEvent.SubscriptionUUID, Organization: subscriptionEvent.TenantDomain}
if subscriptionEvent.Event.Type == subscriptionCreate {
subsEvent := event.Event{Uuid: uuid.New().String(), Type: constants.SubscriptionCreated, TimeStamp: subscriptionEvent.TimeStamp, Subscription: &subscription}
managementserver.AddSubscription(managementserver.Subscription{UUID: subscription.Uuid, SubStatus: subscription.SubStatus, Organization: subscription.Organization, SubscribedAPI: &managementserver.SubscribedAPI{Name: subscription.SubscribedApi.Name, Version: subscription.SubscribedApi.Version}})
managementserver.AddSubscription(managementserver.Subscription{UUID: subscription.Uuid, SubStatus: subscription.SubStatus, Organization: subscription.Organization, RateLimit: subscription.RatelimitTier, SubscribedAPI: &managementserver.SubscribedAPI{Name: subscription.SubscribedApi.Name, Version: subscription.SubscribedApi.Version}})
go utils.SendEvent(&subsEvent)
applicationMappingEvent := event.Event{Uuid: utils.GetUniqueIDOfApplicationMapping(subscriptionEvent.ApplicationUUID, subscriptionEvent.SubscriptionUUID), Type: constants.ApplicationMappingCreated, TimeStamp: subscriptionEvent.TimeStamp, ApplicationMapping: &applicationMapping}
managementserver.AddApplicationMapping(managementserver.ApplicationMapping{UUID: applicationMapping.Uuid, ApplicationRef: applicationMapping.ApplicationRef, SubscriptionRef: applicationMapping.SubscriptionRef, Organization: applicationMapping.Organization})
go utils.SendEvent(&applicationMappingEvent)
} else if subscriptionEvent.Event.Type == subscriptionUpdate {
subsEvent := event.Event{Uuid: uuid.New().String(), Type: constants.SubscriptionUpdated, TimeStamp: subscriptionEvent.TimeStamp, Subscription: &subscription}
managementserver.UpdateSubscription(subscription.Uuid, managementserver.Subscription{UUID: subscription.Uuid, SubStatus: subscription.SubStatus, Organization: subscription.Organization, SubscribedAPI: &managementserver.SubscribedAPI{Name: subscription.SubscribedApi.Name, Version: subscription.SubscribedApi.Version}})
managementserver.UpdateSubscription(subscription.Uuid, managementserver.Subscription{UUID: subscription.Uuid, SubStatus: subscription.SubStatus, Organization: subscription.Organization, RateLimit: subscription.RatelimitTier, SubscribedAPI: &managementserver.SubscribedAPI{Name: subscription.SubscribedApi.Name, Version: subscription.SubscribedApi.Version}})
go utils.SendEvent(&subsEvent)
applicationMappingEvent := event.Event{Uuid: utils.GetUniqueIDOfApplicationMapping(subscriptionEvent.ApplicationUUID, subscriptionEvent.SubscriptionUUID), Type: constants.ApplicationMappingUpdated, TimeStamp: subscriptionEvent.TimeStamp, ApplicationMapping: &applicationMapping}
managementserver.UpdateApplicationMapping(applicationMappingEvent.Uuid, managementserver.ApplicationMapping{UUID: applicationMappingEvent.Uuid, ApplicationRef: applicationMapping.ApplicationRef, SubscriptionRef: applicationMapping.SubscriptionRef, Organization: applicationMapping.Organization})
Expand Down Expand Up @@ -457,10 +455,17 @@ func handlePolicyEvents(data []byte, eventType string, c client.Client) {
}
// TODO: Handle policy events
if strings.EqualFold(eventType, policyCreate) {
logger.LoggerMessaging.Infof("Policy: %s for policy type: %s for tenant: %s", policyEvent.PolicyName, policyEvent.PolicyType, policyEvent.TenantDomain)
synchronizer.FetchRateLimitPoliciesOnEvent(policyEvent.PolicyName, policyEvent.TenantDomain, c)
ratelimitPolicies := managementserver.GetAllRateLimitPolicies()
logger.LoggerMessaging.Infof("Rate Limit Policies Internal Map: %v", ratelimitPolicies)
if strings.EqualFold(policyEvent.PolicyType, "API") {
logger.LoggerMessaging.Infof("Policy: %s for policy type: %s for tenant: %s", policyEvent.PolicyName, policyEvent.PolicyType, policyEvent.TenantDomain)
synchronizer.FetchRateLimitPoliciesOnEvent(policyEvent.PolicyName, policyEvent.TenantDomain, c)
ratelimitPolicies := managementserver.GetAllRateLimitPolicies()
logger.LoggerMessaging.Infof("Rate Limit Policies Internal Map: %v", ratelimitPolicies)
} else if strings.EqualFold(policyEvent.PolicyType, "SUBSCRIPTION") {
logger.LoggerMessaging.Infof("Policy: %s for policy type: %s", policyEvent.PolicyName, policyEvent.PolicyType)
synchronizer.FetchSubscriptionRateLimitPoliciesOnEvent(policyEvent.PolicyName, policyEvent.TenantDomain, c)
ratelimitPolicies := managementserver.GetAllRateLimitPolicies()
logger.LoggerMessaging.Infof("Rate Limit Policies Internal Map: %v", ratelimitPolicies)
}
} else if strings.EqualFold(eventType, policyUpdate) {
logger.LoggerMessaging.Infof("Policy: %s for policy type: %s for tenant: %s", policyEvent.PolicyName, policyEvent.PolicyType, policyEvent.TenantDomain)
synchronizer.FetchRateLimitPoliciesOnEvent(policyEvent.PolicyName, policyEvent.TenantDomain, c)
Expand Down Expand Up @@ -500,14 +505,14 @@ func handlePolicyEvents(data []byte, eventType string, c client.Client) {
return
}

subscriptionPolicy := types.SubscriptionPolicy{ID: subscriptionPolicyEvent.PolicyID, TenantID: -1,
Name: subscriptionPolicyEvent.PolicyName, QuotaType: subscriptionPolicyEvent.QuotaType,
GraphQLMaxComplexity: subscriptionPolicyEvent.GraphQLMaxComplexity,
GraphQLMaxDepth: subscriptionPolicyEvent.GraphQLMaxDepth, RateLimitCount: subscriptionPolicyEvent.RateLimitCount,
RateLimitTimeUnit: subscriptionPolicyEvent.RateLimitTimeUnit, StopOnQuotaReach: subscriptionPolicyEvent.StopOnQuotaReach,
TenantDomain: subscriptionPolicyEvent.TenantDomain, TimeStamp: subscriptionPolicyEvent.TimeStamp}
// subscriptionPolicy := types.SubscriptionPolicy{ID: subscriptionPolicyEvent.PolicyID, TenantID: -1,
// Name: subscriptionPolicyEvent.PolicyName, QuotaType: subscriptionPolicyEvent.QuotaType,
// GraphQLMaxComplexity: subscriptionPolicyEvent.GraphQLMaxComplexity,
// GraphQLMaxDepth: subscriptionPolicyEvent.GraphQLMaxDepth, RateLimitCount: subscriptionPolicyEvent.RateLimitCount,
// RateLimitTimeUnit: subscriptionPolicyEvent.RateLimitTimeUnit, StopOnQuotaReach: subscriptionPolicyEvent.StopOnQuotaReach,
// TenantDomain: subscriptionPolicyEvent.TenantDomain, TimeStamp: subscriptionPolicyEvent.TimeStamp}

logger.LoggerMessaging.Debugf("SubscriptionPolicy event data %v", subscriptionPolicy)
// logger.LoggerMessaging.Debugf("SubscriptionPolicy event data %v", subscriptionPolicy)

// var subscriptionPolicyList *subscription.SubscriptionPolicyList
// if subscriptionPolicyEvent.Event.Type == policyCreate {
Expand Down
119 changes: 117 additions & 2 deletions apim-apk-agent/internal/synchronizer/ratelimit_policy_fetcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,10 @@ import (
)

const (
policiesEndpoint string = "internal/data/v1/api-policies"
policiesByNameEndpoint string = "internal/data/v1/api-policies?policyName="
policiesEndpoint string = "internal/data/v1/api-policies"
policiesByNameEndpoint string = "internal/data/v1/api-policies?policyName="
subscriptionsPoliciesEndpoint string = "internal/data/v1//subscription-policies"
subscriptionsPoliciesByNameEndpoint string = "internal/data/v1//subscription-policies?policyName="
)

// FetchRateLimitPoliciesOnEvent fetches the policies from the control plane on the start up and notification event updates
Expand Down Expand Up @@ -74,6 +76,7 @@ func FetchRateLimitPoliciesOnEvent(ratelimitName string, organization string, c
ehURL += "/" + policiesEndpoint
}
}

logger.LoggerSynchronizer.Debugf("Fetching RateLimit Policies from the URL %v: ", ehURL)

ehUname := ehConfigs.Username
Expand Down Expand Up @@ -158,6 +161,118 @@ func FetchRateLimitPoliciesOnEvent(ratelimitName string, organization string, c
}
}

// FetchSubscriptionRateLimitPoliciesOnEvent fetches the policies from the control plane on the start up and notification event updates
func FetchSubscriptionRateLimitPoliciesOnEvent(ratelimitName string, organization string, c client.Client) {
logger.LoggerSynchronizer.Info("Fetching RateLimit Policies from Control Plane.")

// Read configurations and derive the eventHub details
conf, errReadConfig := config.ReadConfigs()
if errReadConfig != nil {
// This has to be error. For debugging purpose info
logger.LoggerSynchronizer.Errorf("Error reading configs: %v", errReadConfig)
}
// Populate data from the config
ehConfigs := conf.ControlPlane
ehURL := ehConfigs.ServiceURL
// If the eventHub URL is configured with trailing slash
if strings.HasSuffix(ehURL, "/") {
if ratelimitName != "" {
ehURL += subscriptionsPoliciesByNameEndpoint + ratelimitName
} else {
ehURL += subscriptionsPoliciesEndpoint
}
} else {
if ratelimitName != "" {
ehURL += "/" + subscriptionsPoliciesByNameEndpoint + ratelimitName
} else {
ehURL += "/" + subscriptionsPoliciesEndpoint
}
}

logger.LoggerSynchronizer.Infof("Fetching RateLimit Policies from the URL %v: ", ehURL)

ehUname := ehConfigs.Username
ehPass := ehConfigs.Password
basicAuth := "Basic " + pkgAuth.GetBasicAuth(ehUname, ehPass)

// Check if TLS is enabled
skipSSL := ehConfigs.SkipSSLVerification

// Create a HTTP request
req, err := http.NewRequest("GET", ehURL, nil)
if err != nil {
logger.LoggerSynchronizer.Errorf("Error while creating http request for RateLimit Policies Endpoint : %v", err)
}

var queryParamMap map[string]string

if queryParamMap != nil && len(queryParamMap) > 0 {
q := req.URL.Query()
// Making necessary query parameters for the request
for queryParamKey, queryParamValue := range queryParamMap {
q.Add(queryParamKey, queryParamValue)
}
req.URL.RawQuery = q.Encode()
}
// Setting authorization header
req.Header.Set(sync.Authorization, basicAuth)

if organization != "" {
logger.LoggerSynchronizer.Debugf("Setting the organization header for the request: %v", organization)
req.Header.Set("xWSO2Tenant", organization)
} else {
logger.LoggerSynchronizer.Debugf("Setting the organization header for the request: %v", "ALL")
req.Header.Set("xWSO2Tenant", "ALL")
}

// Make the request
logger.LoggerSynchronizer.Debug("Sending the control plane request")
resp, err := tlsutils.InvokeControlPlane(req, skipSSL)
var errorMsg string
if err != nil {
errorMsg = "Error occurred while calling the REST API: " + policiesEndpoint
go retryRLPFetchData(conf, errorMsg, err, c)
return
}
responseBytes, err := ioutil.ReadAll(resp.Body)
logger.LoggerSynchronizer.Debugf("Response String received for Policies: %v", string(responseBytes))

if err != nil {
errorMsg = "Error occurred while reading the response received for: " + policiesEndpoint
go retryRLPFetchData(conf, errorMsg, err, c)
return
}

if resp.StatusCode == http.StatusOK {
var rateLimitPolicyList eventhubTypes.SubscriptionPolicyList
err := json.Unmarshal(responseBytes, &rateLimitPolicyList)
if err != nil {
logger.LoggerSynchronizer.Errorf("Error occurred while unmarshelling RateLimit Policies event data %v", err)
return
}
logger.LoggerSynchronizer.Debugf("Policies received: %v", rateLimitPolicyList.List)
var rateLimitPolicies []eventhubTypes.SubscriptionPolicy = rateLimitPolicyList.List
for _, policy := range rateLimitPolicies {
if policy.DefaultLimit.RequestCount.TimeUnit == "min" {
policy.DefaultLimit.RequestCount.TimeUnit = "Minute"
} else if policy.DefaultLimit.RequestCount.TimeUnit == "hours" {
policy.DefaultLimit.RequestCount.TimeUnit = "Hour"
} else if policy.DefaultLimit.RequestCount.TimeUnit == "days" {
policy.DefaultLimit.RequestCount.TimeUnit = "Day"
}
managementserver.AddSubscriptionPolicy(policy)
logger.LoggerSynchronizer.Infof("RateLimit Policy added to internal map: %v", policy)
// Update the exisitng rate limit policies with current policy
k8sclient.DeploySubscriptionRateLimitPolicyCR(policy, c)

}
} else {
errorMsg = "Failed to fetch data! " + policiesEndpoint + " responded with " +
strconv.Itoa(resp.StatusCode)
go retryRLPFetchData(conf, errorMsg, err, c)
}
}

func retryRLPFetchData(conf *config.Config, errorMessage string, err error, c client.Client) {
logger.LoggerSynchronizer.Debugf("Time Duration for retrying: %v",
conf.ControlPlane.RetryInterval*time.Second)
Expand Down
25 changes: 13 additions & 12 deletions apim-apk-agent/pkg/eventhub/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,22 +113,23 @@ type ApplicationPolicyList struct {

// SubscriptionPolicy for struct list of SubscriptionPolicy
type SubscriptionPolicy struct {
ID int32 `json:"id" json:"policyId"`
TenantID int32 `json:"tenantId"`
Name string `json:"name"`
QuotaType string `json:"quotaType"`
GraphQLMaxComplexity int32 `json:"graphQLMaxComplexity"`
GraphQLMaxDepth int32 `json:"graphQLMaxDepth"`
RateLimitCount int32 `json:"rateLimitCount"`
RateLimitTimeUnit string `json:"rateLimitTimeUnit"`
StopOnQuotaReach bool `json:"stopOnQuotaReach"`
TenantDomain string `json:"tenanDomain,omitempty"`
TimeStamp int64 `json:"timeStamp,omitempty"`
TenantID int32 `json:"tenantId"`
TenantDomain string `json:"tenantDomain,omitempty"`
Name string `json:"name"`
QuotaType string `json:"quotaType"`
GraphQLMaxComplexity int32 `json:"graphQLMaxComplexity"`
GraphQLMaxDepth int32 `json:"graphQLMaxDepth"`
RateLimitCount int32 `json:"rateLimitCount"`
RateLimitTimeUnit string `json:"rateLimitTimeUnit"`
StopOnQuotaReach bool `json:"stopOnQuotaReach"`
DefaultLimit DefaultLimit `json:"defaultLimit"`
TimeStamp int64 `json:"timeStamp,omitempty"`
}

// SubscriptionPolicyList for struct list of SubscriptionPolicy
type SubscriptionPolicyList struct {
List []SubscriptionPolicy `json:"list"`
Count int `json:"count"`
List []SubscriptionPolicy `json:"list"`
}

// AIProviderList for struct list of AIProvider
Expand Down
Loading

0 comments on commit 8bc2ff1

Please sign in to comment.