Merge pull request #422 from kube-tarian/plugin-store-db

plugin store config persist

charts/kad/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.16
+version: 0.2.17
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.25.1"
+appVersion: "1.27.1"

charts/server/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.27
+version: 0.1.28
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.26.1"
+appVersion: "1.28.1"

charts/server/templates/deployment.yaml

@@ -27,6 +27,9 @@ spec:
       serviceAccountName: {{ include "server.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      volumes:
+        - name: plugin-store-clone-dir
+          emptyDir: {}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
@@ -69,8 +72,13 @@ spec:
               value: {{.Values.iam.address}}
             - name: CAPTEN_OAUTH_URL
               value: {{.Values.ory.captenOAuthURL}}
+            - name: PLUGIN_STORE_PROJECT_MOUNT
+              value: {{ .Values.env.pluginsStoreProjectMount }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+          - name: plugin-store-clone-dir
+            mountPath: {{ .Values.env.pluginsStoreProjectMount }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/server/values.yaml

@@ -29,6 +29,7 @@ ory:
 env:
   logLevel: info
   database: astra
+  pluginsStoreProjectMount: /plugin-store
 
 extraEnv:
   - name: AUTH_ENABLED

server/pkg/plugin-store/model.go

@@ -8,8 +8,8 @@ type Config struct {
 	PluginsStoreProjectMount string `envconfig:"PLUGIN_STORE_PROJECT_MOUNT" default:"/plugin-store-clone"`
 	PluginsStorePath         string `envconfig:"PLUGIN_STORE_PATH" default:"/plugin-store"`
 	PluginsFileName          string `envconfig:"PLUGIN_LIST_FILE" default:"plugin-list.yaml"`
-	PluginStoreProjectURL    string `envconfig:"PLUGIN_STORE_PROJECT_URL" default:"https://github.dev/vramk23/capten-plugins"`
-	PluginStoreProjectID     string `envconfig:"PLUGIN_STORE_PROJECT_ID" default:"1"`
+	PluginStoreProjectURL    string `envconfig:"PLUGIN_STORE_PROJECT_URL" default:"https://github.com/vramk23/capten-plugins"`
+	PluginStoreProjectID     string `envconfig:"PLUGIN_STORE_PROJECT_ID" default:"1cf5201d-5f35-4d5b-afe0-4b9d0e0d4cd2"`
 }
 
 type PluginListData struct {

server/pkg/plugin-store/plugin_store_handler.go

@@ -41,8 +41,8 @@ func (p *PluginStore) GetStoreConfig(clusterId string, storeType pluginstorepb.S
 	} else if storeType == pluginstorepb.StoreType_CENTRAL_STORE {
 		return &pluginstorepb.PluginStoreConfig{
 			StoreType:     pluginstorepb.StoreType_CENTRAL_STORE,
-			GitProjectId:  p.cfg.PluginStoreProjectURL,
-			GitProjectURL: p.cfg.PluginStoreProjectID,
+			GitProjectId:  p.cfg.PluginStoreProjectID,
+			GitProjectURL: p.cfg.PluginStoreProjectURL,
 		}, nil
 	} else {
 		return nil, fmt.Errorf("not supported store type")
@@ -55,14 +55,15 @@ func (p *PluginStore) SyncPlugins(clusterId string, storeType pluginstorepb.Stor
 		return err
 	}
 
-	pluginStoreDir, err := p.clonePluginStoreProject(config.GitProjectId, config.GitProjectURL)
+	pluginStoreDir, err := p.clonePluginStoreProject(config.GitProjectURL, config.GitProjectId)
 	if err != nil {
 		return err
 	}
 	defer os.RemoveAll(pluginStoreDir)
 
-	p.log.Infof("Loading plugin data from project %s clone dir %s", config.GitProjectURL, pluginStoreDir)
-	pluginListData, err := os.ReadFile(p.cfg.PluginsStorePath + "/" + p.cfg.PluginsFileName)
+	pluginListFilePath := pluginStoreDir + "/" + p.cfg.PluginsStorePath + "/" + p.cfg.PluginsFileName
+	p.log.Infof("Loading plugin data from %s", pluginListFilePath)
+	pluginListData, err := os.ReadFile(pluginListFilePath)
 	if err != nil {
 		return errors.WithMessage(err, "failed to read store config file")
 	}
@@ -73,7 +74,7 @@ func (p *PluginStore) SyncPlugins(clusterId string, storeType pluginstorepb.Stor
 	}
 
 	for _, pluginName := range plugins.Plugins {
-		err := p.addPluginApp(config.GitProjectId, pluginName)
+		err := p.addPluginApp(config.GitProjectId, pluginStoreDir, pluginName)
 		if err != nil {
 			p.log.Errorf("%v", err)
 			continue
@@ -86,45 +87,46 @@ func (p *PluginStore) SyncPlugins(clusterId string, storeType pluginstorepb.Stor
 func (p *PluginStore) clonePluginStoreProject(projectURL, _ string) (pluginStoreDir string, err error) {
 	pluginStoreDir, err = os.MkdirTemp(p.cfg.PluginsStoreProjectMount, tmpGitProjectCloneStr)
 	if err != nil {
-		err = fmt.Errorf("failed to create template tmp dir, err: %v", err)
+		err = fmt.Errorf("failed to create plugin store tmp dir, err: %v", err)
 		return
 	}
 
+	p.log.Infof("cloning plugin store project %s to %s", projectURL, pluginStoreDir)
 	gitClient := NewGitClient()
 	if err = gitClient.Clone(pluginStoreDir, projectURL, ""); err != nil {
 		os.RemoveAll(pluginStoreDir)
-		err = fmt.Errorf("failed to Clone template repo, err: %v", err)
+		err = fmt.Errorf("failed to Clone plugin store project, err: %v", err)
 		return
 	}
 	return
 }
 
-func (p *PluginStore) addPluginApp(gitProjectId, pluginName string) error {
-	appData, err := os.ReadFile(p.cfg.PluginsStorePath + "/" + pluginName + "/plugin.yaml")
+func (p *PluginStore) addPluginApp(gitProjectId, pluginStoreDir, pluginName string) error {
+	appData, err := os.ReadFile(pluginStoreDir + "/" + p.cfg.PluginsStorePath + "/" + pluginName + "/plugin.yaml")
 	if err != nil {
 		return errors.WithMessagef(err, "failed to read store plugin %s", pluginName)
 	}
 
-	var appConfig Plugin
-	if err := yaml.Unmarshal(appData, &appConfig); err != nil {
+	var pluginData Plugin
+	if err := yaml.Unmarshal(appData, &pluginData); err != nil {
 		return errors.WithMessagef(err, "failed to unmarshall store plugin %s", pluginName)
 	}
 
-	if appConfig.PluginName == "" || len(appConfig.DeploymentConfig.Versions) == 0 {
+	if pluginData.PluginName == "" || len(pluginData.DeploymentConfig.Versions) == 0 {
 		return fmt.Errorf("app name/version is missing for %s", pluginName)
 	}
 
 	plugin := &pluginstorepb.PluginData{
-		PluginName:          appConfig.PluginName,
-		Description:         appConfig.Description,
-		Category:            appConfig.Category,
-		ChartName:           appConfig.DeploymentConfig.ChartName,
-		ChartRepo:           appConfig.DeploymentConfig.ChartRepo,
-		Versions:            appConfig.DeploymentConfig.Versions,
-		DefaultNamespace:    appConfig.DeploymentConfig.DefaultNamespace,
-		PrivilegedNamespace: appConfig.DeploymentConfig.PrivilegedNamespace,
-		PluginEndpoint:      appConfig.PluginConfig.Endpoint,
-		Capabilities:        appConfig.PluginConfig.Capabilities,
+		PluginName:          pluginData.PluginName,
+		Description:         pluginData.Description,
+		Category:            pluginData.Category,
+		ChartName:           pluginData.DeploymentConfig.ChartName,
+		ChartRepo:           pluginData.DeploymentConfig.ChartRepo,
+		Versions:            pluginData.DeploymentConfig.Versions,
+		DefaultNamespace:    pluginData.DeploymentConfig.DefaultNamespace,
+		PrivilegedNamespace: pluginData.DeploymentConfig.PrivilegedNamespace,
+		PluginEndpoint:      pluginData.PluginConfig.Endpoint,
+		Capabilities:        pluginData.PluginConfig.Capabilities,
 	}
 
 	if err := p.dbStore.WritePluginData(gitProjectId, plugin); err != nil {

server/pkg/store/astra/plugin_store.go

@@ -2,6 +2,7 @@ package astra
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/kube-tarian/kad/server/pkg/pb/pluginstorepb"
@@ -10,24 +11,24 @@ import (
 )
 
 const (
-	insertStoreConfig           = `INSERT INTO %s.plugin_store_config (cluster_id uuid, store_type, git_project_id, git_project_url, last_updated_time) VALUES (%d, '%s', '%s', '%s') IF NOT EXISTS`
+	insertStoreConfig           = `INSERT INTO %s.plugin_store_config (cluster_id, store_type, git_project_id, git_project_url, last_updated_time) VALUES (%s, %d, '%s', '%s', '%s') IF NOT EXISTS`
 	readStoreConfigForStoreType = `SELECT store_type, git_project_id, git_project_url, last_updated_time FROM %s.plugin_store_config WHERE cluster_id = %s`
 
-	insertPluginData            = `INSERT INTO %s.plugin_data (git_project_id uuid, plugin_name, last_updated_time, store_type, description, category, icon, chart_name, chart_repo, versions, default_namespace, privileged_namespace, plugin_endpoint, capabilities) VALUES (%s, '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', %t, '%s', '%s') IF NOT EXISTS`
+	insertPluginData            = `INSERT INTO %s.plugin_data (git_project_id, plugin_name, last_updated_time, store_type, description, category, icon, chart_name, chart_repo, versions, default_namespace, privileged_namespace, plugin_endpoint, capabilities) VALUES (%s, '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', %v, '%s', %t, '%s', %v) IF NOT EXISTS`
 	readPlugins                 = `SELECT plugin_name, last_updated_time, store_type, description, category, icon, versions FROM %s.plugin_data WHERE git_project_id = %s`
 	readPluginDataForPluginName = `SELECT plugin_name, last_updated_time, store_type, description, category, icon, chart_name, chart_repo, versions, default_namespace, privileged_namespace, plugin_endpoint, capabilities FROM %s.plugin_data WHERE git_project_id = %s and plugin_name = '%s'`
 )
 
 func (a *AstraServerStore) WritePluginStoreConfig(clusterId string, config *pluginstorepb.PluginStoreConfig) error {
 	query := &pb.Query{
 		Cql: fmt.Sprintf(insertStoreConfig,
-			a.keyspace, config.StoreType, config.GitProjectId, config.GitProjectURL,
+			a.keyspace, clusterId, config.StoreType, config.GitProjectId, config.GitProjectURL,
 			time.Now().Format(time.RFC3339)),
 	}
 
 	_, err := a.c.Session().ExecuteQuery(query)
 	if err != nil {
-		return fmt.Errorf("failed to insert/update the app config into the app_config table : %w", err)
+		return fmt.Errorf("failed to insert/update the store config, %s, %w", query.Cql, err)
 	}
 	return nil
 }
@@ -39,7 +40,7 @@ func (a *AstraServerStore) ReadPluginStoreConfig(clusterId string) (*pluginstore
 
 	response, err := a.c.Session().ExecuteQuery(selectQuery)
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialise db: %w", err)
+		return nil, fmt.Errorf("failed to read store config, %w", err)
 	}
 
 	result := response.GetResultSet()
@@ -64,7 +65,7 @@ func toPluginStoreConfig(row *pb.Row) (*pluginstorepb.PluginStoreConfig, error)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get chart name: %w", err)
 	}
-	gitProjectURL, err := client.ToString(row.Values[1])
+	gitProjectURL, err := client.ToString(row.Values[2])
 	if err != nil {
 		return nil, fmt.Errorf("failed to get chart name: %w", err)
 	}
@@ -81,13 +82,14 @@ func (a *AstraServerStore) WritePluginData(gitProjectId string, pluginData *plug
 		Cql: fmt.Sprintf(insertPluginData,
 			a.keyspace, gitProjectId, pluginData.PluginName, time.Now().Format(time.RFC3339),
 			pluginData.StoreType, pluginData.Description, pluginData.Category, pluginData.Icon,
-			pluginData.ChartName, pluginData.ChartRepo, pluginData.Versions, pluginData.DefaultNamespace,
-			pluginData.PrivilegedNamespace, pluginData.PluginEndpoint, pluginData.Capabilities),
+			pluginData.ChartName, pluginData.ChartRepo, getSQLStringArray(pluginData.Versions),
+			pluginData.DefaultNamespace, pluginData.PrivilegedNamespace, pluginData.PluginEndpoint,
+			getSQLStringArray(pluginData.Capabilities)),
 	}
 
 	_, err := a.c.Session().ExecuteQuery(query)
 	if err != nil {
-		return fmt.Errorf("failed to insert/update the app config into the app_config table : %w", err)
+		return fmt.Errorf("failed to insert/update the plugin data, %s, %w", query.Cql, err)
 	}
 	return nil
 }
@@ -99,7 +101,7 @@ func (a *AstraServerStore) ReadPluginData(gitProjectId string, pluginName string
 
 	response, err := a.c.Session().ExecuteQuery(selectQuery)
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialise db: %w", err)
+		return nil, fmt.Errorf("failed to read plugin data, %w", err)
 	}
 
 	result := response.GetResultSet()
@@ -122,7 +124,7 @@ func (a *AstraServerStore) ReadPlugins(gitProjectId string) ([]*pluginstorepb.Pl
 
 	response, err := a.c.Session().ExecuteQuery(selectQuery)
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialise db: %w", err)
+		return nil, fmt.Errorf("failed to read plugin data, %w", err)
 	}
 
 	result := response.GetResultSet()
@@ -147,10 +149,10 @@ func toPluginData(row *pb.Row, columns []*pb.ColumnSpec) (*pluginstorepb.PluginD
 	if err != nil {
 		return nil, fmt.Errorf("failed to read plugin_name: %w", err)
 	}
-	_, err = client.ToString(row.Values[1])
+	/*_, err = client.ToDate(row.Values[1])
 	if err != nil {
 		return nil, fmt.Errorf("failed to read last_updated_time: %w", err)
-	}
+	}*/
 	storeType, err := client.ToInt(row.Values[2])
 	if err != nil {
 		return nil, fmt.Errorf("failed to read storeType: %w", err)
@@ -221,10 +223,10 @@ func toPlugin(row *pb.Row, columns []*pb.ColumnSpec) (*pluginstorepb.Plugin, err
 	if err != nil {
 		return nil, fmt.Errorf("failed to read plugin_name: %w", err)
 	}
-	_, err = client.ToString(row.Values[1])
+	/*_, err = client.ToDate(row.Values[1])
 	if err != nil {
 		return nil, fmt.Errorf("failed to read last_updated_time: %w", err)
-	}
+	}*/
 	storeType, err := client.ToInt(row.Values[2])
 	if err != nil {
 		return nil, fmt.Errorf("failed to read storeType: %w", err)
@@ -241,7 +243,7 @@ func toPlugin(row *pb.Row, columns []*pb.ColumnSpec) (*pluginstorepb.Plugin, err
 	if err != nil {
 		return nil, fmt.Errorf("failed to read icon: %w", err)
 	}
-	versions, err := client.ToList(row.Values[8], columns[8].Type)
+	versions, err := client.ToList(row.Values[6], columns[6].Type)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read versions: %w", err)
 	}
@@ -276,3 +278,7 @@ func convertToSlice(input interface{}) ([]string, error) {
 		return nil, fmt.Errorf("unsupported type: %T", input)
 	}
 }
+
+func getSQLStringArray(val []string) string {
+	return "['" + strings.Join(val, "', '") + "']"
+}

server/pkg/store/astra/store.go

@@ -32,14 +32,17 @@ func (a *AstraServerStore) InitializeDatabase() error {
 	initDbQueries := []string{
 		fmt.Sprintf(createCaptenClusterTableQuery, a.keyspace),
 		fmt.Sprintf(createAppConfigTableQuery, a.keyspace),
+		fmt.Sprintf(createPluginStoreConfigTableQuery, a.keyspace),
+		fmt.Sprintf(createPluginStoreTableQuery, a.keyspace),
 	}
 	return a.executeDBQueries(initDbQueries)
 }
 
 func (a *AstraServerStore) CleanupDatabase() error {
 	initDbQueries := []string{
-		fmt.Sprintf(dropCaptenClusterTableQuery, a.keyspace),
-		fmt.Sprintf(dropAppConfigTableQuery, a.keyspace),
+		//fmt.Sprintf(dropCaptenClusterTableQuery, a.keyspace),
+		//fmt.Sprintf(dropAppConfigTableQuery, a.keyspace),
+		fmt.Sprintf(dropPluginStoreTableQuery, a.keyspace),
 	}
 	return a.executeDBQueries(initDbQueries)
 }

server/pkg/store/astra/type.go

@@ -3,7 +3,7 @@ package astra
 const (
 	createCaptenClusterTableQuery     = "CREATE TABLE IF NOT EXISTS %s.capten_clusters (cluster_id uuid, org_id uuid, cluster_name text, endpoint text, PRIMARY KEY (org_id, cluster_id));"
 	createAppConfigTableQuery         = "CREATE TABLE IF NOT EXISTS %s.store_app_config(id TEXT, created_time timestamp, last_updated_time timestamp, last_updated_user TEXT, name TEXT, chart_name TEXT, repo_name TEXT, release_name TEXT, repo_url TEXT, namespace TEXT, version TEXT, create_namespace BOOLEAN, privileged_namespace BOOLEAN, launch_ui_url TEXT, launch_ui_redirect_url TEXT, category TEXT, icon TEXT, description TEXT, launch_ui_values TEXT, override_values TEXT, template_values TEXT, plugin_name TEXT, plugin_description TEXT, api_endpoint TEXT, PRIMARY KEY (name, version));"
-	createPluginStoreTableQuery       = "CREATE TABLE IF NOT EXISTS %s.plugin_data(git_project_id TEXT, plugin_name TEXT, last_updated_time timestamp, description TEXT, category TEXT, icon TEXT, chart_name TEXT, chart_repo TEXT, versions LIST<TEXT>, default_namespace TEXT, privileged_namespace BOOLEAN, plugin_endpoint TEXT, capabilities LIST<TEXT>, PRIMARY KEY (git_project_id, plugin_name));"
+	createPluginStoreTableQuery       = "CREATE TABLE IF NOT EXISTS %s.plugin_data(git_project_id uuid, plugin_name TEXT, last_updated_time timestamp, store_type INT, description TEXT, category TEXT, icon TEXT, chart_name TEXT, chart_repo TEXT, versions LIST<TEXT>, default_namespace TEXT, privileged_namespace BOOLEAN, plugin_endpoint TEXT, capabilities LIST<TEXT>, PRIMARY KEY (git_project_id, plugin_name));"
 	createPluginStoreConfigTableQuery = "CREATE TABLE IF NOT EXISTS %s.plugin_store_config(cluster_id uuid, store_type INT, git_project_id TEXT, git_project_url TEXT, last_updated_time timestamp, PRIMARY KEY (cluster_id));"
 	dropCaptenClusterTableQuery       = "DROP TABLE IF EXISTS %s.capten_clusters;"
 	dropAppConfigTableQuery           = "DROP TABLE IF EXISTS %s.store_app_config;"