sync relase 0.6.0 (#249)

* repo-sync-2024-02-04T16:00:21+0800

* fix conflict

* fix unit test

* fix nil pointer

* repo-sync-2024-03-16T09:53:20+0800

---------

Co-authored-by: UniqueMarvin <jiamingyang.jmy@antgroup.com>

build/dockerfile/kuscia-anolis.Dockerfile

@@ -1,5 +1,5 @@
 ARG DEPS_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0"
-ARG KUSCIA_ENVOY_IMAGE="secretflow/kuscia-envoy:0.3.0.dev231122"
+ARG KUSCIA_ENVOY_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.3.0.b0"
 ARG PROM_NODE_EXPORTER="prom/node-exporter:v1.7.0"
 
 FROM ${DEPS_IMAGE} as deps

cmd/kuscia/confloader/kuscia_config.go

@@ -139,9 +139,16 @@ func (lite *LiteKusciaConfig) OverwriteKusciaConfig(kusciaConfig *KusciaConfig)
 	kusciaConfig.Agent.Provider.K8s.LabelsToAdd = lite.Agent.Provider.K8s.LabelsToAdd
 	kusciaConfig.Agent.Provider.K8s.AnnotationsToAdd = lite.Agent.Provider.K8s.AnnotationsToAdd
 	kusciaConfig.Agent.Capacity = lite.Capacity
-	if len(lite.Agent.Plugins) > 0 {
-		kusciaConfig.Agent.Plugins = lite.Agent.Plugins
+
+	for _, p := range lite.Agent.Plugins {
+		for j, pp := range kusciaConfig.Agent.Plugins {
+			if p.Name == pp.Name {
+				kusciaConfig.Agent.Plugins[j] = p
+				break
+			}
+		}
 	}
+
 	kusciaConfig.Master.Endpoint = lite.MasterEndpoint
 	kusciaConfig.DomainRoute.DomainCsrData = generateCsrData(lite.DomainID, lite.DomainKeyData, lite.LiteDeployToken)
 	kusciaConfig.Debug = lite.Debug
@@ -181,9 +188,16 @@ func (autonomy *AutomonyKusciaConfig) OverwriteKusciaConfig(kusciaConfig *Kuscia
 	kusciaConfig.Agent.Provider.K8s.LabelsToAdd = autonomy.Agent.Provider.K8s.LabelsToAdd
 	kusciaConfig.Agent.Provider.K8s.AnnotationsToAdd = autonomy.Agent.Provider.K8s.AnnotationsToAdd
 	kusciaConfig.Agent.Capacity = autonomy.Capacity
-	if len(autonomy.Agent.Plugins) > 0 {
-		kusciaConfig.Agent.Plugins = autonomy.Agent.Plugins
+
+	for _, p := range autonomy.Agent.Plugins {
+		for j, pp := range kusciaConfig.Agent.Plugins {
+			if p.Name == pp.Name {
+				kusciaConfig.Agent.Plugins[j] = p
+				break
+			}
+		}
 	}
+
 	kusciaConfig.ConfLoaders = autonomy.ConfLoaders
 	kusciaConfig.SecretBackends = autonomy.SecretBackends
 	if autonomy.KusciaAPI != nil {

cmd/kuscia/image/cmd/cmd.go

@@ -28,4 +28,5 @@ type Context struct {
 func InstallCommands(rootCmd *cobra.Command, cmdCtx *Context) {
 	rootCmd.AddCommand(loadCommand(cmdCtx))
 	rootCmd.AddCommand(builtinCommand(cmdCtx))
+	rootCmd.AddCommand(mountCommand(cmdCtx))
 }

cmd/kuscia/image/cmd/mount.go

@@ -0,0 +1,59 @@
+// Copyright 2024 Ant Group Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/secretflow/kuscia/pkg/agent/local/store/kii"
+	"github.com/secretflow/kuscia/pkg/agent/local/store/layout"
+	"github.com/secretflow/kuscia/pkg/utils/nlog"
+)
+
+func mountCommand(cmdCtx *Context) *cobra.Command {
+	mountCmd := &cobra.Command{
+		Use:                   "mount MOUNT_ID IMAGE TARGET_PATH",
+		Short:                 "mount a image to path",
+		DisableFlagsInUseLine: true,
+		Example: `
+# Mount image to dir
+kuscia image mount unique_id app:0.1 target_dir/
+`,
+		Args: cobra.ExactArgs(3),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := mountImage(cmdCtx, args[0], args[1], args[2]); err != nil {
+				nlog.Fatal(err)
+			}
+		},
+	}
+
+	return mountCmd
+}
+
+func mountImage(cmdCtx *Context, mountID, image, targetPath string) error {
+	imageName, err := kii.NewImageName(image)
+	if err != nil {
+		return err
+	}
+
+	bundle, err := layout.NewBundle(cmdCtx.StorageDir)
+	if err != nil {
+		return err
+	}
+
+	rBundle := bundle.GetContainerBundle(mountID)
+
+	return cmdCtx.Store.MountImage(imageName, kii.Plain, rBundle.GetFsWorkingDirPath(), targetPath)
+}

cmd/kuscia/image/cmd/mount_test.go

@@ -0,0 +1,38 @@
+// Copyright 2024 Ant Group Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	storetesting "github.com/secretflow/kuscia/pkg/agent/local/store/testing"
+)
+
+func TestMountCommand(t *testing.T) {
+	rootDir := t.TempDir()
+
+	context := &Context{
+		StorageDir: filepath.Join(rootDir, "storage"),
+		Store:      storetesting.NewFakeStore(),
+	}
+
+	cmd := mountCommand(context)
+	args := []string{"123", "app:0.1", filepath.Join(rootDir, "rootfs")}
+	cmd.SetArgs(args)
+	assert.NoError(t, cmd.Execute())
+}

cmd/kuscia/modules/datamesh_test.go

@@ -0,0 +1,26 @@
+// Copyright 2024 Ant Group Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"context"
+	"testing"
+)
+
+func Test_RunDataMesh(t *testing.T) {
+	runCtx, cancel := context.WithCancel(context.Background())
+	dependency := mockDependency(t)
+	RunDataMesh(runCtx, cancel, dependency)
+}

cmd/kuscia/modules/domainroute_test.go

@@ -0,0 +1,61 @@
+// Copyright 2024 Ant Group Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	clientsetfake "k8s.io/client-go/kubernetes/fake"
+	restclient "k8s.io/client-go/rest"
+
+	kusciaclientsetfake "github.com/secretflow/kuscia/pkg/crd/clientset/versioned/fake"
+	"github.com/secretflow/kuscia/pkg/utils/kubeconfig"
+	"github.com/secretflow/kuscia/pkg/utils/kusciaconfig"
+	"github.com/secretflow/kuscia/pkg/utils/nlog"
+)
+
+func Test_RunDomainRoute(t *testing.T) {
+	tmpDir := t.TempDir()
+	path, err := os.Getwd()
+	assert.NoError(t, err)
+	workDir := strings.SplitN(path, "cmd", 2)[0]
+	nlog.Infof("work dir is: %s", workDir)
+	assert.NoError(t, err)
+	dependency := mockDependency(t)
+	dependency.RootDir = workDir
+	dependency.Master = kusciaconfig.MasterConfig{
+		APIServer: &kusciaconfig.APIServerConfig{
+			KubeConfig: filepath.Join(tmpDir, "etc/kubeconfig"),
+			Endpoint:   "https://127.0.0.1:6443",
+		},
+		KusciaAPI: &kusciaconfig.ServiceConfig{
+			Endpoint: "http://127.0.0.1:8092",
+		},
+	}
+	dependency.Clients = &kubeconfig.KubeClients{
+		KubeClient:   clientsetfake.NewSimpleClientset(),
+		KusciaClient: kusciaclientsetfake.NewSimpleClientset(),
+		Kubeconfig: &restclient.Config{
+			Host: "https://127.0.0.1:6443",
+		},
+	}
+	runCtx, cancel := context.WithCancel(context.Background())
+	RunDomainRoute(runCtx, cancel, dependency)
+}

cmd/kuscia/modules/envoy_test.go

@@ -0,0 +1,35 @@
+// Copyright 2024 Ant Group Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"context"
+	"testing"
+
+	"github.com/secretflow/kuscia/cmd/kuscia/confloader"
+)
+
+func TestRunEnvoy(t *testing.T) {
+	dependency := &Dependencies{
+		KusciaConfig: confloader.KusciaConfig{
+			RootDir:  "./",
+			DomainID: "alice",
+		},
+	}
+
+	runCtx, cancel := context.WithCancel(context.Background())
+
+	RunEnvoy(runCtx, cancel, dependency)
+}

cmd/kuscia/modules/kusciaapi_test.go

@@ -0,0 +1,26 @@
+// Copyright 2024 Ant Group Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"context"
+	"testing"
+)
+
+func Test_RunKusciaAPI(t *testing.T) {
+	runCtx, cancel := context.WithCancel(context.Background())
+	dependency := mockDependency(t)
+	RunKusciaAPI(runCtx, cancel, dependency)
+}

cmd/kuscia/modules/modules_test.go

@@ -15,15 +15,21 @@
 package modules
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
+	clientsetfake "k8s.io/client-go/kubernetes/fake"
 
 	"github.com/secretflow/kuscia/cmd/kuscia/confloader"
+	kusciaclientsetfake "github.com/secretflow/kuscia/pkg/crd/clientset/versioned/fake"
+	"github.com/secretflow/kuscia/pkg/kusciaapi/config"
+	"github.com/secretflow/kuscia/pkg/utils/kubeconfig"
+	"github.com/secretflow/kuscia/pkg/utils/tls"
 
 	"github.com/secretflow/kuscia/pkg/utils/common"
 )
@@ -89,3 +95,42 @@ externalTLS:
 	err := yaml.Unmarshal([]byte(content), config)
 	assert.NoError(t, err)
 }
+
+func Test_InitDependencies(t *testing.T) {
+	rootDir := t.TempDir()
+	domainKeyData, err := tls.GenerateKeyData()
+	assert.NoError(t, err)
+	config := confloader.KusciaConfig{
+		RootDir:        rootDir,
+		DomainID:       "alice",
+		DomainKeyData:  domainKeyData,
+		DomainKeyFile:  filepath.Join(rootDir, "domain.key"),
+		DomainCertFile: filepath.Join(rootDir, "domain.crt"),
+		CAKeyData:      domainKeyData,
+		CAKeyFile:      filepath.Join(rootDir, "ca.key"),
+		CACertFile:     filepath.Join(rootDir, "ca.crt"),
+	}
+	InitDependencies(context.Background(), config)
+}
+
+func mockDependency(t *testing.T) *Dependencies {
+	rootDir := t.TempDir()
+	domainKeyData, err := tls.GenerateKeyData()
+	assert.NoError(t, err)
+	dependency := InitDependencies(context.Background(), confloader.KusciaConfig{
+		RootDir:        rootDir,
+		DomainID:       "alice",
+		DomainKeyData:  domainKeyData,
+		DomainKeyFile:  filepath.Join(rootDir, "domain.key"),
+		DomainCertFile: filepath.Join(rootDir, "domain.crt"),
+		CAKeyData:      domainKeyData,
+		CAKeyFile:      filepath.Join(rootDir, "ca.key"),
+		CACertFile:     filepath.Join(rootDir, "ca.crt"),
+		KusciaAPI:      config.NewDefaultKusciaAPIConfig(rootDir),
+	})
+	dependency.Clients = &kubeconfig.KubeClients{
+		KubeClient:   clientsetfake.NewSimpleClientset(),
+		KusciaClient: kusciaclientsetfake.NewSimpleClientset(),
+	}
+	return dependency
+}

cmd/kuscia/modules/ssexporter.go

@@ -40,7 +40,7 @@ func NewSsExporter(i *Dependencies) Module {
 		domainID:           i.DomainID,
 		rootDir:            i.RootDir,
 		metricUpdatePeriod: i.MetricUpdatePeriod,
-		ssExportPort:       string(i.SsExportPort),
+		ssExportPort:       i.SsExportPort,
 	}
 }