diff --git a/pkg/providers/k3d/docker.go b/pkg/providers/k3d/docker.go new file mode 100644 index 00000000..296b9700 --- /dev/null +++ b/pkg/providers/k3d/docker.go @@ -0,0 +1,135 @@ +package k3d + +import ( + "context" + "fmt" + "net" + "net/url" + "os" + "strings" + "sync" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/k3d-io/k3d/v5/pkg/runtimes" + "github.com/k3d-io/k3d/v5/pkg/runtimes/docker" + "github.com/sirupsen/logrus" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +var ( + dockerHost string + once sync.Once +) + +func getDockerHost() string { + once.Do(func() { + var err error + if runtimes.SelectedRuntime.ID() != "docker" { + logrus.Debugf("runtime not docker") + return + } + // TODO find a better way to get container id via docker.sock + // Hostname as container id is not 100% reliable. + hostname := os.Getenv("HOSTNAME_OVERRIDE") + if hostname == "" { + hostname, err = os.Hostname() + if err != nil { + logrus.Debugf("failed to get hostname, %v", err) + return + } + } + + runtime := runtimes.Docker + + if runtime.GetHost() != "" { + logrus.Debugf("runtime host is %s", runtime.GetHost()) + return + } + + if _, err = os.Stat(runtime.GetRuntimePath()); err != nil { + logrus.Debugf("runtime path %s doesn't exist", runtime.GetRuntimePath()) + return + } + nodes, err := getContainersByLabel(context.Background(), map[string]string{ + "org.opencontainers.image.title": "autok3s", + }) + if err != nil { + logrus.Debugf("failed to get container from runtime %s, %v", runtime.ID(), err) + return + } + if len(nodes) == 0 { + logrus.Debug("autok3s docker container not found. Skip finding docker host IP.") + return + } + var currentContainer *types.Container + for i := range nodes { + node := nodes[i] + if strings.HasPrefix(node.ID, hostname) { + currentContainer = &node + break + } + } + if currentContainer == nil { + logrus.Debugf("no container found for hostname %s", hostname) + return + } + if currentContainer.HostConfig.NetworkMode == "host" { + logrus.Debug("do nothing when running host network") + return + } + logrus.Debugf("found container %s", currentContainer.ID) + gw, err := runtime.GetHostIP(context.Background(), currentContainer.HostConfig.NetworkMode) + if err != nil { + logrus.Debugf("failed to get gateway ip for network %s, %v", currentContainer.HostConfig.NetworkMode, err) + return + } + dockerHost = gw.String() + logrus.Infof("found docker host IP %s", dockerHost) + }) + return dockerHost +} + +func getContainersByLabel(ctx context.Context, labels map[string]string) ([]types.Container, error) { + // (0) create docker client + docker, err := docker.GetDockerClient() + if err != nil { + return nil, fmt.Errorf("Failed to create docker client. %+v", err) + } + defer docker.Close() + + filters := filters.NewArgs() + for k, v := range labels { + filters.Add("label", fmt.Sprintf("%s=%s", k, v)) + } + + containers, err := docker.ContainerList(ctx, container.ListOptions{ + Filters: filters, + All: true, + }) + if err != nil { + return nil, fmt.Errorf("failed to list containers: %w", err) + } + + return containers, nil +} + +func OverrideK3dKubeConfigServer(from, to string, config *clientcmdapi.Config) { + if to == "" { + return + } + if from == "" && getDockerHost() != "" { + from = getDockerHost() + } + for key := range config.Clusters { + cluster := config.Clusters[key] + serverURL, _ := url.Parse(cluster.Server) + if serverURL.Hostname() == from { + _, port, _ := net.SplitHostPort(serverURL.Host) + serverURL.Host = fmt.Sprintf("%s:%s", to, port) + } + cluster.Server = serverURL.String() + return + } +} diff --git a/pkg/providers/k3d/k3d.go b/pkg/providers/k3d/k3d.go index e3596502..ca6c972f 100644 --- a/pkg/providers/k3d/k3d.go +++ b/pkg/providers/k3d/k3d.go @@ -45,6 +45,8 @@ var ( type K3d struct { *cluster.ProviderBase `json:",inline"` typesk3d.Options `json:",inline"` + + dockerHost, additionalHost string } func init() { @@ -62,6 +64,7 @@ func newProvider() *K3d { APIPort: k3dAPIPort, Image: k3dImage, }, + dockerHost: getDockerHost(), } } @@ -173,6 +176,11 @@ func (p *K3d) GetProviderOptions(opt []byte) (interface{}, error) { // SetConfig set cluster config. func (p *K3d) SetConfig(config []byte) error { + tmpHost := struct { + AdditionalHost string `json:"additionalHost,omitempty"` + }{} + _ = json.Unmarshal(config, &tmpHost) + c, err := p.SetClusterConfig(config) if err != nil { return err @@ -189,7 +197,7 @@ func (p *K3d) SetConfig(config []byte) error { } targetOption := reflect.ValueOf(opt).Elem() utils.MergeConfig(sourceOption, targetOption) - + p.additionalHost = tmpHost.AdditionalHost return nil } @@ -326,6 +334,8 @@ func (p *K3d) obtainKubeCfg() (kubeCfg, ip string, err error) { return } + OverrideK3dKubeConfigServer(k3d.DefaultAPIHost, p.dockerHost, kubeConfig) + bytes, err := clientcmd.Write(*kubeConfig) if err != nil { return @@ -611,20 +621,15 @@ func (p *K3d) wrapCliFlags(masters, workers int) (*k3dconf.ClusterConfig, error) } if p.APIPort != "" { - exposeAPI, err := k3dutil.ParsePortExposureSpec(p.APIPort, k3d.DefaultAPIPort) + apiPort := p.APIPort + if strings.HasSuffix(apiPort, ":0") { + apiPort = strings.TrimSuffix(apiPort, ":0") + ":random" + } + exposeAPI, err := k3dutil.ParsePortExposureSpec(apiPort, k3d.DefaultAPIPort) if err != nil { return nil, fmt.Errorf("[%s] cluster %s parse port config failed: %w", p.GetProviderName(), p.Name, err) } - cfg.ExposeAPI.HostIP = exposeAPI.Binding.HostIP - - if exposeAPI.Binding.HostPort == "0" { - exposeAPI, err = k3dutil.ParsePortExposureSpec("random", k3d.DefaultAPIPort) - if err != nil { - return nil, fmt.Errorf("[%s] cluster %s parse random port config failed: %w", p.GetProviderName(), p.Name, err) - } - } - cfg.ExposeAPI.HostPort = exposeAPI.Binding.HostPort p.APIPort = fmt.Sprintf("%s:%s", cfg.ExposeAPI.HostIP, cfg.ExposeAPI.HostPort) } @@ -645,6 +650,13 @@ func (p *K3d) wrapCliFlags(masters, workers int) (*k3dconf.ClusterConfig, error) cfg.Options.Runtime.AgentsMemory = p.WorkersMemory } + for _, host := range []string{p.additionalHost, p.dockerHost} { + if host == "" { + continue + } + p.MasterExtraArgs = strings.TrimPrefix(p.MasterExtraArgs+" --tls-san="+host, " ") + } + if p.MasterExtraArgs != "" { cfg.Options.K3sOptions.ExtraArgs = []k3dconf.K3sArgWithNodeFilters{} for _, arg := range strings.Split(p.MasterExtraArgs, " ") { diff --git a/pkg/server/store/cluster/action.go b/pkg/server/store/cluster/action.go index f183fe9f..9d1d86a0 100644 --- a/pkg/server/store/cluster/action.go +++ b/pkg/server/store/cluster/action.go @@ -5,16 +5,20 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" "path/filepath" + "strings" "github.com/cnrancher/autok3s/pkg/common" "github.com/cnrancher/autok3s/pkg/providers" + "github.com/cnrancher/autok3s/pkg/providers/k3d" autok3stypes "github.com/cnrancher/autok3s/pkg/types/apis" "github.com/gorilla/mux" "github.com/rancher/apiserver/pkg/apierror" "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/apiserver/pkg/urlbuilder" "github.com/rancher/wrangler/v2/pkg/schemas/validation" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" @@ -256,6 +260,14 @@ func (d downloadKubeconfig) ServeHTTP(_ http.ResponseWriter, req *http.Request) } } + if strings.HasPrefix(clusterID, "k3d-") { + host, _, _ := net.SplitHostPort(urlbuilder.GetHost(req, "")) + // When parsing empty string as from parameter, the dockerHost will be used as the origin server + // if the dockerHost is empty(e.g. DOCKER_HOST is set), this function will do nothing as the k3d already use the + // proper cluster server address in kubeconfig + k3d.OverrideK3dKubeConfigServer("", host, ¤tCfg) + } + result, err := clientcmd.Write(currentCfg) if err != nil { apiRequest.WriteError(apierror.NewAPIError(validation.ServerError, err.Error())) diff --git a/pkg/server/store/cluster/store.go b/pkg/server/store/cluster/store.go index ee4ff5ab..881d14c1 100644 --- a/pkg/server/store/cluster/store.go +++ b/pkg/server/store/cluster/store.go @@ -3,6 +3,7 @@ package cluster import ( "encoding/json" "fmt" + "net" "strings" "github.com/cnrancher/autok3s/pkg/cluster" @@ -14,6 +15,7 @@ import ( "github.com/rancher/apiserver/pkg/apierror" "github.com/rancher/apiserver/pkg/store/empty" "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/apiserver/pkg/urlbuilder" "github.com/rancher/wrangler/v2/pkg/schemas/validation" "github.com/sirupsen/logrus" ) @@ -24,9 +26,13 @@ type Store struct { } // Create creates cluster based on the request data. -func (c *Store) Create(_ *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) { +func (c *Store) Create(req *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) { providerName := data.Data().String("provider") - b, err := json.Marshal(data.Data()) + // for k3d to add the request host for additional tls-san + objMap := data.Data() + host, _, _ := net.SplitHostPort(urlbuilder.GetHost(req.Request, "")) + objMap.Set("additionalHost", host) + b, err := json.Marshal(objMap) if err != nil { return types.APIObject{}, err }