diff --git a/boot/kubelet.sh b/boot/kubelet.sh index 718e29c..a8ff872 100755 --- a/boot/kubelet.sh +++ b/boot/kubelet.sh @@ -12,5 +12,5 @@ exec $(dirname $0)/nsenter.sh hyperkube kubelet \ --authorization-mode=AlwaysAllow \ --fail-swap-on=false \ --feature-gates DevicePlugins=false,SupportNoneCgroupDriver=true \ - --cgroup-driver none \ + --cgroup-driver=none --cgroups-per-qos=false --enforce-node-allocatable="" \ $@ diff --git a/cleanup.sh b/cleanup.sh index 1750b0f..1b702e6 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e -x cd $(dirname $0) if [ -z $XDG_RUNTIME_DIR ]; then @@ -11,6 +11,6 @@ if [ -z $HOME ]; then fi # use RootlessKit for removing files owned by sub-IDs. -./bin/rootlesskit rm -rf $XDG_RUNTIME_DIR/usernetes $HOME/.local/share/usernetes $HOME/.local/share/docker $HOME/.local/share/containers +./bin/rootlesskit rm -rf $XDG_RUNTIME_DIR/{usernetes,docker*,containerd,runc} $HOME/.local/share/usernetes $HOME/.local/share/docker $HOME/.local/share/containers echo "You may also want to remove manually: ~/.config/{docker,crio,usernetes} ~/.docker ~/.kube" diff --git a/common/common.inc.sh b/common/common.inc.sh index de5862b..3e8b224 100644 --- a/common/common.inc.sh +++ b/common/common.inc.sh @@ -137,3 +137,4 @@ export PATH : ${XDG_CONFIG_HOME=$HOME/.config} : ${XDG_CACHE_HOME=$HOME/.cache} export XDG_DATA_HOME XDG_CONFIG_HOME XDG_CACHE_HOME + diff --git a/run.sh b/run.sh index 3868eb7..1925604 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,13 @@ #!/bin/bash set -eu -o pipefail +# clean up (workaround for crash of previously running instances) +( + if ! [[ -w $XDG_RUNTIME_DIR ]]; then + echo &>2 "XDG_RUNTIME_DIR needs to be set and writable" + exit 1 + fi + rootlesskit=$(realpath $(dirname $0))/bin/rootlesskit + cd $XDG_RUNTIME_DIR + $rootlesskit rm -rf docker docker.* containerd runc crio usernetes +) exec $(dirname $0)/bin/task $@ diff --git a/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch b/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch index d8b3f53..58821ba 100644 --- a/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch +++ b/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch @@ -1,28 +1,100 @@ -From 1c30b8976da49e50c11a214dd4a3be66571094f4 Mon Sep 17 00:00:00 2001 +From e608a7934e23101022c4da789033df7c5fb84713 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 23 Aug 2018 14:14:44 +0900 Subject: [PATCH 2/3] kube-proxy: allow running in userns Signed-off-by: Akihiro Suda --- - cmd/kube-proxy/app/BUILD | 1 + - cmd/kube-proxy/app/server_others.go | 9 ++++++++- - pkg/proxy/userspace/BUILD | 1 + - pkg/proxy/userspace/proxier.go | 6 +++++- - 4 files changed, 15 insertions(+), 2 deletions(-) + cmd/kube-proxy/app/BUILD | 10 ++++++++++ + cmd/kube-proxy/app/server_others.go | 9 ++++++++- + pkg/proxy/userspace/BUILD | 1 + + pkg/proxy/userspace/proxier.go | 6 +++++- + 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD -index 552a6cae68..35843c710b 100644 +index 552a6cae68..d03442ed22 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD -@@ -65,6 +65,7 @@ go_library( - "//staging/src/k8s.io/component-base/config:go_default_library", - "//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library", - "//vendor/github.com/fsnotify/fsnotify:go_default_library", -+ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", +@@ -78,6 +78,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:darwin": [ + "//pkg/proxy/metrics:go_default_library", +@@ -85,6 +86,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:dragonfly": [ + "//pkg/proxy/metrics:go_default_library", +@@ -92,6 +94,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:freebsd": [ + "//pkg/proxy/metrics:go_default_library", +@@ -99,6 +102,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//pkg/proxy/metrics:go_default_library", +@@ -106,6 +110,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:nacl": [ + "//pkg/proxy/metrics:go_default_library", +@@ -113,6 +118,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:netbsd": [ + "//pkg/proxy/metrics:go_default_library", +@@ -120,6 +126,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:openbsd": [ + "//pkg/proxy/metrics:go_default_library", +@@ -127,6 +134,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:plan9": [ + "//pkg/proxy/metrics:go_default_library", +@@ -134,6 +142,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:solaris": [ + "//pkg/proxy/metrics:go_default_library", +@@ -141,6 +150,7 @@ go_library( + "//pkg/util/node:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", ++ "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//pkg/proxy/winkernel:go_default_library", diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 1b31497db0..32310a603a 100644 --- a/cmd/kube-proxy/app/server_others.go @@ -58,17 +130,17 @@ index 1b31497db0..32310a603a 100644 NodeRef: nodeRef, MetricsBindAddress: config.MetricsBindAddress, diff --git a/pkg/proxy/userspace/BUILD b/pkg/proxy/userspace/BUILD -index 87e3da69e9..92ee648153 100644 +index 87e3da69e9..8f148a1470 100644 --- a/pkg/proxy/userspace/BUILD +++ b/pkg/proxy/userspace/BUILD -@@ -35,6 +35,7 @@ go_library( +@@ -34,6 +34,7 @@ go_library( + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/k8s.io/klog:go_default_library", + "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library", + "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ - "@io_bazel_rules_go//go/platform:android": [ diff --git a/pkg/proxy/userspace/proxier.go b/pkg/proxy/userspace/proxier.go index ae55842b30..06d3682cb4 100644 --- a/pkg/proxy/userspace/proxier.go diff --git a/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch b/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch index dff2a52..2340db3 100644 --- a/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch +++ b/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch @@ -1,4 +1,4 @@ -From 25792602eb2cf2d0ee62f454c6093081a6b31153 Mon Sep 17 00:00:00 2001 +From e91da6ba4070a4e427e3ac280808c759e75e0613 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sun, 2 Jun 2019 18:39:05 +0900 Subject: [PATCH 3/3] kubelet: new feature gate: SupportNoneCgroupDriver @@ -18,11 +18,11 @@ Signed-off-by: Akihiro Suda pkg/kubelet/apis/config/types.go | 2 +- pkg/kubelet/cm/cgroup_manager_linux.go | 69 ++++++++++++++++++- pkg/kubelet/cm/cgroup_manager_unsupported.go | 4 +- - pkg/kubelet/cm/container_manager_linux.go | 7 +- + pkg/kubelet/cm/container_manager_linux.go | 10 ++- .../cm/pod_container_manager_linux_test.go | 6 +- pkg/kubelet/dockershim/docker_service.go | 3 +- test/e2e_node/node_container_manager_test.go | 5 +- - 11 files changed, 125 insertions(+), 31 deletions(-) + 11 files changed, 128 insertions(+), 31 deletions(-) diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go index 27c2a9948c..1f602c7ca1 100644 @@ -264,10 +264,10 @@ index 5d77ed7a45..5654d737fd 100644 func (m *unsupportedCgroupManager) Name(_ CgroupName) string { diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go -index 8116edcb63..9157822e9b 100644 +index 8116edcb63..01600d8cfe 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go -@@ -244,7 +244,10 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I +@@ -244,9 +244,15 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I // Turn CgroupRoot from a string (in cgroupfs path format) to internal CgroupName cgroupRoot := ParseCgroupfsToCgroupName(nodeConfig.CgroupRoot) @@ -278,8 +278,13 @@ index 8116edcb63..9157822e9b 100644 + } // Check if Cgroup-root actually exists on the node if nodeConfig.CgroupsPerQOS { ++ if nodeConfig.CgroupDriver == noneDriver { ++ return nil, fmt.Errorf("invalid configuration: cgroups-per-qos is not supported for %s cgroup driver", nodeConfig.CgroupDriver) ++ } // this does default to / when enabled, but this tests against regressions. -@@ -256,7 +259,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I + if nodeConfig.CgroupRoot == "" { + return nil, fmt.Errorf("invalid configuration: cgroups-per-qos was specified and cgroup-root was not specified. To enable the QoS cgroup hierarchy you need to specify a valid cgroup-root") +@@ -256,7 +262,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I // of note, we always use the cgroupfs driver when performing this check since // the input is provided in that format. // this is important because we do not want any name conversion to occur.