Merge pull request #102 from AkihiroSuda/a

Fix v20190603.0 regression
rootless-containers · Jun 3, 2019 · 2fd0ba0 · 2fd0ba0
2 parents 47773f7 + f248c0b
commit 2fd0ba0
Show file tree

Hide file tree

Showing 6 changed files with 116 additions and 28 deletions.
diff --git a/boot/kubelet.sh b/boot/kubelet.sh
@@ -12,5 +12,5 @@ exec $(dirname $0)/nsenter.sh hyperkube kubelet \
 	--authorization-mode=AlwaysAllow \
 	--fail-swap-on=false \
 	--feature-gates DevicePlugins=false,SupportNoneCgroupDriver=true \
-	--cgroup-driver none \
+	--cgroup-driver=none --cgroups-per-qos=false --enforce-node-allocatable="" \
 	$@
diff --git a/cleanup.sh b/cleanup.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -e -x
 cd $(dirname $0)
 if [ -z $XDG_RUNTIME_DIR ]; then
@@ -11,6 +11,6 @@ if [ -z $HOME ]; then
 fi
 
 # use RootlessKit for removing files owned by sub-IDs.
-./bin/rootlesskit rm -rf $XDG_RUNTIME_DIR/usernetes $HOME/.local/share/usernetes $HOME/.local/share/docker $HOME/.local/share/containers
+./bin/rootlesskit rm -rf $XDG_RUNTIME_DIR/{usernetes,docker*,containerd,runc} $HOME/.local/share/usernetes $HOME/.local/share/docker $HOME/.local/share/containers
 
 echo "You may also want to remove manually: ~/.config/{docker,crio,usernetes} ~/.docker ~/.kube"
diff --git a/common/common.inc.sh b/common/common.inc.sh
@@ -137,3 +137,4 @@ export PATH
 : ${XDG_CONFIG_HOME=$HOME/.config}
 : ${XDG_CACHE_HOME=$HOME/.cache}
 export XDG_DATA_HOME XDG_CONFIG_HOME XDG_CACHE_HOME
+
diff --git a/run.sh b/run.sh
@@ -1,3 +1,13 @@
 #!/bin/bash
 set -eu -o pipefail
+# clean up (workaround for crash of previously running instances)
+(
+	if ! [[ -w $XDG_RUNTIME_DIR ]]; then
+		echo &>2 "XDG_RUNTIME_DIR needs to be set and writable"
+		exit 1
+	fi
+	rootlesskit=$(realpath $(dirname $0))/bin/rootlesskit
+	cd $XDG_RUNTIME_DIR
+	$rootlesskit rm -rf docker docker.* containerd runc crio usernetes
+)
 exec $(dirname $0)/bin/task $@
diff --git a/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch b/src/patches/kubernetes/0002-kube-proxy-allow-running-in-userns.patch
@@ -1,28 +1,100 @@
-From 1c30b8976da49e50c11a214dd4a3be66571094f4 Mon Sep 17 00:00:00 2001
+From e608a7934e23101022c4da789033df7c5fb84713 Mon Sep 17 00:00:00 2001
 From: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
 Date: Thu, 23 Aug 2018 14:14:44 +0900
 Subject: [PATCH 2/3] kube-proxy: allow running in userns
 
 Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
 ---
- cmd/kube-proxy/app/BUILD            | 1 +
- cmd/kube-proxy/app/server_others.go | 9 ++++++++-
- pkg/proxy/userspace/BUILD           | 1 +
- pkg/proxy/userspace/proxier.go      | 6 +++++-
- 4 files changed, 15 insertions(+), 2 deletions(-)
+ cmd/kube-proxy/app/BUILD            | 10 ++++++++++
+ cmd/kube-proxy/app/server_others.go |  9 ++++++++-
+ pkg/proxy/userspace/BUILD           |  1 +
+ pkg/proxy/userspace/proxier.go      |  6 +++++-
+ 4 files changed, 24 insertions(+), 2 deletions(-)
 
 diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD
-index 552a6cae68..35843c710b 100644
+index 552a6cae68..d03442ed22 100644
 --- a/cmd/kube-proxy/app/BUILD
 +++ b/cmd/kube-proxy/app/BUILD
-@@ -65,6 +65,7 @@ go_library(
-         "//staging/src/k8s.io/component-base/config:go_default_library",
-         "//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library",
-         "//vendor/github.com/fsnotify/fsnotify:go_default_library",
-+        "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
-         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
-         "//vendor/github.com/spf13/cobra:go_default_library",
-         "//vendor/github.com/spf13/pflag:go_default_library",
+@@ -78,6 +78,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:darwin": [
+             "//pkg/proxy/metrics:go_default_library",
+@@ -85,6 +86,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:dragonfly": [
+             "//pkg/proxy/metrics:go_default_library",
+@@ -92,6 +94,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:freebsd": [
+             "//pkg/proxy/metrics:go_default_library",
+@@ -99,6 +102,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:linux": [
+             "//pkg/proxy/metrics:go_default_library",
+@@ -106,6 +110,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:nacl": [
+             "//pkg/proxy/metrics:go_default_library",
+@@ -113,6 +118,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:netbsd": [
+             "//pkg/proxy/metrics:go_default_library",
+@@ -120,6 +126,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:openbsd": [
+             "//pkg/proxy/metrics:go_default_library",
+@@ -127,6 +134,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:plan9": [
+             "//pkg/proxy/metrics:go_default_library",
+@@ -134,6 +142,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:solaris": [
+             "//pkg/proxy/metrics:go_default_library",
+@@ -141,6 +150,7 @@ go_library(
+             "//pkg/util/node:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+             "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
++            "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         ],
+         "@io_bazel_rules_go//go/platform:windows": [
+             "//pkg/proxy/winkernel:go_default_library",
 diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go
 index 1b31497db0..32310a603a 100644
 --- a/cmd/kube-proxy/app/server_others.go
@@ -58,17 +130,17 @@ index 1b31497db0..32310a603a 100644
  		NodeRef:                nodeRef,
  		MetricsBindAddress:     config.MetricsBindAddress,
 diff --git a/pkg/proxy/userspace/BUILD b/pkg/proxy/userspace/BUILD
-index 87e3da69e9..92ee648153 100644
+index 87e3da69e9..8f148a1470 100644
 --- a/pkg/proxy/userspace/BUILD
 +++ b/pkg/proxy/userspace/BUILD
-@@ -35,6 +35,7 @@ go_library(
+@@ -34,6 +34,7 @@ go_library(
+         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
          "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
          "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-         "//vendor/k8s.io/klog:go_default_library",
 +        "//vendor/github.com/opencontainers/runc/libcontainer/system:go_default_library",
+         "//vendor/k8s.io/klog:go_default_library",
          "//vendor/k8s.io/utils/exec:go_default_library",
      ] + select({
-         "@io_bazel_rules_go//go/platform:android": [
 diff --git a/pkg/proxy/userspace/proxier.go b/pkg/proxy/userspace/proxier.go
 index ae55842b30..06d3682cb4 100644
 --- a/pkg/proxy/userspace/proxier.go

diff --git a/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch b/src/patches/kubernetes/0003-kubelet-new-feature-gate-SupportNoneCgroupDriver.patch
@@ -1,4 +1,4 @@
-From 25792602eb2cf2d0ee62f454c6093081a6b31153 Mon Sep 17 00:00:00 2001
+From e91da6ba4070a4e427e3ac280808c759e75e0613 Mon Sep 17 00:00:00 2001
 From: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
 Date: Sun, 2 Jun 2019 18:39:05 +0900
 Subject: [PATCH 3/3] kubelet: new feature gate: SupportNoneCgroupDriver
@@ -18,11 +18,11 @@ Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
  pkg/kubelet/apis/config/types.go              |  2 +-
  pkg/kubelet/cm/cgroup_manager_linux.go        | 69 ++++++++++++++++++-
  pkg/kubelet/cm/cgroup_manager_unsupported.go  |  4 +-
- pkg/kubelet/cm/container_manager_linux.go     |  7 +-
+ pkg/kubelet/cm/container_manager_linux.go     | 10 ++-
  .../cm/pod_container_manager_linux_test.go    |  6 +-
  pkg/kubelet/dockershim/docker_service.go      |  3 +-
  test/e2e_node/node_container_manager_test.go  |  5 +-
- 11 files changed, 125 insertions(+), 31 deletions(-)
+ 11 files changed, 128 insertions(+), 31 deletions(-)
 
 diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go
 index 27c2a9948c..1f602c7ca1 100644
@@ -264,10 +264,10 @@ index 5d77ed7a45..5654d737fd 100644
 
  func (m *unsupportedCgroupManager) Name(_ CgroupName) string {
 diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
-index 8116edcb63..9157822e9b 100644
+index 8116edcb63..01600d8cfe 100644
 --- a/pkg/kubelet/cm/container_manager_linux.go
 +++ b/pkg/kubelet/cm/container_manager_linux.go
-@@ -244,7 +244,10 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
+@@ -244,9 +244,15 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
 
  	// Turn CgroupRoot from a string (in cgroupfs path format) to internal CgroupName
  	cgroupRoot := ParseCgroupfsToCgroupName(nodeConfig.CgroupRoot)
@@ -278,8 +278,13 @@ index 8116edcb63..9157822e9b 100644
 +	}
  	// Check if Cgroup-root actually exists on the node
  	if nodeConfig.CgroupsPerQOS {
++		if nodeConfig.CgroupDriver == noneDriver {
++			return nil, fmt.Errorf("invalid configuration: cgroups-per-qos is not supported for %s cgroup driver", nodeConfig.CgroupDriver)
++		}
  		// this does default to / when enabled, but this tests against regressions.
-@@ -256,7 +259,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
+ 		if nodeConfig.CgroupRoot == "" {
+ 			return nil, fmt.Errorf("invalid configuration: cgroups-per-qos was specified and cgroup-root was not specified. To enable the QoS cgroup hierarchy you need to specify a valid cgroup-root")
+@@ -256,7 +262,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
  		// of note, we always use the cgroupfs driver when performing this check since
  		// the input is provided in that format.
  		// this is important because we do not want any name conversion to occur.
Original file line number	Diff line number	Diff line change
Expand Up		@@ -137,3 +137,4 @@ export PATH
		: ${XDG_CONFIG_HOME=$HOME/.config}
		: ${XDG_CACHE_HOME=$HOME/.cache}
		export XDG_DATA_HOME XDG_CONFIG_HOME XDG_CACHE_HOME