agent: add kernel cmdline parameter to enable or disable cgroups v2

Add `agent.unified_cgroup_hierarchy` kernel cmdline parameter to enable or
disable cgroups v2 in the guest (like systemd).

Set `agent.unified_cgroup_hierarchy` to `1` or `true` to enable cgroups v2
in the guest.
Set `agent.unified_cgroup_hierarchy` to `0` or `false` to disable cgroups v2
in the guest.
By default cgroups v2 is disabled.

Depends-on: github.com/kata-containers/tests#2356

fixes #749

Signed-off-by: Julio Montes <[email protected]>

README.md

@@ -9,6 +9,7 @@
 * [Enable debug console](#enable-debug-console)
 * [`cpuset` cgroup details](#cpuset-cgroup-details)
 * [Hotplug Timeout](#hotplug-timeout)
+* [Cgroups V2](#cgroups-v2)
 
 This project implements an agent called `kata-agent` that runs inside a virtual machine (VM).
 
@@ -77,5 +78,15 @@ The value of the option is in the [Go duration format][2].
 
 Any invalid values used for `agent.hotplug_timeout` will fall back to the default of 3 seconds.
 
+## Cgroups V2
+
+Same as `systemd`, the `kata-agent` has an option to enable or disable the unified
+cgroup hierarchy (cgroups v2) in the guest through the kernel command line.
+Set `agent.unified_cgroup_hierarchy` to `1` or `true` to enable cgroups v2. For
+example, `agent.unified_cgroup_hierarchy=true` will enable cgroups v2 in the guest.
+Set `agent.unified_cgroup_hierarchy` to `0` or `false` to disable cgroups v2. For
+example, `agent.unified_cgroup_hierarchy=0` will disable cgroups v2 in the guest.
+By default cgroups v2 is disabled.
+
 [1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md
 [2]: https://golang.org/pkg/time/#ParseDuration

agent.go

@@ -63,6 +63,10 @@ var (
 	cgroupMemoryUseHierarchyPath = cgroupMemoryPath + "/memory.use_hierarchy"
 	cgroupMemoryUseHierarchyMode = os.FileMode(0400)
 
+	cgroupControllersPath    = cgroupPath + "/cgroup.controllers"
+	cgroupSubtreeControlPath = cgroupPath + "/cgroup.subtree_control"
+	cgroupSubtreeControlMode = os.FileMode(0644)
+
 	// Set by the build
 	seccompSupport string
 
@@ -180,6 +184,9 @@ var hotplugTimeout = 3 * time.Second
 // Specify the log level
 var logLevel = defaultLogLevel
 
+// Specify whether the agent has to use cgroups v2 or not.
+var unifiedCgroupHierarchy = false
+
 // commType is used to denote the communication channel type used.
 type commType int
 
@@ -1215,6 +1222,12 @@ type initMount struct {
 }
 
 func getCgroupMounts(cgPath string) ([]initMount, error) {
+	if unifiedCgroupHierarchy {
+		return []initMount{
+			{"cgroup2", "cgroup2", cgroupPath, []string{"nosuid", "nodev", "noexec", "relatime", "nsdelegate"}},
+		}, nil
+	}
+
 	f, err := os.Open(cgPath)
 	if err != nil {
 		return []initMount{}, err
@@ -1293,9 +1306,26 @@ func cgroupsMount() error {
 		}
 	}
 
-	// Enable memory hierarchical account.
-	// For more information see https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
-	return ioutil.WriteFile(cgroupMemoryUseHierarchyPath, []byte{'1'}, cgroupMemoryUseHierarchyMode)
+	if !unifiedCgroupHierarchy {
+		// Enable memory hierarchical account.
+		// For more information see https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
+		return ioutil.WriteFile(cgroupMemoryUseHierarchyPath, []byte{'1'}, cgroupMemoryUseHierarchyMode)
+	}
+
+	// Enable all cgroup v2 controllers
+	rawControllers, err := ioutil.ReadFile(cgroupControllersPath)
+	if err != nil {
+		return err
+	}
+
+	var controllers string
+	for _, c := range strings.Fields(string(rawControllers)) {
+		controllers += fmt.Sprintf("+%v ", c)
+	}
+
+	// https://www.kernel.org/doc/Documentation/cgroup-v2.txt
+	return ioutil.WriteFile(cgroupSubtreeControlPath,
+		[]byte(strings.TrimSpace(controllers)), cgroupSubtreeControlMode)
 }
 
 func setupDebugConsoleForVsock(ctx context.Context) error {

agent_test.go

@@ -562,6 +562,23 @@ func TestMountToRootfsFailed(t *testing.T) {
 
 }
 
+func TestGetCgroupMountsUnifiedHierarchy(t *testing.T) {
+	assert := assert.New(t)
+
+	orgUnifiedCgroupHierarchy := unifiedCgroupHierarchy
+	defer func() {
+		unifiedCgroupHierarchy = orgUnifiedCgroupHierarchy
+	}()
+	unifiedCgroupHierarchy = true
+
+	initMounts, err := getCgroupMounts("/sys/fs/cgroups")
+	assert.NoError(err)
+	assert.NotEmpty(initMounts)
+	assert.Len(initMounts, 1)
+	assert.Equal(initMounts[0].fstype, "cgroup2")
+	assert.Equal(initMounts[0].src, "cgroup2")
+}
+
 func TestGetCgroupMountsFailed(t *testing.T) {
 	cgprocDir, err := ioutil.TempDir("", "proc-cgroup")
 	assert.Nil(t, err, "%v", err)

config.go

@@ -18,20 +18,21 @@ import (
 )
 
 const (
-	optionPrefix          = "agent."
-	logLevelFlag          = optionPrefix + "log"
-	logsVSockPortFlag     = optionPrefix + "log_vport"
-	devModeFlag           = optionPrefix + "devmode"
-	traceModeFlag         = optionPrefix + "trace"
-	useVsockFlag          = optionPrefix + "use_vsock"
-	debugConsoleFlag      = optionPrefix + "debug_console"
-	debugConsoleVPortFlag = optionPrefix + "debug_console_vport"
-	hotplugTimeoutFlag    = optionPrefix + "hotplug_timeout"
-	traceModeStatic       = "static"
-	traceModeDynamic      = "dynamic"
-	traceTypeIsolated     = "isolated"
-	traceTypeCollated     = "collated"
-	defaultTraceType      = traceTypeIsolated
+	optionPrefix               = "agent."
+	logLevelFlag               = optionPrefix + "log"
+	logsVSockPortFlag          = optionPrefix + "log_vport"
+	devModeFlag                = optionPrefix + "devmode"
+	traceModeFlag              = optionPrefix + "trace"
+	useVsockFlag               = optionPrefix + "use_vsock"
+	debugConsoleFlag           = optionPrefix + "debug_console"
+	debugConsoleVPortFlag      = optionPrefix + "debug_console_vport"
+	hotplugTimeoutFlag         = optionPrefix + "hotplug_timeout"
+	unifiedCgroupHierarchyFlag = optionPrefix + "unified_cgroup_hierarchy"
+	traceModeStatic            = "static"
+	traceModeDynamic           = "dynamic"
+	traceTypeIsolated          = "isolated"
+	traceTypeCollated          = "collated"
+	defaultTraceType           = traceTypeIsolated
 )
 
 var kernelCmdlineFile = "/proc/cmdline"
@@ -141,6 +142,12 @@ func parseCmdlineOption(option string) error {
 			agentLog.Debug("Param passed to NOT use vsock channel")
 			commCh = serialCh
 		}
+	case unifiedCgroupHierarchyFlag:
+		flag, err := strconv.ParseBool(split[valuePosition])
+		if err != nil {
+			return err
+		}
+		unifiedCgroupHierarchy = flag
 	default:
 		if strings.HasPrefix(split[optionPosition], optionPrefix) {
 			return grpcStatus.Errorf(codes.NotFound, "Unknown option %s", split[optionPosition])

config_test.go

@@ -410,3 +410,41 @@ func TestParseCmdlineOptionHotplugTimeout(t *testing.T) {
 		assert.Equal(d.expectedHotplugTimeout, hotplugTimeout, "test %d (%+v)", i, d)
 	}
 }
+
+func TestParseCmdlineOptionUnifiedCgroupHierarchy(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		option      string
+		expected    bool
+		expectError bool
+	}
+
+	data := []testData{
+		{"agent.unifiedCgroupHierarchy", false, false},
+		{"agent.unified_cgroup_hierarchy", false, false},
+		{"agent.unified_cgroup_hierarchi", false, false},
+		{"agent.unified_cgroup_hierarchy=fal", false, true},
+		{"agent.unified_cgroup_hierarchy=ttt", false, true},
+		{"agent.unified_cgroup_hierarchy=tru", false, true},
+		{"agent.unified_cgroup_hierarchy=5", false, true},
+
+		{"agent.unified_cgroup_hierarchy=false", false, false},
+		{"agent.unified_cgroup_hierarchy=0", false, false},
+
+		{"agent.unified_cgroup_hierarchy=true", true, false},
+		{"agent.unified_cgroup_hierarchy=1", true, false},
+	}
+
+	for _, d := range data {
+		unifiedCgroupHierarchy = false
+
+		err := parseCmdlineOption(d.option)
+		if d.expectError {
+			assert.Error(err)
+		} else {
+			assert.NoError(err)
+		}
+		assert.Equal(d.expected, unifiedCgroupHierarchy)
+	}
+}