From 19bee572e30f835f57d8c6947a15aabdf16b8531 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 16 Oct 2019 21:19:44 +0000 Subject: [PATCH] agent: connect debugging console in a specific vsock port some hypervisors, like firecracker, don't have connected a unix socket to `/dev/console` making impossible to start a debugging console. Use vsock connection as standard input and output (std*) for the shell process. Add cmdline option to specify the vsock port to connect the debugging console. fixes #666 Signed-off-by: Julio Montes --- README.md | 26 ++++++++++++++++++++ agent.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ config.go | 34 +++++++++++++++---------- config_test.go | 42 +++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 12d494695e..b1468edcc6 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,32 @@ allow the agent process to start a debug console. Debug console is only availabl or `sh` is installed in the rootfs or initrd image. Developers can [connect to the virtual machine using the debug console](https://github.com/kata-containers/documentation/blob/master/Developer-Guide.md#connect-to-the-virtual-machine-using-the-debug-console) +### Enable debug console for firecracker + +Firecracker doesn't have a UNIX socket connected to `/dev/console`, hence the +kernel command line option `agent.debug_console` will not work for firecracker. +Fortunately, firecracker supports [`hybrid vsocks`][1], and they can be used to +communicate processes in the guest with processes in the host. +The kernel command line option `agent.debug_console_vport` was added to allow +developers specify on which `vsock` port the debugging console should be connected. + +In firecracker, the UNIX socket that is connected to the `vsock` end is created at +`/var/lib/vc/firecracker/$CID/root/kata.hvsock`, where `$CID` is the container ID. + +Run the following commands to have a debugging console in firecracker. + +```sh +$ conf="/usr/share/defaults/kata-containers/configuration.toml" +$ sudo sed -i 's/^kernel_params.*/kernel_params="agent.debug_console_vport=1026"/g' "${conf}" +$ sudo su -c 'cd /var/lib/vc/firecracker/08facf/root/ && socat stdin unix-connect:kata.hvsock' +CONNECT 1026 +``` + +**NOTE:** Ports 1024 and 1025 are reserved for communication with the agent and gathering of agent logs respectively + ## `cpuset` cgroup details See the [cpuset cgroup documentation](documentation/features/cpuset.md). + + +[1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md \ No newline at end of file diff --git a/agent.go b/agent.go index 96f9e75ea4..23fb435779 100644 --- a/agent.go +++ b/agent.go @@ -171,6 +171,9 @@ var debugConsole = false // Specify a vsock port where logs are written. var logsVSockPort = uint32(0) +// Specify a vsock port where debug console is attached. +var debugConsoleVSockPort = uint32(0) + // commType is used to denote the communication channel type used. type commType int @@ -1287,11 +1290,75 @@ func cgroupsMount() error { return ioutil.WriteFile(cgroupMemoryUseHierarchyPath, []byte{'1'}, cgroupMemoryUseHierarchyMode) } +func setupDebugConsoleForVsock(ctx context.Context) error { + var shellPath string + for _, s := range supportedShells { + var err error + if _, err = os.Stat(s); err == nil { + shellPath = s + break + } + agentLog.WithError(err).WithField("shell", s).Warn("Shell not found") + } + + if shellPath == "" { + return fmt.Errorf("No available shells (checked %v)", supportedShells) + } + + cmd := exec.Command(shellPath, "-i") + cmd.Env = os.Environ() + cmd.SysProcAttr = &syscall.SysProcAttr{ + // Create Session + Setsid: true, + } + + go func() { + for { + select { + case <-ctx.Done(): + // stop the thread + return + default: + dcmd := *cmd + + l, err := vsock.Listen(debugConsoleVSockPort) + if err != nil { + // nobody dialing + continue + } + c, err := l.Accept() + if err != nil { + l.Close() + // no connection + continue + } + + dcmd.Stdin = c + dcmd.Stdout = c + dcmd.Stderr = c + + if err := dcmd.Run(); err != nil { + agentLog.WithError(err).Warn("failed to start debug console") + } + + c.Close() + l.Close() + } + } + }() + + return nil +} + func setupDebugConsole(ctx context.Context, debugConsolePath string) error { if !debugConsole { return nil } + if debugConsoleVSockPort != uint32(0) { + return setupDebugConsoleForVsock(ctx) + } + var shellPath string for _, s := range supportedShells { var err error diff --git a/config.go b/config.go index cb6bbdfd19..b1ffecef5e 100644 --- a/config.go +++ b/config.go @@ -17,19 +17,20 @@ import ( ) const ( - optionPrefix = "agent." - logLevelFlag = optionPrefix + "log" - logsVSockPortFlag = optionPrefix + "log_vport" - devModeFlag = optionPrefix + "devmode" - traceModeFlag = optionPrefix + "trace" - useVsockFlag = optionPrefix + "use_vsock" - debugConsoleFlag = optionPrefix + "debug_console" - kernelCmdlineFile = "/proc/cmdline" - traceModeStatic = "static" - traceModeDynamic = "dynamic" - traceTypeIsolated = "isolated" - traceTypeCollated = "collated" - defaultTraceType = traceTypeIsolated + optionPrefix = "agent." + logLevelFlag = optionPrefix + "log" + logsVSockPortFlag = optionPrefix + "log_vport" + devModeFlag = optionPrefix + "devmode" + traceModeFlag = optionPrefix + "trace" + useVsockFlag = optionPrefix + "use_vsock" + debugConsoleFlag = optionPrefix + "debug_console" + debugConsoleVPortFlag = optionPrefix + "debug_console_vport" + kernelCmdlineFile = "/proc/cmdline" + traceModeStatic = "static" + traceModeDynamic = "dynamic" + traceTypeIsolated = "isolated" + traceTypeCollated = "collated" + defaultTraceType = traceTypeIsolated ) type agentConfig struct { @@ -113,6 +114,13 @@ func (c *agentConfig) parseCmdlineOption(option string) error { return err } logsVSockPort = uint32(port) + case debugConsoleVPortFlag: + port, err := strconv.ParseUint(split[valuePosition], 10, 32) + if err != nil { + return err + } + debugConsole = true + debugConsoleVSockPort = uint32(port) case traceModeFlag: switch split[valuePosition] { case traceTypeIsolated: diff --git a/config_test.go b/config_test.go index acb7a06bcb..817ab0aeb6 100644 --- a/config_test.go +++ b/config_test.go @@ -366,3 +366,45 @@ func TestParseCmdlineOptionDebugConsole(t *testing.T) { assert.True(debugConsole, "test %d (%+v)", i, d) } } + +func TestParseCmdlineOptionDebugConsoleVPort(t *testing.T) { + assert := assert.New(t) + + a := &agentConfig{} + + type testData struct { + option string + expectDebugConsoleEnabled bool + expectedError bool + expectedVPort uint32 + } + + data := []testData{ + {"", false, false, 0}, + {"debug_console_vport", false, false, 0}, + {"debug_console_vport=xxx", false, false, 0}, + {"debug_console_vport=1026", false, false, 0}, + {debugConsoleVPortFlag + "=", false, true, 0}, + {debugConsoleVPortFlag + "=xxxx", false, true, 0}, + {debugConsoleVPortFlag, false, false, 0}, + {debugConsoleVPortFlag + "=1026", false, false, 1026}, + } + + for i, d := range data { + debugConsole = false + debugConsoleVSockPort = 0 + + err := a.parseCmdlineOption(d.option) + if d.expectedError { + assert.Error(err) + } else { + assert.NoError(err) + } + + if d.expectDebugConsoleEnabled { + assert.True(debugConsole, "test %d (%+v)", i, d) + } + + assert.Equal(debugConsoleVSockPort, d.expectedVPort) + } +}