From 1c11fe20ba3497bd8cf3082169a09d8ff21cbae7 Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Mon, 2 Dec 2019 03:47:25 -0800 Subject: [PATCH] shimv2: support runtime config path via annotation So that user can have per pod runtime config which is easiler to manage in some cases. Fixes: #2332 Signed-off-by: Peng Tao --- containerd-shim-v2/create.go | 13 +- containerd-shim-v2/create_test.go | 125 ++++++++++++++++++ virtcontainers/pkg/annotations/annotations.go | 2 + virtcontainers/pkg/oci/utils.go | 4 + 4 files changed, 139 insertions(+), 5 deletions(-) diff --git a/containerd-shim-v2/create.go b/containerd-shim-v2/create.go index fdd7b59280..a46962b92b 100644 --- a/containerd-shim-v2/create.go +++ b/containerd-shim-v2/create.go @@ -59,7 +59,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID()) } - _, err := loadRuntimeConfig(s, r) + _, err := loadRuntimeConfig(s, r, ociSpec.Annotations) if err != nil { return nil, err } @@ -147,10 +147,13 @@ func loadSpec(r *taskAPI.CreateTaskRequest) (*specs.Spec, string, error) { return &ociSpec, bundlePath, nil } -func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest) (*oci.RuntimeConfig, error) { - var configPath string - - if r.Options != nil { +// Config override ordering(high to low): +// 1. podsandbox annotation +// 2. shimv2 create task option +// 3. environment +func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string]string) (*oci.RuntimeConfig, error) { + configPath := oci.GetSandboxConfigPath(anno) + if configPath == "" && r.Options != nil { v, err := typeurl.UnmarshalAny(r.Options) if err != nil { return nil, err diff --git a/containerd-shim-v2/create_test.go b/containerd-shim-v2/create_test.go index 27efb906f8..ea0e286d4d 100644 --- a/containerd-shim-v2/create_test.go +++ b/containerd-shim-v2/create_test.go @@ -8,19 +8,24 @@ package containerdshim import ( "context" + "fmt" "io/ioutil" "os" + "path" "path/filepath" "testing" "github.com/containerd/containerd/namespaces" taskAPI "github.com/containerd/containerd/runtime/v2/task" + crioption "github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1" + "github.com/containerd/typeurl" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" ktu "github.com/kata-containers/runtime/pkg/katatestutils" "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" + vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/pkg/compatoci" "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" ) @@ -369,3 +374,123 @@ func TestCreateContainerConfigFail(t *testing.T) { _, err = s.Create(ctx, req) assert.Error(err) } + +func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) { + if dir == "" { + return "", fmt.Errorf("BUG: need directory") + } + + if hypervisor == "" { + return "", fmt.Errorf("BUG: need hypervisor") + } + + hypervisorPath := path.Join(dir, "hypervisor") + kernelPath := path.Join(dir, "kernel") + kernelParams := "foo=bar xyz" + imagePath := path.Join(dir, "image") + shimPath := path.Join(dir, "shim") + proxyPath := path.Join(dir, "proxy") + netmonPath := path.Join(dir, "netmon") + logDir := path.Join(dir, "logs") + logPath := path.Join(logDir, "runtime.log") + machineType := "machineType" + disableBlockDevice := true + blockDeviceDriver := "virtio-scsi" + enableIOThreads := true + hotplugVFIOOnRootBus := true + disableNewNetNs := false + sharedFS := "virtio-9p" + + configFileOptions := ktu.RuntimeConfigOptions{ + Hypervisor: "qemu", + HypervisorPath: hypervisorPath, + KernelPath: kernelPath, + ImagePath: imagePath, + KernelParams: kernelParams, + MachineType: machineType, + ShimPath: shimPath, + ProxyPath: proxyPath, + NetmonPath: netmonPath, + LogPath: logPath, + DisableBlock: disableBlockDevice, + BlockDeviceDriver: blockDeviceDriver, + EnableIOThreads: enableIOThreads, + HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, + DisableNewNetNs: disableNewNetNs, + SharedFS: sharedFS, + } + + runtimeConfigFileData := ktu.MakeRuntimeConfigFileData(configFileOptions) + + configPath := path.Join(dir, "runtime.toml") + err = ioutil.WriteFile(configPath, []byte(runtimeConfigFileData), os.FileMode(0640)) + if err != nil { + return "", err + } + + files := []string{hypervisorPath, kernelPath, imagePath, shimPath, proxyPath} + + for _, file := range files { + // create the resource (which must be >0 bytes) + err := ioutil.WriteFile(file, []byte("foo"), os.FileMode(0640)) + if err != nil { + return "", err + } + } + + return configPath, nil +} + +func TestCreateLoadRuntimeConfig(t *testing.T) { + assert := assert.New(t) + + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + config, err := createAllRuntimeConfigFiles(tmpdir, "qemu") + assert.NoError(err) + + s := &service{ + id: testSandboxID, + ctx: context.Background(), + } + r := &taskAPI.CreateTaskRequest{} + anno := make(map[string]string) + + // set all to fake path + fakeConfig := "foobar" + anno[vcAnnotations.SandboxConfigPathKey] = fakeConfig + option := &crioption.Options{ConfigPath: fakeConfig} + r.Options, err = typeurl.MarshalAny(option) + assert.NoError(err) + err = os.Setenv("KATA_CONF_FILE", fakeConfig) + assert.NoError(err) + defer os.Setenv("KATA_CONF_FILE", "") + + // fake config should fail + _, err = loadRuntimeConfig(s, r, anno) + assert.Error(err) + + // 1. podsandbox annotation + anno[vcAnnotations.SandboxConfigPathKey] = config + _, err = loadRuntimeConfig(s, r, anno) + assert.NoError(err) + anno[vcAnnotations.SandboxConfigPathKey] = "" + + // 2. shimv2 create task option + option.ConfigPath = config + r.Options, err = typeurl.MarshalAny(option) + assert.NoError(err) + _, err = loadRuntimeConfig(s, r, anno) + assert.NoError(err) + option.ConfigPath = "" + r.Options, err = typeurl.MarshalAny(option) + assert.NoError(err) + + // 3. environment + err = os.Setenv("KATA_CONF_FILE", config) + assert.NoError(err) + _, err = loadRuntimeConfig(s, r, anno) + assert.NoError(err) +} diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index d7396a9485..fa580cc9d7 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -19,6 +19,8 @@ const ( // ContainerTypeKey is the annotation key to fetch container type. ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type" + + SandboxConfigPathKey = kataAnnotationsPrefix + "config_path" ) // Annotations related to Hypervisor configuration diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 51db648be0..98e2a843dd 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -309,6 +309,10 @@ func ContainerType(spec specs.Spec) (vc.ContainerType, error) { return vc.PodSandbox, nil } +func GetSandboxConfigPath(annotations map[string]string) string { + return annotations[vcAnnotations.SandboxConfigPathKey] +} + // SandboxID determines the sandbox ID related to an OCI configuration. This function // is expected to be called only when the container type is "PodContainer". func SandboxID(spec specs.Spec) (string, error) {