annotations: Support annotations to customise kata config

Add support for annotations that allow us to custimise a subset
of the configurations provided in kata conf toml file.
This initial commit adds support for customising vcpus, default max
vcpus, memory and the kernel command line passed as Hypervisor
config.

Replaces #1695
Fixes #1655

Signed-off-by: Archana Shinde <[email protected]>

virtcontainers/pkg/annotations/annotations.go

@@ -69,6 +69,18 @@ const (
 	// The first word is considered as the module name and the rest as its parameters.
 	//
 	KernelModules = kataAnnotAgentPrefix + "kernel_modules"
+
+	// DefaultVCPUs is a sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor.
+	DefaultVCPUs = kataAnnotHypervisorPrefix + "default_vcpus"
+
+	// DefaultVCPUs is a sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor.
+	DefaultMaxVCPUs = kataAnnotHypervisorPrefix + "default_max_vcpus"
+
+	// DefaultMemory is a sandbox annotation for the memory assigned for a VM by the hypervisor.
+	DefaultMemory = kataAnnotHypervisorPrefix + "default_memory"
+
+	// KernelParams is a sandbox annotation for passing additional guest kernel parameters.
+	KernelParams = kataAnnotHypervisorPrefix + "kernel_params"
 )
 
 const (

virtcontainers/pkg/oci/utils.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
+	goruntime "runtime"
 	"strconv"
 	"strings"
 	"syscall"
@@ -321,6 +322,18 @@ func SandboxID(spec specs.Spec) (string, error) {
 	return "", fmt.Errorf("Could not find sandbox ID")
 }
 
+func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
+	addAssetAnnotations(ocispec, config)
+	if err := addHypervisorConfigOverrides(ocispec, config); err != nil {
+		return err
+	}
+
+	if err := addAgentConfigOverrides(ocispec, config); err != nil {
+		return err
+	}
+	return nil
+}
+
 func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) {
 	assetAnnotations := []string{
 		vcAnnotations.KernelPath,
@@ -342,14 +355,65 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) {
 
 		config.Annotations[a] = value
 	}
+}
+
+func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error {
+
+	if value, ok := ocispec.Annotations[vcAnnotations.DefaultVCPUs]; ok {
+		vcpus, err := strconv.ParseUint(value, 10, 32)
+		if err != nil {
+			return fmt.Errorf("Error encountered parsing annotation for default_vcpus: %v, please specify numeric value", err)
+		}
+
+		numCPUs := goruntime.NumCPU()
+
+		// Assign the override only if value is valid, else fallback to what is provided by the config
+		if uint32(vcpus) < uint32(numCPUs) {
+			config.HypervisorConfig.NumVCPUs = uint32(vcpus)
+		}
+	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs]; ok {
+		maxVCPUs, err := strconv.ParseUint(value, 10, 32)
+		if err != nil {
+			return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err)
+		}
+
+		numCPUs := goruntime.NumCPU()
+		max := uint32(maxVCPUs)
+		if max < uint32(numCPUs) && max < vc.MaxQemuVCPUs() {
+			config.HypervisorConfig.DefaultMaxVCPUs = max
+		}
+	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok {
+		memorySz, err := strconv.ParseUint(value, 10, 32)
+		if err != nil || memorySz < 8 {
+			return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err)
+		}
+
+		config.HypervisorConfig.MemorySize = uint32(memorySz)
+	}
 
+	if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok {
+		if value != "" {
+			config.HypervisorConfig.KernelParams = vc.DeserializeParams(strings.Fields(value))
+		}
+	}
+
+	return nil
+}
+
+func addAgentConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error {
 	if value, ok := ocispec.Annotations[vcAnnotations.KernelModules]; ok {
 		if c, ok := config.AgentConfig.(vc.KataAgentConfig); ok {
 			modules := strings.Split(value, KernelModulesSeparator)
 			c.KernelModules = modules
 			config.AgentConfig = c
 		}
 	}
+
+	return nil
 }
 
 // SandboxConfig converts an OCI compatible runtime configuration file
@@ -409,7 +473,9 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
 		Experimental: runtime.Experimental,
 	}
 
-	addAssetAnnotations(ocispec, &sandboxConfig)
+	if err := addAnnotations(ocispec, &sandboxConfig); err != nil {
+		return vc.SandboxConfig{}, err
+	}
 
 	return sandboxConfig, nil
 }

virtcontainers/pkg/oci/utils_test.go

@@ -645,7 +645,7 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestAddAssetAnnotations(t *testing.T) {
+func TestAddAnnotations(t *testing.T) {
 	assert := assert.New(t)
 
 	expectedAnnotations := map[string]string{
@@ -666,7 +666,7 @@ func TestAddAssetAnnotations(t *testing.T) {
 		Annotations: expectedAnnotations,
 	}
 
-	addAssetAnnotations(ocispec, &config)
+	addAnnotations(ocispec, &config)
 	assert.Exactly(expectedAnnotations, config.Annotations)
 
 	expectedAgentConfig := vc.KataAgentConfig{
@@ -677,7 +677,46 @@ func TestAddAssetAnnotations(t *testing.T) {
 	}
 
 	ocispec.Annotations[vcAnnotations.KernelModules] = strings.Join(expectedAgentConfig.KernelModules, KernelModulesSeparator)
-	addAssetAnnotations(ocispec, &config)
+	addAnnotations(ocispec, &config)
 	assert.Exactly(expectedAgentConfig, config.AgentConfig)
 
+	expectedHyperConfig := vc.HypervisorConfig{
+		KernelParams: []vc.Param{
+			{
+				Key:   "vsyscall",
+				Value: "emulate",
+			},
+			{
+				Key:   "iommu",
+				Value: "on",
+			},
+		},
+	}
+
+	ocispec.Annotations[vcAnnotations.KernelParams] = "vsyscall=emulate iommu=on"
+	addAnnotations(ocispec, &config)
+	assert.Exactly(expectedHyperConfig, config.HypervisorConfig)
+
+	ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1"
+	ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "2"
+	ocispec.Annotations[vcAnnotations.DefaultMemory] = "4096"
+
+	addAnnotations(ocispec, &config)
+	assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1))
+	assert.Equal(config.HypervisorConfig.DefaultMaxVCPUs, uint32(2))
+	assert.Equal(config.HypervisorConfig.MemorySize, uint32(4096))
+
+	// In case an absurd large value is provided, the config value if not over-ridden
+	ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "655536"
+	addAnnotations(ocispec, &config)
+	assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1))
+
+	ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "-1"
+	err := addAnnotations(ocispec, &config)
+	assert.NoError(err)
+
+	ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1"
+	ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "-2"
+	err = addAnnotations(ocispec, &config)
+	assert.NoError(err)
 }