Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

Commit

Permalink
Merge pull request #2972 from egernst/cpuset-fixes
Browse files Browse the repository at this point in the history 
cpuset: fixes to address VM sizing and constraining based on cpuset.mems
  • Loading branch information
Julio Montes authored Sep 25, 2020
2 parents cf5d190 + 8ef2946 commit 9934054
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 53 deletions.
3 changes: 2 additions & 1 deletion virtcontainers/pkg/cgroups/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,14 +329,15 @@ func (m *Manager) RemoveDevice(device string) error {
return fmt.Errorf("device %v not found in the cgroup", device)
}

func (m *Manager) SetCPUSet(cpuset string) error {
func (m *Manager) SetCPUSet(cpuset, memset string) error {
cgroups, err := m.GetCgroups()
if err != nil {
return err
}

m.Lock()
cgroups.CpusetCpus = cpuset
cgroups.CpusetMems = memset
m.Unlock()

return m.Apply()
Expand Down
56 changes: 39 additions & 17 deletions virtcontainers/sandbox.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1867,7 +1867,10 @@ func (s *Sandbox) updateResources() error {
return fmt.Errorf("sandbox config is nil")
}

sandboxVCPUs := s.calculateSandboxCPUs()
sandboxVCPUs, err := s.calculateSandboxCPUs()
if err != nil {
return err
}
// Add default vcpus for sandbox
sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs

Expand Down Expand Up @@ -1927,8 +1930,9 @@ func (s *Sandbox) calculateSandboxMemory() int64 {
return memorySandbox
}

func (s *Sandbox) calculateSandboxCPUs() uint32 {
func (s *Sandbox) calculateSandboxCPUs() (uint32, error) {
mCPU := uint32(0)
cpusetCount := int(0)

for _, c := range s.config.Containers {
// Do not hot add again non-running containers resources
Expand All @@ -1942,9 +1946,22 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 {
mCPU += utils.CalculateMilliCPUs(*cpu.Quota, *cpu.Period)
}

set, err := cpuset.Parse(cpu.Cpus)
if err != nil {
return 0, nil
}
cpusetCount += set.Size()
}
}
return utils.CalculateVCpusFromMilliCpus(mCPU)

// If we aren't being constrained, then we could have two scenarios:
// 1. BestEffort QoS: no proper support today in Kata.
// 2. We could be constrained only by CPUSets. Check for this:
if mCPU == 0 && cpusetCount > 0 {
return uint32(cpusetCount), nil
}

return utils.CalculateVCpusFromMilliCpus(mCPU), nil
}

// GetHypervisorType is used for getting Hypervisor name currently used.
Expand All @@ -1963,15 +1980,13 @@ func (s *Sandbox) cgroupsUpdate() error {
// in the Kata sandbox cgroup (inherited). Check to see if sandbox cpuset needs to be
// updated.
if s.config.SandboxCgroupOnly {
cpuset, err := s.getSandboxCPUSet()
cpuset, memset, err := s.getSandboxCPUSet()
if err != nil {
return err
}

if cpuset != "" {
if err := s.cgroupMgr.SetCPUSet(cpuset); err != nil {
return err
}
if err := s.cgroupMgr.SetCPUSet(cpuset, memset); err != nil {
return err
}

return nil
Expand Down Expand Up @@ -2268,23 +2283,30 @@ func (s *Sandbox) GetOOMEvent() (string, error) {
return s.agent.getOOMEvent()
}

// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets
// as a string in canonical linux CPU list format
func (s *Sandbox) getSandboxCPUSet() (string, error) {
// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets'
// cpus and mems as a string in canonical linux CPU/mems list format
func (s *Sandbox) getSandboxCPUSet() (string, string, error) {
if s.config == nil {
return "", nil
return "", "", nil
}

result := cpuset.NewCPUSet()
cpuResult := cpuset.NewCPUSet()
memResult := cpuset.NewCPUSet()
for _, ctr := range s.config.Containers {
if ctr.Resources.CPU != nil {
currSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus)
currCPUSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus)
if err != nil {
return "", "", fmt.Errorf("unable to parse CPUset.cpus for container %s: %v", ctr.ID, err)
}
cpuResult = cpuResult.Union(currCPUSet)

currMemSet, err := cpuset.Parse(ctr.Resources.CPU.Mems)
if err != nil {
return "", fmt.Errorf("unable to parse CPUset for container %s: %v", ctr.ID, err)
return "", "", fmt.Errorf("unable to parse CPUset.mems for container %s: %v", ctr.ID, err)
}
result = result.Union(currSet)
memResult = memResult.Union(currMemSet)
}
}

return result.String(), nil
return cpuResult.String(), memResult.String(), nil
}
97 changes: 62 additions & 35 deletions virtcontainers/sandbox_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,18 @@ func TestCreateMockSandbox(t *testing.T) {
func TestCalculateSandboxCPUs(t *testing.T) {
sandbox := &Sandbox{}
sandbox.config = &SandboxConfig{}

unconstrained := newTestContainerConfigNoop("cont-00001")
constrained := newTestContainerConfigNoop("cont-00001")
constrained := newTestContainerConfigNoop("cont-00002")
unconstrainedCpusets0_1 := newTestContainerConfigNoop("cont-00003")
unconstrainedCpusets2 := newTestContainerConfigNoop("cont-00004")
constrainedCpusets0_7 := newTestContainerConfigNoop("cont-00005")
quota := int64(4000)
period := uint64(1000)
constrained.Resources.CPU = &specs.LinuxCPU{Period: &period, Quota: &quota}

unconstrainedCpusets0_1.Resources.CPU = &specs.LinuxCPU{Cpus: "0-1"}
unconstrainedCpusets2.Resources.CPU = &specs.LinuxCPU{Cpus: "2"}
constrainedCpusets0_7.Resources.CPU = &specs.LinuxCPU{Period: &period, Quota: &quota, Cpus: "0-7"}
tests := []struct {
name string
containers []ContainerConfig
Expand All @@ -123,11 +129,14 @@ func TestCalculateSandboxCPUs(t *testing.T) {
{"2-constrained", []ContainerConfig{constrained, constrained}, 8},
{"3-mix-constraints", []ContainerConfig{unconstrained, constrained, constrained}, 8},
{"3-constrained", []ContainerConfig{constrained, constrained, constrained}, 12},
{"unconstrained-1-cpuset", []ContainerConfig{unconstrained, unconstrained, unconstrainedCpusets0_1}, 2},
{"unconstrained-2-cpuset", []ContainerConfig{unconstrainedCpusets0_1, unconstrainedCpusets2}, 3},
{"constrained-cpuset", []ContainerConfig{constrainedCpusets0_7}, 4},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sandbox.config.Containers = tt.containers
got := sandbox.calculateSandboxCPUs()
got, _ := sandbox.calculateSandboxCPUs()
assert.Equal(t, got, tt.want)
})
}
Expand Down Expand Up @@ -1422,24 +1431,25 @@ func TestSandbox_SetupSandboxCgroup(t *testing.T) {
}
}

func getContainerConfigWithCPUSet(cpuset string) ContainerConfig {
func getContainerConfigWithCPUSet(cpuset, memset string) ContainerConfig {
return ContainerConfig{
Resources: specs.LinuxResources{
CPU: &specs.LinuxCPU{
Cpus: cpuset,
Mems: memset,
},
},
}
}

func getSimpleSandbox(cpuset0, cpuset1, cpuset2 string) *Sandbox {
func getSimpleSandbox(cpusets, memsets [3]string) *Sandbox {
sandbox := Sandbox{}

sandbox.config = &SandboxConfig{
Containers: []ContainerConfig{
getContainerConfigWithCPUSet(cpuset0),
getContainerConfigWithCPUSet(cpuset1),
getContainerConfigWithCPUSet(cpuset2),
getContainerConfigWithCPUSet(cpusets[0], memsets[0]),
getContainerConfigWithCPUSet(cpusets[1], memsets[1]),
getContainerConfigWithCPUSet(cpusets[2], memsets[2]),
},
}

Expand All @@ -1449,80 +1459,97 @@ func getSimpleSandbox(cpuset0, cpuset1, cpuset2 string) *Sandbox {
func TestGetSandboxCpuSet(t *testing.T) {

tests := []struct {
name string
cpuset0 string
cpuset1 string
cpuset2 string
result string
wantErr bool
name string
cpusets [3]string
memsets [3]string
cpuResult string
memResult string
wantErr bool
}{
{
"single, no cpuset",
"",
"",
[3]string{"", "", ""},
[3]string{"", "", ""},
"",
"",
false,
},
{
"single cpuset",
[3]string{"0", "", ""},
[3]string{"", "", ""},
"0",
"",
"",
"0",
false,
},
{
"two duplicate cpuset",
"0",
[3]string{"0", "0", ""},
[3]string{"", "", ""},
"0",
"",
"0",
false,
},
{
"3 cpusets",
"0-3",
"5-7",
"1",
[3]string{"0-3", "5-7", "1"},
[3]string{"", "", ""},
"0-3,5-7",
"",
false,
},

{
"weird, but should be okay",
"0-3",
"99999",
"",
[3]string{"0-3", "99999", ""},
[3]string{"", "", ""},
"0-3,99999",
"",
false,
},
{
"two, overlapping cpuset",
[3]string{"0-3", "1-2", ""},
[3]string{"", "", ""},
"0-3",
"1-2",
"",
"0-3",
false,
},
{
"garbage, should fail",
"7 beard-seconds",
"Audrey + 7",
"Elliott - 17",
[3]string{"7 beard-seconds", "Audrey + 7", "Elliott - 17"},
[3]string{"", "", ""},
"",
"",
true,
},
{
"cpuset and memset",
[3]string{"0-3", "1-2", ""},
[3]string{"0", "1", "0-1"},
"0-3",
"0-1",
false,
},
{
"memset",
[3]string{"0-3", "1-2", ""},
[3]string{"0", "3", ""},
"0-3",
"0,3",
false,
},
}
for _, tt := range tests {

t.Run(tt.name, func(t *testing.T) {
s := getSimpleSandbox(tt.cpuset0, tt.cpuset1, tt.cpuset2)
res, err := s.getSandboxCPUSet()
s := getSimpleSandbox(tt.cpusets, tt.memsets)
res, _, err := s.getSandboxCPUSet()
if (err != nil) != tt.wantErr {
t.Errorf("getSandboxCPUSet() error = %v, wantErr %v", err, tt.wantErr)
}
if res != tt.result {
t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.result)
if res != tt.cpuResult {
t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.cpuResult)
}
})
}
Expand Down

0 comments on commit 9934054

Please sign in to comment.