diff --git a/Gopkg.lock b/Gopkg.lock index 15cb233c..425ce081 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -29,7 +29,7 @@ [[projects]] name = "github.com/containers/virtcontainers" packages = [".","pkg/annotations","pkg/cni","pkg/ethtool","pkg/hyperstart","pkg/oci","pkg/uuid","pkg/vcMock"] - revision = "506f076ba79c77ae00da9daf60e4fed747cd8ab3" + revision = "4589f5bbf82318dac126d75c6aaeb78428dfa551" [[projects]] name = "github.com/davecgh/go-spew" @@ -111,6 +111,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "bee732b22990fc7ebc0225845b8cf7648e9a76a813deec226fdbfebff948ad51" + inputs-digest = "9f3cee761a2dfac357203a6e56bb8ce8235cbdcc9347681e083d2ca9cc7137a0" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 1b7e569f..80c257f9 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -84,10 +84,6 @@ name = "github.com/containernetworking/plugins" revision = "e256564546e8d1ca8a36911f8445c11929043221" -[[constraint]] - name = "github.com/containers/virtcontainers" - revision = "506f076ba79c77ae00da9daf60e4fed747cd8ab3" - [[constraint]] name = "github.com/davecgh/go-spew" revision = "346938d642f2ec3594ed81d874461961cd0faa76" @@ -147,3 +143,7 @@ [[constraint]] name = "golang.org/x/sys" revision = "314a259e304ff91bd6985da2a7149bbf91237993" + +[[constraint]] + name = "github.com/containers/virtcontainers" + revision = "4589f5bbf82318dac126d75c6aaeb78428dfa551" diff --git a/vendor/github.com/containers/virtcontainers/api.go b/vendor/github.com/containers/virtcontainers/api.go index 06497d3a..80c56b43 100644 --- a/vendor/github.com/containers/virtcontainers/api.go +++ b/vendor/github.com/containers/virtcontainers/api.go @@ -38,6 +38,10 @@ func SetLogger(logger logrus.FieldLogger) { // CreatePod is the virtcontainers pod creation entry point. // CreatePod creates a pod and its containers. It does not start them. func CreatePod(podConfig PodConfig) (VCPod, error) { + return createPodFromConfig(podConfig) +} + +func createPodFromConfig(podConfig PodConfig) (*Pod, error) { // Create the pod. p, err := createPod(podConfig) if err != nil { @@ -168,8 +172,12 @@ func StartPod(podID string) (VCPod, error) { return nil, err } + return startPod(p) +} + +func startPod(p *Pod) (*Pod, error) { // Start it - err = p.start() + err := p.start() if err != nil { return nil, err } @@ -214,14 +222,7 @@ func StopPod(podID string) (VCPod, error) { // RunPod is the virtcontainers pod running entry point. // RunPod creates a pod and its containers and then it starts them. func RunPod(podConfig PodConfig) (VCPod, error) { - // Create the pod. - p, err := createPod(podConfig) - if err != nil { - return nil, err - } - - // Store it. - err = p.storePod() + p, err := createPodFromConfig(podConfig) if err != nil { return nil, err } @@ -232,59 +233,7 @@ func RunPod(podConfig PodConfig) (VCPod, error) { } defer unlockPod(lockFile) - // Initialize the network. - netNsPath, netNsCreated, err := p.network.init(p.config.NetworkConfig) - if err != nil { - return nil, err - } - - // Execute prestart hooks inside netns - err = p.network.run(netNsPath, func() error { - return p.config.Hooks.preStartHooks() - }) - if err != nil { - return nil, err - } - - // Add the network - networkNS, err := p.network.add(*p, p.config.NetworkConfig, netNsPath, netNsCreated) - if err != nil { - return nil, err - } - - // Store the network - err = p.storage.storePodNetwork(p.id, networkNS) - if err != nil { - return nil, err - } - - // Start the VM - err = p.startVM(netNsPath) - if err != nil { - return nil, err - } - - // Start shims - if err := p.startShims(); err != nil { - return nil, err - } - - // Start the pod - err = p.start() - if err != nil { - p.delete() - return nil, err - } - - // Execute poststart hooks inside netns - err = p.network.run(networkNS.NetNsPath, func() error { - return p.config.Hooks.postStartHooks() - }) - if err != nil { - return nil, err - } - - return p, nil + return startPod(p) } // ListPod is the virtcontainers pod listing entry point. diff --git a/vendor/github.com/containers/virtcontainers/api_test.go b/vendor/github.com/containers/virtcontainers/api_test.go index fba9cded..e4d2da22 100644 --- a/vendor/github.com/containers/virtcontainers/api_test.go +++ b/vendor/github.com/containers/virtcontainers/api_test.go @@ -274,7 +274,7 @@ func TestCreatePodFailing(t *testing.T) { config := PodConfig{} p, err := CreatePod(config) - if p != nil || err == nil { + if p.(*Pod) != nil || err == nil { t.Fatal() } } @@ -595,6 +595,9 @@ func TestStatusPodSuccessfulStateReady(t *testing.T) { KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), + DefaultVCPUs: defaultVCPUs, + DefaultMemSz: defaultMemSzMiB, + DefaultBridges: defaultBridges, } expectedStatus := PodStatus{ @@ -648,6 +651,9 @@ func TestStatusPodSuccessfulStateRunning(t *testing.T) { KernelPath: filepath.Join(testDir, testKernel), ImagePath: filepath.Join(testDir, testImage), HypervisorPath: filepath.Join(testDir, testHypervisor), + DefaultVCPUs: defaultVCPUs, + DefaultMemSz: defaultMemSzMiB, + DefaultBridges: defaultBridges, } expectedStatus := PodStatus{ diff --git a/vendor/github.com/containers/virtcontainers/bridge.go b/vendor/github.com/containers/virtcontainers/bridge.go new file mode 100644 index 00000000..c25aba98 --- /dev/null +++ b/vendor/github.com/containers/virtcontainers/bridge.go @@ -0,0 +1,104 @@ +// +// Copyright (c) 2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import "fmt" + +type bridgeType string + +const ( + pciBridge bridgeType = "pci" + pcieBridge = "pcie" +) + +const pciBridgeMaxCapacity = 30 + +// Bridge is a bridge where devices can be hot plugged +type Bridge struct { + // Address contains information about devices plugged and its address in the bridge + Address map[uint32]string + + // Type is the type of the bridge (pci, pcie, etc) + Type bridgeType + + //ID is used to identify the bridge in the hypervisor + ID string +} + +// NewBridges creates n new pci(e) bridges depending of the machine type +func NewBridges(n uint32, machine string) []Bridge { + var bridges []Bridge + var bt bridgeType + + switch machine { + case QemuQ35: + // currently only pci bridges are supported + // qemu-2.10 will introduce pcie bridges + fallthrough + case QemuPC: + bt = pciBridge + default: + return nil + } + + for i := uint32(0); i < n; i++ { + bridges = append(bridges, Bridge{ + Type: bt, + ID: fmt.Sprintf("%s-bridge-%d", bt, i), + Address: make(map[uint32]string), + }) + } + + return bridges +} + +// addDevice on success adds the device ID to the bridge and return the address +// where the device was added, otherwise an error is returned +func (b *Bridge) addDevice(ID string) (uint32, error) { + var addr uint32 + + // looking for the first available address + for i := uint32(1); i <= pciBridgeMaxCapacity; i++ { + if _, ok := b.Address[i]; !ok { + addr = i + break + } + } + + if addr == 0 { + return 0, fmt.Errorf("Unable to hot plug device on bridge: there are not empty slots") + } + + // save address and device + b.Address[addr] = ID + return addr, nil +} + +// removeDevice on success removes the device ID from the bridge and return nil, +// otherwise an error is returned +func (b *Bridge) removeDevice(ID string) error { + // check if the device was hot plugged in the bridge + for addr, devID := range b.Address { + if devID == ID { + // free address to re-use the same slot with other devices + delete(b.Address, addr) + return nil + } + } + + return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID) +} diff --git a/vendor/github.com/containers/virtcontainers/bridge_test.go b/vendor/github.com/containers/virtcontainers/bridge_test.go new file mode 100644 index 00000000..2ca13561 --- /dev/null +++ b/vendor/github.com/containers/virtcontainers/bridge_test.go @@ -0,0 +1,65 @@ +// +// Copyright (c) 2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewBridges(t *testing.T) { + assert := assert.New(t) + var countBridges uint32 = 1 + + bridges := NewBridges(countBridges, "") + assert.Nil(bridges) + + bridges = NewBridges(countBridges, QemuQ35) + assert.Len(bridges, int(countBridges)) + + b := bridges[0] + assert.NotEmpty(b.ID) + assert.NotNil(b.Address) +} + +func TestAddRemoveDevice(t *testing.T) { + assert := assert.New(t) + var countBridges uint32 = 1 + + // create a bridge + bridges := NewBridges(countBridges, "") + assert.Nil(bridges) + bridges = NewBridges(countBridges, QemuQ35) + assert.Len(bridges, int(countBridges)) + + // add device + devID := "abc123" + b := bridges[0] + addr, err := b.addDevice(devID) + assert.NoError(err) + if addr < 1 { + assert.Fail("address cannot be less then 1") + } + + // remove device + err = b.removeDevice("") + assert.Error(err) + + err = b.removeDevice(devID) + assert.NoError(err) +} diff --git a/vendor/github.com/containers/virtcontainers/cni.go b/vendor/github.com/containers/virtcontainers/cni.go index 23aa3358..81af7680 100644 --- a/vendor/github.com/containers/virtcontainers/cni.go +++ b/vendor/github.com/containers/virtcontainers/cni.go @@ -123,7 +123,7 @@ func (n *cni) updateEndpointsFromScan(networkNS *NetworkNamespace) error { return err } - for idx, endpoint := range endpoints { + for _, endpoint := range endpoints { for _, ep := range networkNS.Endpoints { if ep.Name() == endpoint.Name() { // Update endpoint properties with info from @@ -131,8 +131,12 @@ func (n *cni) updateEndpointsFromScan(networkNS *NetworkNamespace) error { // cannot provide it. prop := endpoint.Properties() prop.DNS = ep.Properties().DNS - ep.SetProperties(prop) - endpoints[idx] = ep + endpoint.SetProperties(prop) + + switch e := endpoint.(type) { + case *VirtualEndpoint: + e.NetPair = ep.(*VirtualEndpoint).NetPair + } break } } diff --git a/vendor/github.com/containers/virtcontainers/container.go b/vendor/github.com/containers/virtcontainers/container.go index 2318a39b..0d089a39 100644 --- a/vendor/github.com/containers/virtcontainers/container.go +++ b/vendor/github.com/containers/virtcontainers/container.go @@ -225,11 +225,7 @@ func (c *Container) startShim() error { c.process = *process - if err := c.storeProcess(); err != nil { - return err - } - - return nil + return c.storeProcess() } func (c *Container) storeProcess() error { @@ -806,11 +802,7 @@ func (c *Container) hotplugDrive() error { return err } - if err := c.setStateFstype(fsType); err != nil { - return err - } - - return nil + return c.setStateFstype(fsType) } // isDriveUsed checks if a drive has been used for container rootfs diff --git a/vendor/github.com/containers/virtcontainers/device.go b/vendor/github.com/containers/virtcontainers/device.go index 1805978c..9f4b48b4 100644 --- a/vendor/github.com/containers/virtcontainers/device.go +++ b/vendor/github.com/containers/virtcontainers/device.go @@ -477,9 +477,5 @@ func bindDevicetoHost(bdf, hostDriver, vendorDeviceID string) error { "driver-path": bindDriverPath, }).Info("Binding back device to host driver") - if err := writeToFile(bindDriverPath, []byte(bdf)); err != nil { - return err - } - - return nil + return writeToFile(bindDriverPath, []byte(bdf)) } diff --git a/vendor/github.com/containers/virtcontainers/filesystem.go b/vendor/github.com/containers/virtcontainers/filesystem.go index d26b2aae..706ebda3 100644 --- a/vendor/github.com/containers/virtcontainers/filesystem.go +++ b/vendor/github.com/containers/virtcontainers/filesystem.go @@ -42,6 +42,9 @@ const ( // networkFileType represents a network file type (pod only) networkFileType + // hypervisorFileType represents a hypervisor file type (pod only) + hypervisorFileType + // processFileType represents a process file type processFileType @@ -64,6 +67,9 @@ const stateFile = "state.json" // networkFile is the file name storing a pod network. const networkFile = "network.json" +// hypervisorFile is the file name storing a hypervisor's state. +const hypervisorFile = "hypervisor.json" + // processFile is the file name storing a container process. const processFile = "process.json" @@ -109,6 +115,10 @@ type resourceStorage interface { fetchPodNetwork(podID string) (NetworkNamespace, error) storePodNetwork(podID string, networkNS NetworkNamespace) error + // Hypervisor resources + fetchHypervisorState(podID string, state interface{}) error + storeHypervisorState(podID string, state interface{}) error + // Container resources storeContainerResource(podID, containerID string, resource podResource, data interface{}) error deleteContainerResources(podID, containerID string, resources []podResource) error @@ -319,7 +329,7 @@ func (fs *filesystem) fetchDeviceFile(fileData []byte, devices *[]Device) error func resourceNeedsContainerID(podSpecific bool, resource podResource) bool { switch resource { - case lockFileType, networkFileType: + case lockFileType, networkFileType, hypervisorFileType: // pod-specific resources return false default: @@ -342,7 +352,7 @@ func resourceDir(podSpecific bool, podID, containerID string, resource podResour case configFileType: path = configStoragePath break - case stateFileType, networkFileType, processFileType, lockFileType, mountsFileType, devicesFileType: + case stateFileType, networkFileType, processFileType, lockFileType, mountsFileType, devicesFileType, hypervisorFileType: path = runStoragePath break default: @@ -378,6 +388,8 @@ func (fs *filesystem) resourceURI(podSpecific bool, podID, containerID string, r filename = stateFile case networkFileType: filename = networkFile + case hypervisorFileType: + filename = hypervisorFile case processFileType: filename = processFile case lockFileType: @@ -429,6 +441,7 @@ func (fs *filesystem) commonResourceChecks(podSpecific bool, podID, containerID case configFileType: case stateFileType: case networkFileType: + case hypervisorFileType: case processFileType: case mountsFileType: case devicesFileType: @@ -557,11 +570,7 @@ func (fs *filesystem) fetchResource(podSpecific bool, podID, containerID string, return err } - if err := fs.fetchFile(path, resource, data); err != nil { - return err - } - - return nil + return fs.fetchFile(path, resource, data) } func (fs *filesystem) storePodResource(podID string, resource podResource, data interface{}) error { @@ -598,10 +607,23 @@ func (fs *filesystem) fetchPodNetwork(podID string) (NetworkNamespace, error) { return networkNS, nil } +func (fs *filesystem) fetchHypervisorState(podID string, state interface{}) error { + return fs.fetchResource(true, podID, "", hypervisorFileType, state) +} + func (fs *filesystem) storePodNetwork(podID string, networkNS NetworkNamespace) error { return fs.storePodResource(podID, networkFileType, networkNS) } +func (fs *filesystem) storeHypervisorState(podID string, state interface{}) error { + hypervisorFile, _, err := fs.resourceURI(true, podID, "", hypervisorFileType) + if err != nil { + return err + } + + return fs.storeFile(hypervisorFile, state) +} + func (fs *filesystem) deletePodResources(podID string, resources []podResource) error { if resources == nil { resources = []podResource{configFileType, stateFileType} diff --git a/vendor/github.com/containers/virtcontainers/hyperstart.go b/vendor/github.com/containers/virtcontainers/hyperstart.go index 2c7fbea7..3691b77a 100644 --- a/vendor/github.com/containers/virtcontainers/hyperstart.go +++ b/vendor/github.com/containers/virtcontainers/hyperstart.go @@ -365,11 +365,7 @@ func (h *hyper) createPod(pod *Pod) (err error) { return err } - if err := pod.hypervisor.addDevice(sharedVolume, fsDev); err != nil { - return err - } - - return nil + return pod.hypervisor.addDevice(sharedVolume, fsDev) } func (h *hyper) capabilities() capabilities { @@ -440,6 +436,15 @@ func (h *hyper) startPod(pod Pod) error { // stopPod is the agent Pod stopping implementation for hyperstart. func (h *hyper) stopPod(pod Pod) error { + proxyCmd := hyperstartProxyCmd{ + cmd: hyperstart.DestroyPod, + message: nil, + } + + if _, err := h.proxy.sendCmd(proxyCmd); err != nil { + return err + } + return nil } diff --git a/vendor/github.com/containers/virtcontainers/hypervisor.go b/vendor/github.com/containers/virtcontainers/hypervisor.go index 0e23958a..9ebdc62c 100644 --- a/vendor/github.com/containers/virtcontainers/hypervisor.go +++ b/vendor/github.com/containers/virtcontainers/hypervisor.go @@ -44,6 +44,8 @@ const ( defaultVCPUs = 1 // 2 GiB defaultMemSzMiB = 2048 + + defaultBridges = 1 ) // deviceType describes a virtualized device type. @@ -161,6 +163,10 @@ type HypervisorConfig struct { // Pod configuration VMConfig.Memory overwrites this. DefaultMemSz uint32 + // DefaultBridges specifies default number of bridges for the VM. + // Bridges can be used to hot plug devices + DefaultBridges uint32 + // MemPrealloc specifies if the memory should be pre-allocated MemPrealloc bool @@ -203,6 +209,10 @@ func (conf *HypervisorConfig) valid() (bool, error) { conf.DefaultMemSz = defaultMemSzMiB } + if conf.DefaultBridges == 0 { + conf.DefaultBridges = defaultBridges + } + return true, nil } @@ -434,9 +444,10 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) { // hypervisor is the virtcontainers hypervisor interface. // The default hypervisor implementation is Qemu. type hypervisor interface { - init(config HypervisorConfig) error + init(pod *Pod) error createPod(podConfig PodConfig) error - startPod(startCh, stopCh chan struct{}) error + startPod() error + waitPod(timeout int) error stopPod() error pausePod() error resumePod() error @@ -445,4 +456,5 @@ type hypervisor interface { hotplugRemoveDevice(devInfo interface{}, devType deviceType) error getPodConsole(podID string) string capabilities() capabilities + getState() interface{} } diff --git a/vendor/github.com/containers/virtcontainers/hypervisor_test.go b/vendor/github.com/containers/virtcontainers/hypervisor_test.go index 5e2f23a0..10de21ce 100644 --- a/vendor/github.com/containers/virtcontainers/hypervisor_test.go +++ b/vendor/github.com/containers/virtcontainers/hypervisor_test.go @@ -179,6 +179,7 @@ func TestHypervisorConfigDefaults(t *testing.T) { HypervisorPath: "", DefaultVCPUs: defaultVCPUs, DefaultMemSz: defaultMemSzMiB, + DefaultBridges: defaultBridges, } if reflect.DeepEqual(hypervisorConfig, hypervisorConfigDefaultsExpected) == false { t.Fatal() diff --git a/vendor/github.com/containers/virtcontainers/mock_hypervisor.go b/vendor/github.com/containers/virtcontainers/mock_hypervisor.go index 785b5c29..9ee6ac2e 100644 --- a/vendor/github.com/containers/virtcontainers/mock_hypervisor.go +++ b/vendor/github.com/containers/virtcontainers/mock_hypervisor.go @@ -19,8 +19,8 @@ package virtcontainers type mockHypervisor struct { } -func (m *mockHypervisor) init(config HypervisorConfig) error { - valid, err := config.valid() +func (m *mockHypervisor) init(pod *Pod) error { + valid, err := pod.config.HypervisorConfig.valid() if valid == false || err != nil { return err } @@ -36,9 +36,11 @@ func (m *mockHypervisor) createPod(podConfig PodConfig) error { return nil } -func (m *mockHypervisor) startPod(startCh, stopCh chan struct{}) error { - var msg struct{} - startCh <- msg +func (m *mockHypervisor) startPod() error { + return nil +} + +func (m *mockHypervisor) waitPod(timeout int) error { return nil } @@ -69,3 +71,7 @@ func (m *mockHypervisor) hotplugRemoveDevice(devInfo interface{}, devType device func (m *mockHypervisor) getPodConsole(podID string) string { return "" } + +func (m *mockHypervisor) getState() interface{} { + return nil +} diff --git a/vendor/github.com/containers/virtcontainers/mock_hypervisor_test.go b/vendor/github.com/containers/virtcontainers/mock_hypervisor_test.go index f155c4dc..da52cf51 100644 --- a/vendor/github.com/containers/virtcontainers/mock_hypervisor_test.go +++ b/vendor/github.com/containers/virtcontainers/mock_hypervisor_test.go @@ -19,31 +19,34 @@ package virtcontainers import ( "fmt" "testing" - "time" ) func TestMockHypervisorInit(t *testing.T) { var m *mockHypervisor - wrongConfig := HypervisorConfig{ - KernelPath: "", - ImagePath: "", - HypervisorPath: "", + pod := &Pod{ + config: &PodConfig{ + HypervisorConfig: HypervisorConfig{ + KernelPath: "", + ImagePath: "", + HypervisorPath: "", + }, + }, } - err := m.init(wrongConfig) - if err == nil { + // wrong config + if err := m.init(pod); err == nil { t.Fatal() } - rightConfig := HypervisorConfig{ + pod.config.HypervisorConfig = HypervisorConfig{ KernelPath: fmt.Sprintf("%s/%s", testDir, testKernel), ImagePath: fmt.Sprintf("%s/%s", testDir, testImage), HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - err = m.init(rightConfig) - if err != nil { + // right config + if err := m.init(pod); err != nil { t.Fatal(err) } } @@ -53,8 +56,7 @@ func TestMockHypervisorCreatePod(t *testing.T) { config := PodConfig{} - err := m.createPod(config) - if err != nil { + if err := m.createPod(config); err != nil { t.Fatal(err) } } @@ -62,24 +64,23 @@ func TestMockHypervisorCreatePod(t *testing.T) { func TestMockHypervisorStartPod(t *testing.T) { var m *mockHypervisor - startCh := make(chan struct{}) - stopCh := make(chan struct{}) + if err := m.startPod(); err != nil { + t.Fatal(err) + } +} - go m.startPod(startCh, stopCh) +func TestMockHypervisorWaitPod(t *testing.T) { + var m *mockHypervisor - select { - case <-startCh: - break - case <-time.After(time.Second): - t.Fatal("Timeout waiting for start notification") + if err := m.waitPod(0); err != nil { + t.Fatal(err) } } func TestMockHypervisorStopPod(t *testing.T) { var m *mockHypervisor - err := m.stopPod() - if err != nil { + if err := m.stopPod(); err != nil { t.Fatal(err) } } @@ -87,8 +88,7 @@ func TestMockHypervisorStopPod(t *testing.T) { func TestMockHypervisorAddDevice(t *testing.T) { var m *mockHypervisor - err := m.addDevice(nil, imgDev) - if err != nil { + if err := m.addDevice(nil, imgDev); err != nil { t.Fatal(err) } } diff --git a/vendor/github.com/containers/virtcontainers/pod.go b/vendor/github.com/containers/virtcontainers/pod.go index e210c02f..1a4832d1 100644 --- a/vendor/github.com/containers/virtcontainers/pod.go +++ b/vendor/github.com/containers/virtcontainers/pod.go @@ -23,7 +23,6 @@ import ( "strings" "sync" "syscall" - "time" "github.com/sirupsen/logrus" ) @@ -39,6 +38,10 @@ const controlSocket = "ctrl.sock" // to understand if the VM is still alive or not. const monitorSocket = "monitor.sock" +// vmStartTimeout represents the time in seconds a pod can wait before +// to consider the VM starting operation failed. +const vmStartTimeout = 10 + // stateString is a string representing a pod state. type stateString string @@ -593,10 +596,6 @@ func doFetchPod(podConfig PodConfig) (*Pod, error) { return nil, err } - if err := hypervisor.init(podConfig.HypervisorConfig); err != nil { - return nil, err - } - proxy, err := newProxy(podConfig.ProxyType) if err != nil { return nil, err @@ -637,6 +636,11 @@ func doFetchPod(podConfig PodConfig) (*Pod, error) { return nil, err } + if err := p.hypervisor.init(p); err != nil { + p.storage.deletePodResources(p.id, nil) + return nil, err + } + if err := p.hypervisor.createPod(podConfig); err != nil { p.storage.deletePodResources(p.id, nil) return nil, err @@ -744,34 +748,13 @@ func (p *Pod) startSetState() error { return nil } -// startVM starts the VM, ensuring it is started before it returns or issuing -// an error in case of timeout. Then it connects to the agent inside the VM. +// startVM starts the VM. func (p *Pod) startVM(netNsPath string) error { - vmStartedCh := make(chan struct{}) - vmStoppedCh := make(chan struct{}) - const timeout = time.Duration(10) * time.Second - - l := p.Logger() - l.Info("Starting VM") - - go func() { - p.network.run(netNsPath, func() error { - err := p.hypervisor.startPod(vmStartedCh, vmStoppedCh) - return err - }) - }() + p.Logger().Info("Starting VM") - // Wait for the pod started notification - select { - case <-vmStartedCh: - break - case <-time.After(timeout): - return fmt.Errorf("Did not receive the pod started notification (timeout %ds)", timeout) - } - - l.Info("VM started") - - return nil + return p.network.run(netNsPath, func() error { + return p.hypervisor.startPod() + }) } // startShims registers all containers to the proxy and starts one @@ -831,6 +814,14 @@ func (p *Pod) start() error { return err } + l := p.Logger() + + if err := p.hypervisor.waitPod(vmStartTimeout); err != nil { + return err + } + + l.Info("VM started") + if _, _, err := p.proxy.connect(*p, false); err != nil { return err } @@ -851,7 +842,7 @@ func (p *Pod) start() error { } } - p.Logger().Info("started") + l.Info("started") return nil } @@ -942,11 +933,7 @@ func (p *Pod) stopVM() error { return err } - if err := p.hypervisor.stopPod(); err != nil { - return err - } - - return nil + return p.hypervisor.stopPod() } // stop stops a pod. The containers that are making the pod @@ -983,11 +970,7 @@ func (p *Pod) stop() error { return err } - if err := p.stopSetStates(); err != nil { - return err - } - - return nil + return p.stopSetStates() } func (p *Pod) pause() error { @@ -995,11 +978,7 @@ func (p *Pod) pause() error { return err } - if err := p.pauseSetStates(); err != nil { - return err - } - - return nil + return p.pauseSetStates() } func (p *Pod) resume() error { @@ -1007,11 +986,7 @@ func (p *Pod) resume() error { return err } - if err := p.resumeSetStates(); err != nil { - return err - } - - return nil + return p.resumeSetStates() } // list lists all pod running on the host. @@ -1097,11 +1072,7 @@ func (p *Pod) setContainerState(containerID string, state stateString) error { // Let container handle its state update cImpl := c.(*Container) - if err := cImpl.setContainerState(state); err != nil { - return err - } - - return nil + return cImpl.setContainerState(state) } func (p *Pod) setContainersState(state stateString) error { diff --git a/vendor/github.com/containers/virtcontainers/qemu.go b/vendor/github.com/containers/virtcontainers/qemu.go index 7d42ea85..3b235d86 100644 --- a/vendor/github.com/containers/virtcontainers/qemu.go +++ b/vendor/github.com/containers/virtcontainers/qemu.go @@ -21,7 +21,6 @@ import ( "fmt" "os" "path/filepath" - "regexp" "strings" "sync" "time" @@ -32,11 +31,16 @@ import ( ) type qmpChannel struct { - ctx context.Context - path string - disconnectCh chan struct{} - wg sync.WaitGroup - qmp *ciaoQemu.QMP + ctx context.Context + path string + wg sync.WaitGroup + qmp *ciaoQemu.QMP +} + +// QemuState keeps Qemu's state +type QemuState struct { + Bridges []Bridge + UUID string } // qemu is an Hypervisor interface implementation for the Linux qemu hypervisor. @@ -53,6 +57,10 @@ type qemu struct { qemuConfig ciaoQemu.Config nestedRun bool + + pod *Pod + + state QemuState } const defaultQemuPath = "/usr/bin/qemu-system-x86_64" @@ -74,6 +82,8 @@ const ( const qmpCapErrMsg = "Failed to negoatiate QMP capabilities" +const qmpSockPathSizeLimit = 107 + // Mapping between machine types and QEMU binary paths. var qemuPaths = map[string]string{ QemuPCLite: "/usr/bin/qemu-lite-system-x86_64", @@ -380,6 +390,33 @@ func (q *qemu) appendFSDevices(devices []ciaoQemu.Device, podConfig PodConfig) [ return devices } +func (q *qemu) appendBridges(devices []ciaoQemu.Device, podConfig PodConfig) ([]ciaoQemu.Device, error) { + bus := "pci.0" + if podConfig.HypervisorConfig.HypervisorMachineType == QemuQ35 { + bus = "pcie.0" + } + + for idx, b := range q.state.Bridges { + t := ciaoQemu.PCIBridge + if b.Type == pcieBridge { + t = ciaoQemu.PCIEBridge + } + + devices = append(devices, + ciaoQemu.BridgeDevice{ + Type: t, + Bus: bus, + ID: b.ID, + // Each bridge is required to be assigned a unique chassis id > 0 + Chassis: (idx + 1), + SHPC: true, + }, + ) + } + + return devices, nil +} + func (q *qemu) appendConsoles(devices []ciaoQemu.Device, podConfig PodConfig) []ciaoQemu.Device { serial := ciaoQemu.SerialDevice{ Driver: ciaoQemu.VirtioSerial, @@ -435,25 +472,6 @@ func (q *qemu) appendImage(devices []ciaoQemu.Device, podConfig PodConfig) ([]ci return devices, nil } -func (q *qemu) forceUUIDFormat(str string) string { - re := regexp.MustCompile(`[^[0-9,a-f,A-F]]*`) - hexStr := re.ReplaceAllLiteralString(str, ``) - - slice := []byte(hexStr) - sliceLen := len(slice) - - var uuidSlice uuid.UUID - uuidLen := len(uuidSlice) - - if sliceLen > uuidLen { - copy(uuidSlice[:], slice[:uuidLen]) - } else { - copy(uuidSlice[:], slice) - } - - return uuidSlice.String() -} - func (q *qemu) getMachine(name string) (ciaoQemu.Machine, error) { for _, m := range supportedQemuMachines { if m.Type == name { @@ -498,19 +516,32 @@ func (q *qemu) buildPath() error { } // init intializes the Qemu structure. -func (q *qemu) init(config HypervisorConfig) error { - valid, err := config.valid() +func (q *qemu) init(pod *Pod) error { + valid, err := pod.config.HypervisorConfig.valid() if valid == false || err != nil { return err } - q.config = config + q.config = pod.config.HypervisorConfig + q.pod = pod + + if err := pod.storage.fetchHypervisorState(pod.id, &q.state); err != nil { + q.Logger().Debug("Creating bridges") + q.state.Bridges = NewBridges(q.config.DefaultBridges, q.config.HypervisorMachineType) + + q.Logger().Debug("Creating UUID") + q.state.UUID = uuid.Generate().String() + + if err := pod.storage.storeHypervisorState(pod.id, q.state); err != nil { + return err + } + } - if err = q.buildPath(); err != nil { + if err := q.buildPath(); err != nil { return err } - if err = q.buildKernelParams(config); err != nil { + if err := q.buildKernelParams(q.config); err != nil { return err } @@ -521,7 +552,7 @@ func (q *qemu) init(config HypervisorConfig) error { q.Logger().WithField("inside-vm", fmt.Sprintf("%t", nested)).Debug("Checking nesting environment") - if config.DisableNestingChecks { + if q.config.DisableNestingChecks { //Intentionally ignore the nesting check q.nestedRun = false } else { @@ -531,40 +562,6 @@ func (q *qemu) init(config HypervisorConfig) error { return nil } -func (q *qemu) qmpMonitor(connectedCh chan struct{}) { - defer func(qemu *qemu) { - if q.qmpMonitorCh.qmp != nil { - q.qmpMonitorCh.qmp.Shutdown() - } - - q.qmpMonitorCh.wg.Done() - }(q) - - cfg := ciaoQemu.QMPConfig{Logger: newQMPLogger()} - qmp, ver, err := ciaoQemu.QMPStart(q.qmpMonitorCh.ctx, q.qmpMonitorCh.path, cfg, q.qmpMonitorCh.disconnectCh) - if err != nil { - q.Logger().WithError(err).Error("Failed to connect to QEMU instance") - return - } - - q.qmpMonitorCh.qmp = qmp - - q.Logger().WithFields(logrus.Fields{ - "qmp-major-version": ver.Major, - "qmp-minor-version": ver.Minor, - "qmp-micro-version": ver.Micro, - "qmp-capabilities": strings.Join(ver.Capabilities, ","), - }).Infof("QMP details") - - err = q.qmpMonitorCh.qmp.ExecuteQMPCapabilities(q.qmpMonitorCh.ctx) - if err != nil { - q.Logger().WithError(err).Error(qmpCapErrMsg) - return - } - - close(connectedCh) -} - func (q *qemu) setCPUResources(podConfig PodConfig) ciaoQemu.SMP { vcpus := q.config.DefaultVCPUs if podConfig.VMConfig.VCPUs > 0 { @@ -606,6 +603,23 @@ func (q *qemu) setMemoryResources(podConfig PodConfig) (ciaoQemu.Memory, error) return memory, nil } +func (q *qemu) qmpSocketPath(socketName string) (string, error) { + parentDirPath := filepath.Join(runStoragePath, q.pod.id) + if len(parentDirPath) > qmpSockPathSizeLimit { + return "", fmt.Errorf("Parent directory path %q is too long "+ + "(%d characters), could not add any path for the QMP socket", + parentDirPath, len(parentDirPath)) + } + + path := fmt.Sprintf("%s/%s-%s", parentDirPath, q.state.UUID, socketName) + + if len(path) > qmpSockPathSizeLimit { + return path[:qmpSockPathSizeLimit], nil + } + + return path, nil +} + // createPod is the Hypervisor pod creation implementation for ciaoQemu. func (q *qemu) createPod(podConfig PodConfig) error { var devices []ciaoQemu.Device @@ -661,14 +675,28 @@ func (q *qemu) createPod(podConfig PodConfig) error { DriftFix: "slew", } + if q.state.UUID == "" { + return fmt.Errorf("UUID should not be empty") + } + + monitorSockPath, err := q.qmpSocketPath(monitorSocket) + if err != nil { + return err + } + q.qmpMonitorCh = qmpChannel{ ctx: context.Background(), - path: fmt.Sprintf("%s/%s/%s", runStoragePath, podConfig.ID, monitorSocket), + path: monitorSockPath, + } + + controlSockPath, err := q.qmpSocketPath(controlSocket) + if err != nil { + return err } q.qmpControlCh = qmpChannel{ ctx: context.Background(), - path: fmt.Sprintf("%s/%s/%s", runStoragePath, podConfig.ID, controlSocket), + path: controlSockPath, } qmpSockets := []ciaoQemu.QMPSocket{ @@ -693,6 +721,11 @@ func (q *qemu) createPod(podConfig PodConfig) error { return err } + devices, err = q.appendBridges(devices, podConfig) + if err != nil { + return err + } + cpuModel := "host" if q.nestedRun { cpuModel += ",pmu=off" @@ -705,7 +738,7 @@ func (q *qemu) createPod(podConfig PodConfig) error { qemuConfig := ciaoQemu.Config{ Name: fmt.Sprintf("pod-%s", podConfig.ID), - UUID: q.forceUUIDFormat(podConfig.ID), + UUID: q.state.UUID, Path: q.path, Ctx: q.qmpMonitorCh.ctx, Machine: machine, @@ -728,16 +761,61 @@ func (q *qemu) createPod(podConfig PodConfig) error { } // startPod will start the Pod's VM. -func (q *qemu) startPod(startCh, stopCh chan struct{}) error { +func (q *qemu) startPod() error { strErr, err := ciaoQemu.LaunchQemu(q.qemuConfig, newQMPLogger()) if err != nil { return fmt.Errorf("%s", strErr) } - // Start the QMP monitoring thread - q.qmpMonitorCh.disconnectCh = stopCh - q.qmpMonitorCh.wg.Add(1) - q.qmpMonitor(startCh) + return nil +} + +// waitPod will wait for the Pod's VM to be up and running. +func (q *qemu) waitPod(timeout int) error { + defer func(qemu *qemu) { + if q.qmpMonitorCh.qmp != nil { + q.qmpMonitorCh.qmp.Shutdown() + } + }(q) + + if timeout < 0 { + return fmt.Errorf("Invalid timeout %ds", timeout) + } + + cfg := ciaoQemu.QMPConfig{Logger: newQMPLogger()} + + var qmp *ciaoQemu.QMP + var ver *ciaoQemu.QMPVersion + var err error + + timeStart := time.Now() + for { + disconnectCh := make(chan struct{}) + qmp, ver, err = ciaoQemu.QMPStart(q.qmpMonitorCh.ctx, q.qmpMonitorCh.path, cfg, disconnectCh) + if err == nil { + break + } + + if int(time.Now().Sub(timeStart).Seconds()) > timeout { + return fmt.Errorf("Failed to connect to QEMU instance (timeout %ds): %v", timeout, err) + } + + time.Sleep(time.Duration(50) * time.Millisecond) + } + + q.qmpMonitorCh.qmp = qmp + + q.Logger().WithFields(logrus.Fields{ + "qmp-major-version": ver.Major, + "qmp-minor-version": ver.Minor, + "qmp-micro-version": ver.Micro, + "qmp-capabilities": strings.Join(ver.Capabilities, ","), + }).Infof("QMP details") + + if err = q.qmpMonitorCh.qmp.ExecuteQMPCapabilities(q.qmpMonitorCh.ctx); err != nil { + q.Logger().WithError(err).Error(qmpCapErrMsg) + return err + } return nil } @@ -745,11 +823,10 @@ func (q *qemu) startPod(startCh, stopCh chan struct{}) error { // stopPod will stop the Pod's VM. func (q *qemu) stopPod() error { cfg := ciaoQemu.QMPConfig{Logger: newQMPLogger()} - q.qmpControlCh.disconnectCh = make(chan struct{}) - const timeout = time.Duration(10) * time.Second + disconnectCh := make(chan struct{}) q.Logger().Info("Stopping Pod") - qmp, _, err := ciaoQemu.QMPStart(q.qmpControlCh.ctx, q.qmpControlCh.path, cfg, q.qmpControlCh.disconnectCh) + qmp, _, err := ciaoQemu.QMPStart(q.qmpControlCh.ctx, q.qmpControlCh.path, cfg, disconnectCh) if err != nil { q.Logger().WithError(err).Error("Failed to connect to QEMU instance") return err @@ -761,19 +838,7 @@ func (q *qemu) stopPod() error { return err } - if err := qmp.ExecuteQuit(q.qmpMonitorCh.ctx); err != nil { - return err - } - - // Wait for the VM disconnection notification - select { - case <-q.qmpControlCh.disconnectCh: - break - case <-time.After(timeout): - return fmt.Errorf("Did not receive the VM disconnection notification (timeout %ds)", timeout) - } - - return nil + return qmp.ExecuteQuit(q.qmpMonitorCh.ctx) } func (q *qemu) togglePausePod(pause bool) error { @@ -836,6 +901,34 @@ func (q *qemu) qmpSetup() (*ciaoQemu.QMP, error) { return qmp, nil } +func (q *qemu) addDeviceToBridge(ID string) (string, string, error) { + var err error + var addr uint32 + + // looking for an empty address in the bridges + for _, b := range q.state.Bridges { + addr, err = b.addDevice(ID) + if err == nil { + return fmt.Sprintf("0x%x", addr), b.ID, nil + } + } + + return "", "", err +} + +func (q *qemu) removeDeviceFromBridge(ID string) error { + var err error + for _, b := range q.state.Bridges { + err = b.removeDevice(ID) + if err == nil { + // device was removed correctly + return nil + } + } + + return err +} + func (q *qemu) hotplugBlockDevice(drive Drive, op operation) error { defer func(qemu *qemu) { if q.qmpMonitorCh.qmp != nil { @@ -858,10 +951,21 @@ func (q *qemu) hotplugBlockDevice(drive Drive, op operation) error { } driver := "virtio-blk-pci" - if err := q.qmpMonitorCh.qmp.ExecuteDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, ""); err != nil { + + addr, bus, err := q.addDeviceToBridge(drive.ID) + if err != nil { return err } + + if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus); err != nil { + return err + } + } else { + if err := q.removeDeviceFromBridge(drive.ID); err != nil { + return err + } + if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil { return err } @@ -885,11 +989,19 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati } func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) error { - return q.hotplugDevice(devInfo, devType, addDevice) + if err := q.hotplugDevice(devInfo, devType, addDevice); err != nil { + return err + } + + return q.pod.storage.storeHypervisorState(q.pod.id, q.state) } func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) error { - return q.hotplugDevice(devInfo, devType, removeDevice) + if err := q.hotplugDevice(devInfo, devType, removeDevice); err != nil { + return err + } + + return q.pod.storage.storeHypervisorState(q.pod.id, q.state) } func (q *qemu) pausePod() error { @@ -930,3 +1042,7 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { func (q *qemu) getPodConsole(podID string) string { return filepath.Join(runStoragePath, podID, defaultConsole) } + +func (q *qemu) getState() interface{} { + return q.state +} diff --git a/vendor/github.com/containers/virtcontainers/qemu_test.go b/vendor/github.com/containers/virtcontainers/qemu_test.go index f18b55a2..d2ab7663 100644 --- a/vendor/github.com/containers/virtcontainers/qemu_test.go +++ b/vendor/github.com/containers/virtcontainers/qemu_test.go @@ -34,6 +34,7 @@ func newQemuConfig() HypervisorConfig { HypervisorPath: testQemuPath, DefaultVCPUs: defaultVCPUs, DefaultMemSz: defaultMemSzMiB, + DefaultBridges: defaultBridges, } } @@ -375,8 +376,25 @@ func TestQemuInit(t *testing.T) { qemuConfig := newQemuConfig() q := &qemu{} - err := q.init(qemuConfig) - if err != nil { + pod := &Pod{ + id: "testPod", + storage: &filesystem{}, + config: &PodConfig{ + HypervisorConfig: qemuConfig, + }, + } + + // Create parent dir path for hypervisor.json + parentDir := filepath.Join(runStoragePath, pod.id) + if err := os.MkdirAll(parentDir, dirMode); err != nil { + t.Fatalf("Could not create parent directory %s: %v", parentDir, err) + } + + if err := q.init(pod); err != nil { + t.Fatal(err) + } + + if err := os.RemoveAll(parentDir); err != nil { t.Fatal(err) } @@ -396,6 +414,29 @@ func TestQemuInit(t *testing.T) { } } +func TestQemuInitMissingParentDirFail(t *testing.T) { + qemuConfig := newQemuConfig() + q := &qemu{} + + pod := &Pod{ + id: "testPod", + storage: &filesystem{}, + config: &PodConfig{ + HypervisorConfig: qemuConfig, + }, + } + + // Ensure parent dir path for hypervisor.json does not exist. + parentDir := filepath.Join(runStoragePath, pod.id) + if err := os.RemoveAll(parentDir); err != nil { + t.Fatal(err) + } + + if err := q.init(pod); err == nil { + t.Fatal("Qemu init() expected to fail because of missing parent directory for storage") + } +} + func TestQemuSetCPUResources(t *testing.T) { vcpus := 1