Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

Commit

Permalink
shimv2: plugin the block backed rootfs directly instead of mount it
Browse files Browse the repository at this point in the history 
When the container's rootfs is block storage backed such as devmapper,
shimv2 will not mount it on the host, instead it insert it into hypervisor
as a block device directly.

If kata's config set "disable_block_device_use" as true, it will mount
the rootfs onto host as before.

Fixes:#1158

Signed-off-by: lifupan <[email protected]>
  • Loading branch information
@lifupan
lifupan committed Apr 2, 2019
1 parent 628ea46 commit 52c66d2
Show file tree
Hide file tree
Showing 3 changed files with 133 additions and 64 deletions.
152 changes: 116 additions & 36 deletions containerd-shim-v2/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,29 +13,34 @@ import (
"github.com/containerd/typeurl"
vc "github.com/kata-containers/runtime/virtcontainers"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
"github.com/pkg/errors"
"os"
"path/filepath"

taskAPI "github.com/containerd/containerd/runtime/v2/task"

"github.com/kata-containers/runtime/pkg/katautils"
"github.com/opencontainers/runtime-spec/specs-go"

containerd_types "github.com/containerd/containerd/api/types"
"github.com/containerd/containerd/mount"
"github.com/sirupsen/logrus"
// only register the proto type
_ "github.com/containerd/containerd/runtime/linux/runctypes"
crioption "github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1"
)

func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns string) (*container, error) {

detach := !r.Terminal

// Checks the MUST and MUST NOT from OCI runtime specification
bundlePath, err := validBundle(r.ID, r.Bundle)
if err != nil {
return nil, err
rootFs := vc.RootFs{Mounted: s.mount}
if len(r.Rootfs) == 1 {
m := r.Rootfs[0]
rootFs.Source = m.Source
rootFs.Type = m.Type
rootFs.Options = m.Options
}

ociSpec, err := oci.ParseConfigJSON(bundlePath)
detach := !r.Terminal
ociSpec, bundlePath, err := loadSpec(r, netns)
if err != nil {
return nil, err
}
Expand All @@ -45,32 +50,8 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns
return nil, err
}

// Todo:
// Since there is a bug in kata for sharedPidNs, here to
// remove the pidns to disable the sharePidNs temporarily,
// once kata fixed this issue, we can remove this line.
// For the bug, please see:
// https://github.com/kata-containers/runtime/issues/930
removeNamespace(&ociSpec, specs.PIDNamespace)

//set the network namespace path
//this set will be applied to sandbox's
//network config and has nothing to
//do with containers in the sandbox since
//networkNamespace has been ignored by
//kata-agent in sandbox.

for _, n := range ociSpec.Linux.Namespaces {
if n.Type != specs.NetworkNamespace {
continue
}

if n.Path == "" {
n.Path = netns
}
}

disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
rootfs := filepath.Join(r.Bundle, "rootfs")

switch containerType {
case vc.PodSandbox:
Expand All @@ -83,8 +64,23 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns
return nil, err
}

defer func() {
if err != nil && s.mount {
if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
logrus.WithError(err2).Warn("failed to cleanup rootfs mount")
}
}
}()

s.mount = true
if err = checkAndMount(s, r); err != nil {
return nil, err
}

rootFs.Mounted = s.mount

katautils.HandleFactory(ctx, vci, s.config)
sandbox, _, err := katautils.CreateSandbox(ctx, vci, ociSpec, *s.config, r.ID, bundlePath, "", disableOutput, false, true)
sandbox, _, err := katautils.CreateSandbox(ctx, vci, *ociSpec, *s.config, rootFs, r.ID, bundlePath, "", disableOutput, false, true)
if err != nil {
return nil, err
}
Expand All @@ -95,20 +91,74 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns
return nil, fmt.Errorf("BUG: Cannot start the container, since the sandbox hasn't been created")
}

_, err = katautils.CreateContainer(ctx, vci, s.sandbox, ociSpec, r.ID, bundlePath, "", disableOutput, true)
if s.mount {
defer func() {
if err != nil {
if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
logrus.WithError(err2).Warn("failed to cleanup rootfs mount")
}
}
}()

if err = doMount(r.Rootfs, rootfs); err != nil {
return nil, err
}
}

_, err = katautils.CreateContainer(ctx, vci, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput, true)
if err != nil {
return nil, err
}
}

container, err := newContainer(s, r, containerType, &ociSpec)
container, err := newContainer(s, r, containerType, ociSpec)
if err != nil {
return nil, err
}

return container, nil
}

func loadSpec(r *taskAPI.CreateTaskRequest, netns string) (*oci.CompatOCISpec, string, error) {
// Checks the MUST and MUST NOT from OCI runtime specification
bundlePath, err := validBundle(r.ID, r.Bundle)
if err != nil {
return nil, "", err
}

ociSpec, err := oci.ParseConfigJSON(bundlePath)
if err != nil {
return nil, "", err
}

//set the network namespace path
//this set will be applied to sandbox's
//network config and has nothing to
//do with containers in the sandbox since
//networkNamespace has been ignored by
//kata-agent in sandbox.

for _, n := range ociSpec.Linux.Namespaces {
if n.Type != specs.NetworkNamespace {
continue
}

if n.Path == "" {
n.Path = netns
}
}

// Todo:
// Since there is a bug in kata for sharedPidNs, here to
// remove the pidns to disable the sharePidNs temporarily,
// once kata fixed this issue, we can remove this line.
// For the bug, please see:
// https://github.com/kata-containers/runtime/issues/930
removeNamespace(&ociSpec, specs.PIDNamespace)

return &ociSpec, bundlePath, nil
}

func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest) (*oci.RuntimeConfig, error) {
var configPath string

Expand Down Expand Up @@ -142,3 +192,33 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest) (*oci.RuntimeCo

return &runtimeConfig, nil
}

func checkAndMount(s *service, r *taskAPI.CreateTaskRequest) error {
if len(r.Rootfs) == 1 {
m := r.Rootfs[0]

if katautils.IsBlockDevice(m.Source) && !s.config.HypervisorConfig.DisableBlockDeviceUse {
s.mount = false
return nil
}
}
rootfs := filepath.Join(r.Bundle, "rootfs")
if err := doMount(r.Rootfs, rootfs); err != nil {
return err
}
return nil
}

func doMount(mounts []*containerd_types.Mount, rootfs string) error {
for _, rm := range mounts {
m := &mount.Mount{
Type: rm.Type,
Source: rm.Source,
Options: rm.Options,
}
if err := m.Mount(rootfs); err != nil {
return errors.Wrapf(err, "failed to mount rootfs component %v", m)
}
}
return nil
}
8 changes: 5 additions & 3 deletions containerd-shim-v2/delete.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,11 @@ func deleteContainer(ctx context.Context, s *service, c *container) error {
return err
}

rootfs := path.Join(c.bundle, "rootfs")
if err := mount.UnmountAll(rootfs, 0); err != nil {
logrus.WithError(err).Warn("failed to cleanup rootfs mount")
if s.mount {
rootfs := path.Join(c.bundle, "rootfs")
if err := mount.UnmountAll(rootfs, 0); err != nil {
logrus.WithError(err).Warn("failed to cleanup rootfs mount")
}
}

delete(s.containers, c.id)
Expand Down
37 changes: 12 additions & 25 deletions containerd-shim-v2/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,13 @@ import (
"io/ioutil"
"os"
sysexec "os/exec"
"path/filepath"
"sync"
"syscall"
"time"

eventstypes "github.com/containerd/containerd/api/events"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/events"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/namespaces"
cdruntime "github.com/containerd/containerd/runtime"
cdshim "github.com/containerd/containerd/runtime/v2/shim"
Expand Down Expand Up @@ -72,6 +70,7 @@ func New(ctx context.Context, id string, publisher events.Publisher) (cdshim.Shi
containers: make(map[string]*container),
events: make(chan interface{}, chSize),
ec: make(chan exit, bufferSize),
mount: false,
}

go s.processExits()
Expand Down Expand Up @@ -99,6 +98,10 @@ type service struct {
// pid directly.
pid uint32

// if the container's rootfs is block device backed, kata shimv2
// will not do the rootfs mount.
mount bool

context context.Context
sandbox vc.VCSandbox
containers map[string]*container
Expand Down Expand Up @@ -310,39 +313,23 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *
s.mu.Lock()
defer s.mu.Unlock()

var c *container
var netns string

//the network namespace created by cni plugin
netns, err := namespaces.NamespaceRequired(ctx)
netns, err = namespaces.NamespaceRequired(ctx)
if err != nil {
return nil, errors.Wrap(err, "create namespace")
}

rootfs := filepath.Join(r.Bundle, "rootfs")
defer func() {
if err != nil {
if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
logrus.WithError(err2).Warn("failed to cleanup rootfs mount")
}
}
}()
for _, rm := range r.Rootfs {
m := &mount.Mount{
Type: rm.Type,
Source: rm.Source,
Options: rm.Options,
}
if err := m.Mount(rootfs); err != nil {
return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m)
}
}

container, err := create(ctx, s, r, netns)
c, err = create(ctx, s, r, netns)
if err != nil {
return nil, err
}

container.status = task.StatusCreated
c.status = task.StatusCreated

s.containers[r.ID] = container
s.containers[r.ID] = c

s.send(&eventstypes.TaskCreate{
ContainerID: r.ID,
Expand Down

0 comments on commit 52c66d2

Please sign in to comment.