diff --git a/pkg/pillar/containerd/containerd.go b/pkg/pillar/containerd/containerd.go index bebba4664b..1454b234e6 100644 --- a/pkg/pillar/containerd/containerd.go +++ b/pkg/pillar/containerd/containerd.go @@ -21,6 +21,7 @@ import ( "github.com/containerd/containerd/api/services/tasks/v1" "github.com/containerd/containerd/cio" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/mount" @@ -34,12 +35,11 @@ import ( "github.com/lf-edge/eve/pkg/pillar/utils/persist" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" - "github.com/opencontainers/runtime-spec/specs-go" + runtimespecs "github.com/opencontainers/runtime-spec/specs-go" "github.com/vishvananda/netlink" v1stat "github.com/containerd/cgroups/stats/v1" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - spec "github.com/opencontainers/image-spec/specs-go/v1" + imagespecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" ) @@ -179,7 +179,7 @@ func (client *Client) CtrWriteBlob(ctx context.Context, blobHash string, expecte return fmt.Errorf("CtrWriteBlob: exception while validating hash format of %s. %v", blobHash, err) } if err := content.WriteBlob(ctx, client.contentStore, blobHash, reader, - spec.Descriptor{Digest: expectedDigest, Size: int64(expectedSize)}); err != nil { + imagespecs.Descriptor{Digest: expectedDigest, Size: int64(expectedSize)}); err != nil { return fmt.Errorf("CtrWriteBlob: Exception while writing blob: %s. %s", blobHash, err.Error()) } return nil @@ -207,7 +207,7 @@ func (client *Client) CtrReadBlob(ctx context.Context, blobHash string) (io.Read if err != nil { return nil, fmt.Errorf("CtrReadBlob: Exception getting info of blob: %s. %s", blobHash, err.Error()) } - readerAt, err := client.contentStore.ReaderAt(ctx, spec.Descriptor{Digest: shaDigest}) + readerAt, err := client.contentStore.ReaderAt(ctx, imagespecs.Descriptor{Digest: shaDigest}) if err != nil { return nil, fmt.Errorf("CtrReadBlob: Exception while reading blob: %s. %s", blobHash, err.Error()) } @@ -304,20 +304,48 @@ func (client *Client) CtrDeleteImage(ctx context.Context, reference string) erro return client.ctrdClient.ImageService().Delete(ctx, reference) } -// CtrPrepareSnapshot creates snapshot for the given image +// CtrCreateEmptySnapshot creates an empty snapshot with the given snapshotID or returns the existing snapshot if it already exists. +func (client *Client) CtrCreateEmptySnapshot(ctx context.Context, snapshotID string) ([]mount.Mount, error) { + if err := client.verifyCtr(ctx, true); err != nil { + return nil, fmt.Errorf("CtrCreateEmptySnapshot: exception while verifying ctrd client: %s", err.Error()) + } + snapshotter := client.ctrdClient.SnapshotService(defaultSnapshotter) + snapshotMount, err := snapshotter.Mounts(ctx, snapshotID) + if errdefs.IsNotFound(err) { + logrus.Debugf("Snapshot %s does not exist, creating it", snapshotID) + snapshotMount, err = client.CtrPrepareSnapshot(ctx, snapshotID, nil) + if err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } else { + logrus.Debugf("Snapshot %s already exists, reusing it", snapshotID) + } + return snapshotMount, nil +} + +// CtrPrepareSnapshot creates snapshot for the given image or a clean one if no image is provided. func (client *Client) CtrPrepareSnapshot(ctx context.Context, snapshotID string, image containerd.Image) ([]mount.Mount, error) { if err := client.verifyCtr(ctx, true); err != nil { return nil, fmt.Errorf("CtrPrepareSnapshot: exception while verifying ctrd client: %s", err.Error()) } - // use rootfs unpacked image to create a writable snapshot with default snapshotter - diffIDs, err := image.RootFS(ctx) - if err != nil { - err = fmt.Errorf("CtrPrepareSnapshot: Could not load rootfs of image: %v. %v", image.Name(), err) - return nil, err + + var parent string + if image == nil { + // create a clean writable snapshot if no image is provided + parent = "" + } else { + // use rootfs unpacked image to create a writable snapshot with default snapshotter + diffIDs, err := image.RootFS(ctx) + if err != nil { + err = fmt.Errorf("CtrPrepareSnapshot: Could not load rootfs of image: %v. %v", image.Name(), err) + return nil, err + } + parent = identity.ChainID(diffIDs).String() } snapshotter := client.ctrdClient.SnapshotService(defaultSnapshotter) - parent := identity.ChainID(diffIDs).String() labels := map[string]string{"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339)} return snapshotter.Prepare(ctx, snapshotID, parent, snapshots.WithLabels(labels)) } @@ -354,6 +382,22 @@ func (client *Client) CtrListSnapshotInfo(ctx context.Context) ([]snapshots.Info return snapshotInfoList, nil } +// CtrSnapshotExists checks if a snapshot with the given snapshotName exists in containerd's snapshot store +func (client *Client) CtrSnapshotExists(ctx context.Context, snapshotName string) (bool, error) { + if err := client.verifyCtr(ctx, true); err != nil { + return false, err + } + + snapshotter := client.ctrdClient.SnapshotService(defaultSnapshotter) + _, err := snapshotter.Stat(ctx, snapshotName) + if errdefs.IsNotFound(err) { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + // CtrGetSnapshotUsage returns snapshot's usage for snapshotID present in containerd's snapshot store func (client *Client) CtrGetSnapshotUsage(ctx context.Context, snapshotID string) (*snapshots.Usage, error) { if err := client.verifyCtr(ctx, true); err != nil { @@ -546,7 +590,7 @@ func (client *Client) CtrListTaskIds(ctx context.Context) ([]string, error) { } // CtrNewContainer starts a new container with a specific spec and specOpts -func (client *Client) CtrNewContainer(ctx context.Context, spec specs.Spec, specOpts []oci.SpecOpts, name string, containerImage containerd.Image) (containerd.Container, error) { +func (client *Client) CtrNewContainer(ctx context.Context, spec runtimespecs.Spec, specOpts []oci.SpecOpts, name string, containerImage containerd.Image) (containerd.Container, error) { opts := []containerd.NewContainerOpts{ containerd.WithImage(containerImage), @@ -560,13 +604,13 @@ func (client *Client) CtrNewContainer(ctx context.Context, spec specs.Spec, spec // CtrNewContainerWithPersist starts a new container with /persist mounted func (client *Client) CtrNewContainerWithPersist(ctx context.Context, name string, containerImage containerd.Image) (containerd.Container, error) { - var spec specs.Spec + var spec runtimespecs.Spec - spec.Root = &specs.Root{ + spec.Root = &runtimespecs.Root{ Readonly: false, } - mount := specs.Mount{ + mount := runtimespecs.Mount{ Destination: "/persist", Type: "bind", Source: "/persist", @@ -575,7 +619,7 @@ func (client *Client) CtrNewContainerWithPersist(ctx context.Context, name strin specOpts := []oci.SpecOpts{ oci.WithDefaultSpec(), oci.WithImageConfig(containerImage), - oci.WithMounts([]specs.Mount{mount}), + oci.WithMounts([]runtimespecs.Mount{mount}), oci.WithDefaultUnixDevices, } @@ -808,7 +852,7 @@ func prepareProcess(pid int, VifList []types.VifInfo) error { logrus.Infof("prepareProcess(%d, %v)", pid, VifList) for _, iface := range VifList { if iface.Vif == "" { - return fmt.Errorf("Interface requires a name") + return fmt.Errorf("interface requires a name") } var link netlink.Link @@ -846,8 +890,8 @@ func prepareProcess(pid int, VifList []types.VifInfo) error { return nil } -func getSavedImageInfo(containerPath string) (ocispec.Image, error) { - var image ocispec.Image +func getSavedImageInfo(containerPath string) (imagespecs.Image, error) { + var image imagespecs.Image data, err := os.ReadFile(filepath.Join(containerPath, imageConfigFilename)) if err != nil { diff --git a/pkg/pillar/containerd/oci.go b/pkg/pillar/containerd/oci.go index f5fe3d71a3..0f273c9f93 100644 --- a/pkg/pillar/containerd/oci.go +++ b/pkg/pillar/containerd/oci.go @@ -45,13 +45,14 @@ var dhcpcdScript = []string{"eve", "exec", "pillar", "/opt/zededa/bin/dhcpcd.sh" // for all the different task usecases type ociSpec struct { specs.Spec - name string - client *Client - exposedPorts map[string]struct{} - volumes map[string]struct{} - labels map[string]string - stopSignal string - service bool + name string + client *Client + exposedPorts map[string]struct{} + volumes map[string]struct{} + labels map[string]string + stopSignal string + service bool + containerOpts []containerd.NewContainerOpts } // OCISpec provides methods to manipulate OCI runtime specifications and create containers based on them @@ -114,7 +115,42 @@ func (s *ociSpec) AddLoader(volume string) error { return err } - spec.Root = &specs.Root{Readonly: true, Path: filepath.Join(volume, "rootfs")} + // we're gonna use a little hack: since we already have the rootfs of a xen-tools container + // laid out on disk, but don't have it in a form of a snapshot or an image, we're going to + // create an empty snapshot and then overlay the rootfs on top of it - this way we can save + // ourselves copying the rootfs around and still have the newest version of xen-tools on every + // boot, while the original xen-tools rootfs stays read-only + + ctrdCtx, done := s.client.CtrNewUserServicesCtx() + defer done() + + // create a clean snapshot + snapshotName := s.name + snapshotMount, err := s.client.CtrCreateEmptySnapshot(ctrdCtx, snapshotName) + if err != nil { + return err + } + + // remove fs from the end of snapshotMount + snapshotPath := strings.TrimSuffix(snapshotMount[0].Source, "/fs") + logrus.Debugf("Snapshot path: %s", snapshotPath) + + xenToolsMount := specs.Mount{ + Type: "overlay", + Source: "overlay", + Destination: "/", + Options: []string{ + "index=off", + "workdir=" + snapshotPath + "/work", + "upperdir=" + snapshotPath + "/fs", + "lowerdir=" + volume + "/rootfs", + }} + + // we need to prepend the loader mount to the existing mounts to make sure it's mounted first because it's the rootfs + spec.Mounts = append([]specs.Mount{xenToolsMount}, spec.Mounts...) + + s.containerOpts = append(s.containerOpts, containerd.WithSnapshot(snapshotName)) + spec.Linux.Resources = s.Linux.Resources spec.Linux.CgroupsPath = s.Linux.CgroupsPath @@ -261,11 +297,14 @@ func (s *ociSpec) Load(file *os.File) error { func (s *ociSpec) CreateContainer(removeExisting bool) error { ctrdCtx, done := s.client.CtrNewUserServicesCtx() defer done() - _, err := s.client.ctrdClient.NewContainer(ctrdCtx, s.name, containerd.WithSpec(&s.Spec)) + + containerOpts := append(s.containerOpts, containerd.WithSpec(&s.Spec)) + + _, err := s.client.ctrdClient.NewContainer(ctrdCtx, s.name, containerOpts...) // if container exists, is stopped and we are asked to remove existing - try that if err != nil && removeExisting { _ = s.client.CtrDeleteContainer(ctrdCtx, s.name) - _, err = s.client.ctrdClient.NewContainer(ctrdCtx, s.name, containerd.WithSpec(&s.Spec)) + _, err = s.client.ctrdClient.NewContainer(ctrdCtx, s.name, containerOpts...) } return err } diff --git a/pkg/pillar/containerd/oci_test.go b/pkg/pillar/containerd/oci_test.go index 3fc4ebec24..3ac956365f 100644 --- a/pkg/pillar/containerd/oci_test.go +++ b/pkg/pillar/containerd/oci_test.go @@ -4,14 +4,12 @@ package containerd import ( - "encoding/json" "fmt" "log" "net" "os" "path" "path/filepath" - "reflect" "testing" zconfig "github.com/lf-edge/eve-api/go/config" @@ -475,6 +473,9 @@ func TestCreateMountPointExecEnvFiles(t *testing.T) { } ] }` + + client := initClient(t) + //create a temp dir to hold resulting files dir, _ := os.MkdirTemp("/tmp", "podfiles") rootDir := path.Join(dir, "runx") @@ -497,7 +498,6 @@ func TestCreateMountPointExecEnvFiles(t *testing.T) { t.Errorf("failed to write to a runtime spec file %v", err) } - client := &Client{} spec, err := client.NewOciSpec("test", false) if err != nil { t.Errorf("failed to create new OCI spec %v", err) @@ -623,6 +623,8 @@ func TestPrepareMount(t *testing.T) { ] }` + client := initClient(t) + err := os.MkdirAll(path.Join(oldTempRootPath, "tmp"), 0777) if err != nil { t.Errorf("TestPrepareMount: Failed to create %s: %s", oldTempRootPath, err.Error()) @@ -677,7 +679,6 @@ func TestPrepareMount(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - client := &Client{} spec, err := client.NewOciSpec("test", false) if err != nil { t.Errorf("failed to create new OCI spec") @@ -790,10 +791,10 @@ func TestEnvs(t *testing.T) { g.Expect(spec.Process.Args).To(Equal([]string{"echo", "hello"})) } -func TestAddLoader(t *testing.T) { - g := NewGomegaWithT(t) - specTemplate := ociSpec{ +func specTemplate(client *Client) ociSpec { + return ociSpec{ name: "test", + client: client, volumes: map[string]struct{}{"/myvol": {}, "/hisvol": {}}, Spec: specs.Spec{ Process: &specs.Process{ @@ -809,8 +810,39 @@ func TestAddLoader(t *testing.T) { Linux: &specs.Linux{CgroupsPath: "/foo/bar/baz"}, }, } - spec1 := deepCopy(specTemplate).(ociSpec) - spec2 := deepCopy(specTemplate).(ociSpec) +} + +func initClient(t *testing.T) *Client { + client, err := NewContainerdClient(false) + if err != nil { + t.Skipf("test must be run on a system with a functional containerd") + } + + t.Cleanup(func() { + ctrdCtx, done := client.CtrNewUserServicesCtx() + defer done() + + // clean up the snapshots that were created in AddLoader() + snapshots, err := client.CtrListSnapshotInfo(ctrdCtx) + if err != nil { + t.Errorf("failed to list snapshots %v", err) + } + for _, snapshot := range snapshots { + if err := client.CtrRemoveSnapshot(ctrdCtx, snapshot.Name); err != nil { + t.Errorf("failed to remove snapshot %s %v", snapshot.Name, err) + } + } + + // add more things here if other containerd artifacts were created + }) + + return client +} + +func TestAddLoader(t *testing.T) { + client := initClient(t) + + g := NewGomegaWithT(t) tmpdir, err := os.MkdirTemp("/tmp", "volume") if err != nil { @@ -822,37 +854,31 @@ func TestAddLoader(t *testing.T) { log.Fatalf("failed to create tmpfile %v", err) } + spec1 := specTemplate(client) g.Expect(spec1.AddLoader("/foo/bar/baz")).To(HaveOccurred()) g.Expect(spec1.AddLoader(tmpdir)).ToNot(HaveOccurred()) - g.Expect(spec1.Root).To(Equal(&specs.Root{Path: filepath.Join(tmpdir, "rootfs"), Readonly: true})) + g.Expect(spec1.Root).To(Equal(&specs.Root{Path: "rootfs", Readonly: false})) g.Expect(spec1.Linux.CgroupsPath).To(Equal("/foo/bar/baz")) - g.Expect(spec1.Mounts[9]).To(Equal(specs.Mount{Destination: "/mnt/rootfs/test", Type: "bind", Source: "/test", Options: []string{"ro"}})) - g.Expect(spec1.Mounts[0]).To(Equal(specs.Mount{Destination: "/dev", Type: "bind", Source: "/dev", Options: []string{"rw", "rbind", "rshared"}})) + g.Expect(spec1.Mounts[10]).To(Equal(specs.Mount{Destination: "/mnt/rootfs/test", Type: "bind", Source: "/test", Options: []string{"ro"}})) + g.Expect(spec1.Mounts[1]).To(Equal(specs.Mount{Destination: "/dev", Type: "bind", Source: "/dev", Options: []string{"rw", "rbind", "rshared"}})) + spec2 := specTemplate(client) spec2.Root.Path = tmpdir g.Expect(spec2.AddLoader(tmpdir)).ToNot(HaveOccurred()) - g.Expect(spec2.Root).To(Equal(&specs.Root{Path: filepath.Join(tmpdir, "rootfs"), Readonly: true})) + g.Expect(spec2.Root).To(Equal(&specs.Root{Path: "rootfs", Readonly: false})) g.Expect(spec2.Linux.CgroupsPath).To(Equal("/foo/bar/baz")) - g.Expect(spec2.Mounts[11]).To(Equal(specs.Mount{Destination: "/mnt/rootfs/test", Type: "bind", Source: "/test", Options: []string{"ro"}})) - g.Expect(spec2.Mounts[10]).To(Equal(specs.Mount{Destination: "/mnt/modules", Type: "bind", Source: "/lib/modules", Options: []string{"rbind", "ro", "rslave"}})) - g.Expect(spec2.Mounts[9]).To(Equal(specs.Mount{Destination: "/mnt", Type: "bind", Source: path.Join(tmpdir, ".."), Options: []string{"rbind", "rw", "rslave"}})) - g.Expect(spec2.Mounts[0]).To(Equal(specs.Mount{Destination: "/dev", Type: "bind", Source: "/dev", Options: []string{"rw", "rbind", "rshared"}})) -} - -func deepCopy(in interface{}) interface{} { - b, _ := json.Marshal(in) - p := reflect.New(reflect.TypeOf(in)) - output := p.Interface() - _ = json.Unmarshal(b, output) - val := reflect.ValueOf(output) - val = val.Elem() - return val.Interface() + g.Expect(spec2.Mounts[12]).To(Equal(specs.Mount{Destination: "/mnt/rootfs/test", Type: "bind", Source: "/test", Options: []string{"ro"}})) + g.Expect(spec2.Mounts[11]).To(Equal(specs.Mount{Destination: "/mnt/modules", Type: "bind", Source: "/lib/modules", Options: []string{"rbind", "ro", "rslave"}})) + g.Expect(spec2.Mounts[10]).To(Equal(specs.Mount{Destination: "/mnt", Type: "bind", Source: path.Join(tmpdir, ".."), Options: []string{"rbind", "rw", "rslave"}})) + g.Expect(spec2.Mounts[1]).To(Equal(specs.Mount{Destination: "/dev", Type: "bind", Source: "/dev", Options: []string{"rw", "rbind", "rshared"}})) } func TestDenyAllDevicesInSpec(t *testing.T) { t.Parallel() + client := initClient(t) + // create a temp dir to hold resulting files dir, _ := os.MkdirTemp("/tmp", "podfiles") rootDir := path.Join(dir, "runx") @@ -870,7 +896,6 @@ func TestDenyAllDevicesInSpec(t *testing.T) { t.Errorf("failed to write to a runtime spec file %v", err) } - client := &Client{} spec, err := client.NewOciSpec("test", false) if err != nil { t.Errorf("failed to create new OCI spec %v", err) diff --git a/pkg/pillar/hypervisor/containerd.go b/pkg/pillar/hypervisor/containerd.go index fc8fbcc7c2..e5a6c08647 100644 --- a/pkg/pillar/hypervisor/containerd.go +++ b/pkg/pillar/hypervisor/containerd.go @@ -191,6 +191,11 @@ func (ctx ctrdContext) Delete(domainName string) error { if err := ctx.ctrdClient.CtrDeleteContainer(ctrdCtx, domainName); err != nil { return err } + if persistentSnapshotExists, _ := ctx.ctrdClient.CtrSnapshotExists(ctrdCtx, domainName); persistentSnapshotExists { + if err := ctx.ctrdClient.CtrRemoveSnapshot(ctrdCtx, domainName); err != nil { + return err + } + } vifsTaskDir := filepath.Join(vifsDir, domainName) if err := os.RemoveAll(vifsTaskDir); err != nil { return logError("cannot clear vifs task dir %s: %v", vifsTaskDir, err) @@ -203,10 +208,17 @@ func (ctx ctrdContext) Cleanup(domainName string) error { ctrdCtx, done := ctx.ctrdClient.CtrNewUserServicesCtx() defer done() container, _ := ctx.ctrdClient.CtrLoadContainer(ctrdCtx, domainName) - if container == nil { - return nil + if container != nil { + if err := ctx.Delete(domainName); err != nil { + return err + } } - return ctx.Delete(domainName) + if persistentSnapshotExists, _ := ctx.ctrdClient.CtrSnapshotExists(ctrdCtx, domainName); persistentSnapshotExists { + if err := ctx.ctrdClient.CtrRemoveSnapshot(ctrdCtx, domainName); err != nil { + return err + } + } + return nil } func (ctx ctrdContext) Annotations(domainName string) (map[string]string, error) { diff --git a/pkg/pillar/hypervisor/hypervisor.go b/pkg/pillar/hypervisor/hypervisor.go index c242f36f49..4993c431ba 100644 --- a/pkg/pillar/hypervisor/hypervisor.go +++ b/pkg/pillar/hypervisor/hypervisor.go @@ -20,6 +20,10 @@ import ( "github.com/sirupsen/logrus" ) +const ( + xenToolsPath = "/containers/services/xen-tools" +) + // Hypervisor provides methods for manipulating domains on the host type Hypervisor interface { Name() string diff --git a/pkg/pillar/hypervisor/kvm.go b/pkg/pillar/hypervisor/kvm.go index 88157d70a4..41eca912fc 100644 --- a/pkg/pillar/hypervisor/kvm.go +++ b/pkg/pillar/hypervisor/kvm.go @@ -847,7 +847,7 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig if err != nil { return logError("failed to load OCI spec for domain %s: %v", status.DomainName, err) } - if err = spec.AddLoader("/containers/services/xen-tools"); err != nil { + if err = spec.AddLoader(xenToolsPath); err != nil { return logError("failed to add kvm hypervisor loader to domain %s: %v", status.DomainName, err) } overhead, err := vmmOverhead(domainName, domainUUID, int64(config.Memory), int64(config.VMMMaxMem), int64(config.MaxCpus), int64(config.VCpus), config.IoAdapterList, aa, globalConfig) diff --git a/pkg/pillar/hypervisor/xen.go b/pkg/pillar/hypervisor/xen.go index 729f0d7432..f25fa63339 100644 --- a/pkg/pillar/hypervisor/xen.go +++ b/pkg/pillar/hypervisor/xen.go @@ -133,7 +133,7 @@ func (ctx xenContext) Setup(status types.DomainStatus, config types.DomainConfig if err != nil { return logError("failed to load OCI spec for domain %s: %v", status.DomainName, err) } - if err = spec.AddLoader("/containers/services/xen-tools"); err != nil { + if err = spec.AddLoader(xenToolsPath); err != nil { return logError("failed to add xen hypervisor loader to domain %s: %v", status.DomainName, err) } spec.Get().Process.Args = []string{"/etc/xen/scripts/xen-start", status.DomainName, file.Name()}