diff --git a/pkg/oci/oci.go b/pkg/oci/oci.go index 78eb7885..5de7e8e3 100644 --- a/pkg/oci/oci.go +++ b/pkg/oci/oci.go @@ -2,10 +2,20 @@ package oci import ( "context" + "os" + "path" + "runtime" + "sync" + "github.com/klauspost/pgzip" + "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/umoci" "github.com/opencontainers/umoci/oci/casext" + "github.com/opencontainers/umoci/oci/layer" "github.com/pkg/errors" + "stackerbuild.io/stacker/pkg/log" + "stackerbuild.io/stacker/pkg/squashfs" ) func LookupManifest(oci casext.Engine, tag string) (ispec.Manifest, error) { @@ -76,3 +86,108 @@ func UpdateImageConfig(oci casext.Engine, name string, newConfig ispec.Image, ne return desc, nil } + +func hasDirEntries(dir string) bool { + ents, err := os.ReadDir(dir) + if err != nil { + return false + } + return len(ents) != 0 +} + +var tarEx sync.Mutex + +// UnpackOne - unpack a single layer (Descriptor) found in ociDir to extractDir +// +// The result of calling unpackOne is either error or the contents available +// at the provided extractDir. The extractDir should be either empty or +// fully populated with this layer. +func UnpackOne(l ispec.Descriptor, ociDir string, extractDir string) error { + // population of a dir is not atomic, at least for tar extraction. + // As a result, we could hasDirEntries(extractDir) at the same time that + // something is un-populating that dir due to a failed extraction (like + // os.RemoveAll below). + // There needs to be a lock on the extract dir (scoped to the overlay storage backend). + // A sync.RWMutex would work well here since it is safe to check as long + // as no one is populating or unpopulating. + if hasDirEntries(extractDir) { + // the directory was already populated. + return nil + } + + if squashfs.IsSquashfsMediaType(l.MediaType) { + return squashfs.ExtractSingleSquash( + path.Join(ociDir, "blobs", "sha256", l.Digest.Encoded()), extractDir) + } + switch l.MediaType { + case ispec.MediaTypeImageLayer, ispec.MediaTypeImageLayerGzip: + tarEx.Lock() + defer tarEx.Unlock() + + oci, err := umoci.OpenLayout(ociDir) + if err != nil { + return err + } + defer oci.Close() + + compressed, err := oci.GetBlob(context.Background(), l.Digest) + if err != nil { + return err + } + defer compressed.Close() + + uncompressed, err := pgzip.NewReader(compressed) + if err != nil { + return err + } + + err = layer.UnpackLayer(extractDir, uncompressed, nil) + if err != nil { + if rmErr := os.RemoveAll(extractDir); rmErr != nil { + log.Errorf("Failed to remove dir '%s' after failed extraction: %v", extractDir, rmErr) + } + } + return err + } + return errors.Errorf("unknown media type %s", l.MediaType) +} + +// Unpack an image with "tag" from "ociLayout" into paths returned by "pathfunc" +func Unpack(ociLayout, tag string, pathfunc func(digest.Digest) string) (int, error) { + oci, err := umoci.OpenLayout(ociLayout) + if err != nil { + return -1, err + } + defer oci.Close() + + manifest, err := LookupManifest(oci, tag) + if err != nil { + return -1, err + } + + pool := NewThreadPool(runtime.NumCPU()) + + seen := map[digest.Digest]bool{} + for _, curLayer := range manifest.Layers { + // avoid calling UnpackOne twice for the same digest + if seen[curLayer.Digest] { + continue + } + seen[curLayer.Digest] = true + + // copy layer to avoid race on pool access. + l := curLayer + pool.Add(func(ctx context.Context) error { + return UnpackOne(l, ociLayout, pathfunc(l.Digest)) + }) + } + + pool.DoneAddingJobs() + + err = pool.Run() + if err != nil { + return -1, err + } + + return len(manifest.Layers), nil +} diff --git a/pkg/overlay/pool.go b/pkg/oci/pool.go similarity index 98% rename from pkg/overlay/pool.go rename to pkg/oci/pool.go index 24d9ff3d..0b67f231 100644 --- a/pkg/overlay/pool.go +++ b/pkg/oci/pool.go @@ -1,4 +1,4 @@ -package overlay +package oci import ( "context" diff --git a/pkg/overlay/overlay.go b/pkg/overlay/overlay.go index 3feec586..1b4a5ea2 100644 --- a/pkg/overlay/overlay.go +++ b/pkg/overlay/overlay.go @@ -14,6 +14,7 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/sys/unix" + "stackerbuild.io/stacker/pkg/oci" "stackerbuild.io/stacker/pkg/types" ) @@ -142,14 +143,6 @@ func (o *overlay) SetupEmptyRootfs(name string) error { return ovl.write(o.config, name) } -func hasDirEntries(dir string) bool { - ents, err := os.ReadDir(dir) - if err != nil { - return false - } - return len(ents) != 0 -} - func (o *overlay) snapshot(source string, target string) error { err := o.Create(target) if err != nil { @@ -168,7 +161,7 @@ func (o *overlay) snapshot(source string, target string) error { } ociDir := path.Join(o.config.StackerDir, "layer-bases", "oci") for _, layer := range manifest.Layers { - err := unpackOne(layer, ociDir, overlayPath(o.config.RootFSDir, layer.Digest, "overlay")) + err := oci.UnpackOne(layer, ociDir, overlayPath(o.config.RootFSDir, layer.Digest, "overlay")) if err != nil { return errors.Wrapf(err, "Failed mounting %#v", layer) } diff --git a/pkg/overlay/pack.go b/pkg/overlay/pack.go index 38109cc0..69af6196 100644 --- a/pkg/overlay/pack.go +++ b/pkg/overlay/pack.go @@ -8,12 +8,9 @@ import ( "os" "path" "path/filepath" - "runtime" "strings" - "sync" "time" - "github.com/klauspost/pgzip" "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/umoci" @@ -24,14 +21,13 @@ import ( "github.com/pkg/xattr" "stackerbuild.io/stacker/pkg/lib" "stackerbuild.io/stacker/pkg/log" + "stackerbuild.io/stacker/pkg/oci" stackeroci "stackerbuild.io/stacker/pkg/oci" "stackerbuild.io/stacker/pkg/squashfs" "stackerbuild.io/stacker/pkg/storage" "stackerbuild.io/stacker/pkg/types" ) -var tarEx sync.Mutex - // Container image layers are often tar.gz, however there is nothing in the // spec or documentation which standardizes compression params which can cause // different layer hashes even for the same tar. So picking compression params @@ -51,56 +47,6 @@ func overlayPath(rootfs string, d digest.Digest, subdirs ...string) string { return path.Join(dirs...) } -func (o *overlay) Unpack(tag, name string) error { - cacheDir := path.Join(o.config.StackerDir, "layer-bases", "oci") - oci, err := umoci.OpenLayout(cacheDir) - if err != nil { - return err - } - defer oci.Close() - - manifest, err := stackeroci.LookupManifest(oci, tag) - if err != nil { - return err - } - - pool := NewThreadPool(runtime.NumCPU()) - - seen := map[digest.Digest]bool{} - for _, curLayer := range manifest.Layers { - // avoid calling unpackOne twice for the same digest - if seen[curLayer.Digest] { - continue - } - seen[curLayer.Digest] = true - - // copy layer to avoid race on pool access. - l := curLayer - pool.Add(func(ctx context.Context) error { - return unpackOne(l, cacheDir, overlayPath(o.config.RootFSDir, l.Digest, "overlay")) - }) - } - - pool.DoneAddingJobs() - - err = pool.Run() - if err != nil { - return err - } - - err = o.Create(name) - if err != nil { - return err - } - - ovl, err := newOverlayMetadataFromOCI(oci, tag) - if err != nil { - return err - } - - return ovl.write(o.config, name) -} - func ConvertAndOutput(config types.StackerConfig, tag, name string, layerType types.LayerType) error { cacheDir := path.Join(config.StackerDir, "layer-bases", "oci") cacheOCI, err := umoci.OpenLayout(cacheDir) @@ -681,57 +627,32 @@ func repackOverlay(config types.StackerConfig, name string, layerTypes []types.L return ovl.write(config, name) } -// unpackOne - unpack a single layer (Descriptor) found in ociDir to extractDir -// -// The result of calling unpackOne is either error or the contents available -// at the provided extractDir. The extractDir should be either empty or -// fully populated with this layer. -func unpackOne(l ispec.Descriptor, ociDir string, extractDir string) error { - // population of a dir is not atomic, at least for tar extraction. - // As a result, we could hasDirEntries(extractDir) at the same time that - // something is un-populating that dir due to a failed extraction (like - // os.RemoveAll below). - // There needs to be a lock on the extract dir (scoped to the overlay storage backend). - // A sync.RWMutex would work well here since it is safe to check as long - // as no one is populating or unpopulating. - if hasDirEntries(extractDir) { - // the directory was already populated. - return nil - } +func (o *overlay) Unpack(tag, name string) error { + cacheDir := path.Join(o.config.StackerDir, "layer-bases", "oci") - if squashfs.IsSquashfsMediaType(l.MediaType) { - return squashfs.ExtractSingleSquash( - path.Join(ociDir, "blobs", "sha256", l.Digest.Encoded()), extractDir) + pathfunc := func(digest digest.Digest) string { + return overlayPath(o.config.RootFSDir, digest, "overlay") } - switch l.MediaType { - case ispec.MediaTypeImageLayer, ispec.MediaTypeImageLayerGzip: - tarEx.Lock() - defer tarEx.Unlock() - oci, err := umoci.OpenLayout(ociDir) - if err != nil { - return err - } - defer oci.Close() + _, err := oci.Unpack(cacheDir, tag, pathfunc) + if err != nil { + return err + } - compressed, err := oci.GetBlob(context.Background(), l.Digest) - if err != nil { - return err - } - defer compressed.Close() + err = o.Create(name) + if err != nil { + return err + } - uncompressed, err := pgzip.NewReader(compressed) - if err != nil { - return err - } + oci, err := umoci.OpenLayout(cacheDir) + if err != nil { + return err + } - err = layer.UnpackLayer(extractDir, uncompressed, nil) - if err != nil { - if rmErr := os.RemoveAll(extractDir); rmErr != nil { - log.Errorf("Failed to remove dir '%s' after failed extraction: %v", extractDir, rmErr) - } - } + ovl, err := newOverlayMetadataFromOCI(oci, tag) + if err != nil { return err } - return errors.Errorf("unknown media type %s", l.MediaType) + + return ovl.write(o.config, name) } diff --git a/pkg/stacker/import.go b/pkg/stacker/import.go index 377b64ff..84f91806 100644 --- a/pkg/stacker/import.go +++ b/pkg/stacker/import.go @@ -13,6 +13,7 @@ import ( "github.com/vbatts/go-mtree" "stackerbuild.io/stacker/pkg/lib" "stackerbuild.io/stacker/pkg/log" + "stackerbuild.io/stacker/pkg/oci" "stackerbuild.io/stacker/pkg/types" ) @@ -294,6 +295,38 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st } return p, nil + } else if url.Scheme == "docker" { + if idest != "" && idest[len(idest)-1:] != "/" { + return "", errors.Errorf("The destination path must be directory: %s", idest) + } + + is := types.ImageSource{Type: "docker", Url: i} + if err := importContainersImage(is, c, false); err != nil { + return "", err + } + + tag, err := is.ParseTag() + if err != nil { + return "", err + } + + pathfunc := func(digest digest.Digest) string { + _ = os.Remove(cache) + return cache + } + + ociDir := path.Join(c.StackerDir, "layer-bases", "oci") + + n, err := oci.Unpack(ociDir, tag, pathfunc) + if err != nil { + return "", err + } + + if n > 1 { + return "", errors.Errorf("Currently supporting single-layer container image imports") + } + + return cache, nil } return "", errors.Errorf("unsupported url scheme %s", i) diff --git a/test/import.bats b/test/import.bats index 629e1784..4e8e6452 100644 --- a/test/import.bats +++ b/test/import.bats @@ -517,3 +517,33 @@ src_folder_dest_non_existent_folder_case4: EOF stacker build } + +@test "importing container images" { + cat > stacker.yaml <