support more sources through specific schemes (#7)

Co-authored-by: CrazyMax <[email protected]>

Author: crazy-max

.github/workflows/e2e.yml

@@ -10,7 +10,7 @@ on:
   pull_request:
 
 jobs:
-  e2e:
+  registry:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -34,8 +34,6 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
@@ -66,3 +64,69 @@ jobs:
         name: Dist content
         run: |
           tree -nh ./dist
+
+  docker-daemon:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Build
+        uses: docker/bake-action@v1
+        with:
+          targets: binary
+      -
+        name: Create Dockerfile
+        run: |
+          mkdir ./bin/test
+          cat > ./bin/test/Dockerfile <<EOL
+          FROM alpine
+          RUN mkdir hello && echo "Hello, world!" > /hello/world
+          EOL
+      -
+        name: Build image and load
+        uses: docker/build-push-action@v2
+        with:
+          context: ./bin/test
+          load: true
+          tags: image:local
+      -
+        name: Run
+        run: |
+          ./bin/undock --rm-dist --include /hello docker-daemon://image:local ./dist
+      -
+        name: Dist content
+        run: |
+          tree -nh ./dist
+
+  docker-archive:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Build
+        uses: docker/bake-action@v1
+        with:
+          targets: binary
+      -
+        name: Create docker archive
+        run: |
+          docker pull crazymax/buildx-pkg:latest
+          docker save crazymax/buildx-pkg:latest > archive.tar
+      -
+        name: Run
+        run: |
+          ./bin/undock --rm-dist docker-archive://archive.tar ./dist
+      -
+        name: Dist content
+        run: |
+          tree -nh ./dist

README.md

@@ -22,6 +22,7 @@ local machine with a single command.
 
 ## Features
 
+* Many source support (docker, archive, store, oci, tar)
 * Can extract multi-platform images
 * Include a subset of files/dirs
 * Cache support

docs/index.md

@@ -14,7 +14,7 @@
 
 ---
 
-## What is Undock?
+## What's Undock?
 
 **Undock** is a CLI application that allows you to extract contents of a
 container image in a local folder. This can be useful if you use a registry
@@ -28,6 +28,7 @@ See the [usage examples](usage/examples.md) for more info.
 
 ## Features
 
+* Many source support (docker, archive, store, oci, tar)
 * Can extract multi-platform images
 * Include a subset of files/dirs
 * Cache support

docs/usage/cli.md

@@ -3,7 +3,7 @@
 ## Usage
 
 ```shell
-undock [options]
+undock [options] <source> <dist>
 ```
 
 ## Options
@@ -15,7 +15,7 @@ Usage: undock <source> <dist>
 Extract contents of a container image in a local folder. More info: https://github.com/crazy-max/undock
 
 Arguments:
-  <source>    Source image from a registry. (eg. alpine:latest)
+  <source>    Source image. (eg. alpine:latest)
   <dist>      Dist folder. (eg. ./dist)
 
 Flags:
@@ -29,11 +29,43 @@ Flags:
       --platform=STRING        Enforce platform for source image. (eg. linux/amd64)
       --all                    Extract all architectures if source is a manifest list.
       --include=INCLUDE,...    Include a subset of files/dirs from the source image.
-      --insecure               Allow contacting the registry over HTTP, or HTTPS with failed TLS verification.
+      --insecure               Allow contacting the registry or docker daemon over HTTP, or HTTPS with failed TLS verification.
       --rm-dist                Removes dist folder.
       --wrap                   For a manifest list, merge output in dist folder.
 ```
 
+### Source image
+
+`source` argument can be a container image from a registry, a local docker
+image, a container store reference, etc. Following schemes can be used:
+
+* `containers-storage://<store>`: image located in a local container storage[^1].
+* `docker://<ref>`: image in a registry implementing the "Docker Registry HTTP API V2"[^1].
+* `docker-archive://<path>`: image is stored in the `docker-save` formatted file[^1].
+* `docker-daemon://<ref>`: image stored in the docker daemon's internal storage[^1].
+* `oci://<path>`: image compliant with the "Open Container Image Layout Specification"[^1].
+* `oci-archive://<path>`: image compliant with the "Open Container Image Layout Specification" stored as a tar archive[^1].
+* `ostree://<ref>`: image in the local ostree repository[^1].
+
+!!! note
+    `docker://` is used by default if scheme unset.
+
+```console
+# registry image
+undock --rm-dist crazymax/buildx-pkg:latest ./dist
+# or
+undock --rm-dist docker://crazymax/buildx-pkg:latest ./dist
+
+# archive docker image
+docker pull crazymax/buildx-pkg:latest
+docker save crazymax/buildx-pkg:latest > archive.tar
+undock --rm-dist docker-archive://archive.tar ./dist
+
+# local docker image
+docker build -t myimage:local .
+undock --rm-dist docker-daemon://myimage:local ./dist
+```
+
 ## Environment variables
 
 Following environment variables can be used in place:
@@ -44,3 +76,5 @@ Following environment variables can be used in place:
 | `LOG_JSON`         | `false`       | Enable JSON logging output |
 | `LOG_CALLER`       | `false`       | Enable to add `file:line` of the caller |
 | `LOG_NOCOLOR`      | `false`       | Disable the colorized output |
+
+[^1]: See [containers image transport page](https://github.com/containers/image/blob/main/docs/containers-transports.5.md) for more info.

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/alecthomas/kong v0.4.0
 	github.com/containerd/containerd v1.5.9
 	github.com/containers/image/v5 v5.19.1
+	github.com/docker/docker v20.10.12+incompatible
 	github.com/mholt/archiver/v4 v4.0.0-alpha.4
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84
@@ -34,7 +35,6 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/docker v20.10.12+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect

internal/app/undock.go

@@ -2,18 +2,17 @@ package app
 
 import (
 	"context"
-	"fmt"
 	"os"
-	"path"
 	"path/filepath"
+	"strings"
 
 	"github.com/containerd/containerd/platforms"
-	"github.com/containers/image/v5/manifest"
 	"github.com/crazy-max/undock/internal/config"
+	"github.com/crazy-max/undock/internal/extractor"
+	ximage "github.com/crazy-max/undock/internal/extractor/image"
+	"github.com/crazy-max/undock/pkg/image"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
-	"github.com/rs/zerolog/log"
-	"golang.org/x/sync/errgroup"
 )
 
 // Undock represents an active undock object
@@ -70,76 +69,33 @@ func (c *Undock) Start() error {
 		return errors.Wrapf(err, "failed to create dist folder %q", c.cli.Dist)
 	}
 
-	logger := log.With().Str("src", c.cli.Source).Logger()
+	var (
+		xcli *extractor.Client
+		err  error
+	)
 
-	logger.Info().Msg("Extracting source image")
-	manblob, cachedir, err := c.cacheSource(logger, c.cli.Source)
-	if err != nil {
-		return errors.Wrap(err, "cannot cache source")
+	switch {
+	case isImageScheme(c.cli.Source):
+		xcli, err = ximage.New(c.ctx, c.meta, c.cli, c.platform)
+	default:
+		return errors.Errorf("unsupported source %q", c.cli.Source)
 	}
-
-	type manifestEntry struct {
-		platform specs.Platform
-		manifest *manifest.OCI1
+	if err != nil {
+		return err
 	}
 
-	var mans []manifestEntry
+	return xcli.Extract()
+}
 
-	mtype := manifest.GuessMIMEType(manblob)
-	if mtype == specs.MediaTypeImageManifest {
-		man, err := manifest.OCI1FromManifest(manblob)
-		if err != nil {
-			return errors.Wrap(err, "cannot create OCI manifest instance from blob")
-		}
-		mans = append(mans, manifestEntry{
-			platform: c.platform,
-			manifest: man,
-		})
-	} else if mtype == specs.MediaTypeImageIndex {
-		ocindex, err := manifest.OCI1IndexFromManifest(manblob)
-		if err != nil {
-			return errors.Wrap(err, "cannot create OCI manifest index instance from blob")
-		}
-		for _, m := range ocindex.Manifests {
-			mblob, err := os.ReadFile(path.Join(cachedir, "blobs", m.Digest.Algorithm().String(), m.Digest.Hex()))
-			if err != nil {
-				return errors.Wrapf(err, "cannot read OCI manifest JSON for platform %s", platforms.Format(*m.Platform))
-			}
-			man, err := manifest.OCI1FromManifest(mblob)
-			if err != nil {
-				return errors.Wrap(err, "cannot create OCI manifest instance from blob")
-			}
-			mans = append(mans, manifestEntry{
-				platform: *m.Platform,
-				manifest: man,
-			})
+func isImageScheme(source string) bool {
+	schemes := []string{"containers-storage", "docker", "docker-archive", "docker-daemon", "oci", "oci-archive", "ostree"}
+	for _, scheme := range schemes {
+		if strings.HasPrefix(source, scheme+"://") {
+			return true
 		}
 	}
-
-	eg, _ := errgroup.WithContext(c.ctx)
-	for _, mane := range mans {
-		func(mane manifestEntry) {
-			eg.Go(func() error {
-				dest := c.cli.Dist
-				if !c.cli.Wrap && len(mans) > 1 {
-					dest = path.Join(c.cli.Dist, fmt.Sprintf("%s_%s%s", mane.platform.OS, mane.platform.Architecture, mane.platform.Variant))
-				}
-				if err := os.MkdirAll(dest, 0700); err != nil {
-					return errors.Wrapf(err, "failed to create destination folder %q", dest)
-				}
-				for _, layer := range mane.manifest.LayerInfos() {
-					sublogger := logger.With().Str("platform", platforms.Format(mane.platform)).Str("blob", layer.Digest.String()).Logger()
-					sublogger.Info().Msgf("Extracting blob")
-					if err = c.extract(sublogger, path.Join(cachedir, "blobs", layer.Digest.Algorithm().String(), layer.Digest.Hex()), dest); err != nil {
-						return err
-					}
-				}
-				return nil
-			})
-		}(mane)
-	}
-
-	return eg.Wait()
+	_, err := image.Reference(source)
+	return err == nil
 }
 
 // Close closes undock

internal/config/cli.go

@@ -15,10 +15,10 @@ type Cli struct {
 
 	All      bool     `kong:"name=all,default=false,help='Extract all architectures if source is a manifest list.'"`
 	Includes []string `kong:"name=include,help='Include a subset of files/dirs from the source image.'"`
-	Insecure bool     `kong:"name=insecure,default=false,help='Allow contacting the registry over HTTP, or HTTPS with failed TLS verification.'"`
+	Insecure bool     `kong:"name=insecure,default=false,help='Allow contacting the registry or docker daemon over HTTP, or HTTPS with failed TLS verification.'"`
 	RmDist   bool     `kong:"name=rm-dist,default=false,help='Removes dist folder.'"`
 	Wrap     bool     `kong:"name=wrap,default=false,help='For a manifest list, merge output in dist folder.'"`
 
-	Source string `kong:"arg,required,name=source,help='Source image from a registry. (eg. alpine:latest)'"`
+	Source string `kong:"arg,required,name=source,help='Source image. (eg. alpine:latest)'"`
 	Dist   string `kong:"arg,required,name=dist,type=path,help='Dist folder. (eg. ./dist)'"`
 }

internal/app/extract.go renamed to internal/extractor/blob.go

@@ -1,4 +1,4 @@
-package app
+package extractor
 
 import (
 	"context"
@@ -14,7 +14,16 @@ import (
 	"github.com/rs/zerolog"
 )
 
-func (c *Undock) extract(logger zerolog.Logger, filename string, dest string) error {
+// ExtractBlobOpts holds extract blob options
+type ExtractBlobOpts struct {
+	Context  context.Context
+	Logger   zerolog.Logger
+	Includes []string
+}
+
+func ExtractBlob(filename string, dest string, opts ExtractBlobOpts) error {
+	opts.Logger.Info().Msgf("Extracting blob")
+
 	var r io.ReadCloser
 	dt, err := os.Open(filename)
 	if err != nil {
@@ -26,7 +35,7 @@ func (c *Undock) extract(logger zerolog.Logger, filename string, dest string) er
 	if err != nil {
 		return err
 	}
-	logger.Debug().Msgf("Blob format %s detected", format.Name())
+	opts.Logger.Debug().Msgf("Blob format %s detected", format.Name())
 
 	var u archiver.Extractor
 	var d archiver.Decompressor
@@ -56,23 +65,21 @@ func (c *Undock) extract(logger zerolog.Logger, filename string, dest string) er
 	}
 
 	var pathsInArchive []string
-	if len(c.cli.Includes) > 0 {
-		for _, inc := range c.cli.Includes {
-			inc = strings.TrimPrefix(inc, "/")
-			if len(inc) > 0 {
-				pathsInArchive = append(pathsInArchive, inc)
-			}
+	for _, inc := range opts.Includes {
+		inc = strings.TrimPrefix(inc, "/")
+		if len(inc) > 0 {
+			pathsInArchive = append(pathsInArchive, inc)
 		}
 	}
 	if len(pathsInArchive) == 0 {
 		pathsInArchive = nil
 	}
 
-	err = u.Extract(c.ctx, r, pathsInArchive, func(ctx context.Context, f archiver.File) error {
+	err = u.Extract(opts.Context, r, pathsInArchive, func(ctx context.Context, f archiver.File) error {
 		if f.FileInfo.IsDir() {
-			logger.Trace().Msgf("Extracting %s", f.NameInArchive)
+			opts.Logger.Trace().Msgf("Extracting %s", f.NameInArchive)
 		} else {
-			logger.Debug().Msgf("Extracting %s", f.NameInArchive)
+			opts.Logger.Debug().Msgf("Extracting %s", f.NameInArchive)
 		}
 
 		path := filepath.Join(dest, f.NameInArchive)