From 69a4a9b651c6659f4ef55634815618fa06c15054 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Tue, 30 Oct 2018 14:27:48 -0700 Subject: [PATCH] Move all image processing logic into utils, and expose publically --- cmd/analyze.go | 2 +- cmd/diff.go | 2 +- cmd/root.go | 173 +++------------------------------------- pkg/util/image_utils.go | 167 +++++++++++++++++++++++++++++++++++++- 4 files changed, 178 insertions(+), 166 deletions(-) diff --git a/cmd/analyze.go b/cmd/analyze.go index e8000968..0f673f97 100644 --- a/cmd/analyze.go +++ b/cmd/analyze.go @@ -59,7 +59,7 @@ func analyzeImage(imageName string, analyzerArgs []string) error { return errors.Wrap(err, "getting analyzers") } - image, err := getImageForName(imageName) + image, err := getImage(imageName) if err != nil { return errors.Wrapf(err, "error retrieving image %s", imageName) } diff --git a/cmd/diff.go b/cmd/diff.go index 8596c559..80020c65 100644 --- a/cmd/diff.go +++ b/cmd/diff.go @@ -73,7 +73,7 @@ func checkFilenameFlag(_ []string) error { // processImage is a concurrency-friendly wrapper around getImageForName func processImage(imageName string, imageMap map[string]*pkgutil.Image, wg *sync.WaitGroup, errChan chan<- error) { defer wg.Done() - image, err := getImageForName(imageName) + image, err := getImage(imageName) if err != nil { errChan <- fmt.Errorf("error retrieving image %s: %s", imageName, err) } diff --git a/cmd/root.go b/cmd/root.go index e471c94f..eef5f5b9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,26 +19,15 @@ package cmd import ( goflag "flag" "fmt" - "io/ioutil" - "net/http" "os" "path/filepath" "sort" "strings" - "time" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/daemon" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/GoogleContainerTools/container-diff/differs" pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util" "github.com/GoogleContainerTools/container-diff/util" - "github.com/google/go-containerregistry/pkg/v1" homedir "github.com/mitchellh/go-homedir" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -55,11 +44,6 @@ var format string type validatefxn func(args []string) error -const ( - DaemonPrefix = "daemon://" - RemotePrefix = "remote://" -) - var RootCmd = &cobra.Command{ Use: "container-diff", Short: "container-diff is a tool for analyzing and comparing container images", @@ -130,162 +114,27 @@ func checkIfValidAnalyzer(_ []string) error { return nil } -// getImageForName infers the source of an image and retrieves a v1.Image reference to it. -// Once a reference is obtained, it attempts to unpack the v1.Image's reader's contents -// into a temp directory on the local filesystem. -func getImageForName(imageName string) (pkgutil.Image, error) { - logrus.Infof("retrieving image: %s", imageName) - var img v1.Image - var err error - if pkgutil.IsTar(imageName) { - start := time.Now() - img, err = tarball.ImageFromPath(imageName, nil) - if err != nil { - return pkgutil.Image{}, errors.Wrap(err, "retrieving tar from path") - } - elapsed := time.Now().Sub(start) - logrus.Infof("retrieving image ref from tar took %f seconds", elapsed.Seconds()) - } else if strings.HasPrefix(imageName, DaemonPrefix) { - // remove the daemon prefix - imageName = strings.Replace(imageName, DaemonPrefix, "", -1) - - ref, err := name.ParseReference(imageName, name.WeakValidation) - if err != nil { - return pkgutil.Image{}, errors.Wrap(err, "parsing image reference") - } - - start := time.Now() - // TODO(nkubala): specify gzip.NoCompression here when functional options are supported - img, err = daemon.Image(ref, daemon.WithBufferedOpener()) - if err != nil { - return pkgutil.Image{}, errors.Wrap(err, "retrieving image from daemon") - } - elapsed := time.Now().Sub(start) - logrus.Infof("retrieving local image ref took %f seconds", elapsed.Seconds()) - } else { - // either has remote prefix or has no prefix, in which case we force remote - imageName = strings.Replace(imageName, RemotePrefix, "", -1) - ref, err := name.ParseReference(imageName, name.WeakValidation) - if err != nil { - return pkgutil.Image{}, errors.Wrap(err, "parsing image reference") - } - auth, err := authn.DefaultKeychain.Resolve(ref.Context().Registry) - if err != nil { - return pkgutil.Image{}, errors.Wrap(err, "resolving auth") - } - start := time.Now() - img, err = remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport)) - if err != nil { - return pkgutil.Image{}, errors.Wrap(err, "retrieving remote image") - } - elapsed := time.Now().Sub(start) - logrus.Infof("retrieving remote image ref took %f seconds", elapsed.Seconds()) - } - - // create tempdir and extract fs into it - var layers []pkgutil.Layer - if includeLayers() { - start := time.Now() - imgLayers, err := img.Layers() - if err != nil { - return pkgutil.Image{}, errors.Wrap(err, "getting image layers") - } - for _, layer := range imgLayers { - layerStart := time.Now() - digest, err := layer.Digest() - path, err := getExtractPathForName(digest.String()) - if err != nil { - return pkgutil.Image{ - Layers: layers, - }, errors.Wrap(err, "getting extract path for layer") - } - if err := pkgutil.GetFileSystemForLayer(layer, path, nil); err != nil { - return pkgutil.Image{ - Layers: layers, - }, errors.Wrap(err, "getting filesystem for layer") +func includeLayers() bool { + for _, t := range types { + for _, a := range differs.LayerAnalyzers { + if t == a { + return true } - layers = append(layers, pkgutil.Layer{ - FSPath: path, - Digest: digest, - }) - elapsed := time.Now().Sub(layerStart) - logrus.Infof("time elapsed retrieving layer: %fs", elapsed.Seconds()) } - elapsed := time.Now().Sub(start) - logrus.Infof("time elapsed retrieving image layers: %fs", elapsed.Seconds()) - } - - imageDigest, err := getImageDigest(img) - if err != nil { - return pkgutil.Image{}, err - } - path, err := getExtractPathForName(pkgutil.RemoveTag(imageName) + "@" + imageDigest.String()) - if err != nil { - return pkgutil.Image{}, err - } - // extract fs into provided dir - if err := pkgutil.GetFileSystemForImage(img, path, nil); err != nil { - return pkgutil.Image{ - FSPath: path, - Layers: layers, - }, errors.Wrap(err, "getting filesystem for image") } - return pkgutil.Image{ - Image: img, - Source: imageName, - FSPath: path, - Digest: imageDigest, - Layers: layers, - }, nil -} - -func getImageDigest(image v1.Image) (digest v1.Hash, err error) { - start := time.Now() - digest, err = image.Digest() - if err != nil { - return digest, err - } - elapsed := time.Now().Sub(start) - logrus.Infof("time elapsed retrieving image digest: %fs", elapsed.Seconds()) - return digest, nil + return false } -func getExtractPathForName(name string) (string, error) { - var path string +func getImage(imageName string) (pkgutil.Image, error) { + var cachePath string var err error if !noCache { - path, err = cacheDir(name) - if err != nil { - return "", err - } - // if cachedir doesn't exist, create it - if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { - err = os.MkdirAll(path, 0700) - if err != nil { - return "", err - } - logrus.Infof("caching filesystem at %s", path) - } - } else { - // otherwise, create tempdir - logrus.Infof("skipping caching") - path, err = ioutil.TempDir("", strings.Replace(name, "/", "", -1)) + cachePath, err = cacheDir(imageName) if err != nil { - return "", err - } - } - return path, nil -} - -func includeLayers() bool { - for _, t := range types { - for _, a := range differs.LayerAnalyzers { - if t == a { - return true - } + return pkgutil.Image{}, err } } - return false + return pkgutil.GetImage(imageName, includeLayers(), cachePath) } func cacheDir(imageName string) (string, error) { diff --git a/pkg/util/image_utils.go b/pkg/util/image_utils.go index 875d280b..c3b38676 100644 --- a/pkg/util/image_utils.go +++ b/pkg/util/image_utils.go @@ -21,19 +21,33 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "os" "path/filepath" "regexp" "sort" "strings" + "time" - "github.com/docker/docker/pkg/system" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + + "github.com/docker/docker/pkg/system" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -const tagRegexStr = ".*:([^/]+$)" +const ( + daemonPrefix = "daemon://" + remotePrefix = "remote://" + + tagRegexStr = ".*:([^/]+$)" +) type Layer struct { FSPath string @@ -52,6 +66,155 @@ type ImageHistoryItem struct { CreatedBy string `json:"created_by"` } +// GetImageForName retrieves an image by name alone. +// It does not return layer information, or respect caching. +func GetImageForName(imageName string) (Image, error) { + return GetImage(imageName, false, "") +} + +// GetImage infers the source of an image and retrieves a v1.Image reference to it. +// Once a reference is obtained, it attempts to unpack the v1.Image's reader's contents +// into a temp directory on the local filesystem. +func GetImage(imageName string, includeLayers bool, cacheDir string) (Image, error) { + logrus.Infof("retrieving image: %s", imageName) + var img v1.Image + var err error + if IsTar(imageName) { + start := time.Now() + img, err = tarball.ImageFromPath(imageName, nil) + if err != nil { + return Image{}, errors.Wrap(err, "retrieving tar from path") + } + elapsed := time.Now().Sub(start) + logrus.Infof("retrieving image ref from tar took %f seconds", elapsed.Seconds()) + } else if strings.HasPrefix(imageName, daemonPrefix) { + // remove the daemon prefix + imageName = strings.Replace(imageName, daemonPrefix, "", -1) + + ref, err := name.ParseReference(imageName, name.WeakValidation) + if err != nil { + return Image{}, errors.Wrap(err, "parsing image reference") + } + + start := time.Now() + // TODO(nkubala): specify gzip.NoCompression here when functional options are supported + img, err = daemon.Image(ref, daemon.WithBufferedOpener()) + if err != nil { + return Image{}, errors.Wrap(err, "retrieving image from daemon") + } + elapsed := time.Now().Sub(start) + logrus.Infof("retrieving local image ref took %f seconds", elapsed.Seconds()) + } else { + // either has remote prefix or has no prefix, in which case we force remote + imageName = strings.Replace(imageName, remotePrefix, "", -1) + ref, err := name.ParseReference(imageName, name.WeakValidation) + if err != nil { + return Image{}, errors.Wrap(err, "parsing image reference") + } + auth, err := authn.DefaultKeychain.Resolve(ref.Context().Registry) + if err != nil { + return Image{}, errors.Wrap(err, "resolving auth") + } + start := time.Now() + img, err = remote.Image(ref, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport)) + if err != nil { + return Image{}, errors.Wrap(err, "retrieving remote image") + } + elapsed := time.Now().Sub(start) + logrus.Infof("retrieving remote image ref took %f seconds", elapsed.Seconds()) + } + + // create tempdir and extract fs into it + var layers []Layer + if includeLayers { + start := time.Now() + imgLayers, err := img.Layers() + if err != nil { + return Image{}, errors.Wrap(err, "getting image layers") + } + for _, layer := range imgLayers { + layerStart := time.Now() + digest, err := layer.Digest() + path, err := getExtractPathForName(digest.String(), cacheDir) + if err != nil { + return Image{ + Layers: layers, + }, errors.Wrap(err, "getting extract path for layer") + } + if err := GetFileSystemForLayer(layer, path, nil); err != nil { + return Image{ + Layers: layers, + }, errors.Wrap(err, "getting filesystem for layer") + } + layers = append(layers, Layer{ + FSPath: path, + Digest: digest, + }) + elapsed := time.Now().Sub(layerStart) + logrus.Infof("time elapsed retrieving layer: %fs", elapsed.Seconds()) + } + elapsed := time.Now().Sub(start) + logrus.Infof("time elapsed retrieving image layers: %fs", elapsed.Seconds()) + } + + imageDigest, err := getImageDigest(img) + if err != nil { + return Image{}, err + } + path, err := getExtractPathForName(RemoveTag(imageName)+"@"+imageDigest.String(), cacheDir) + if err != nil { + return Image{}, err + } + // extract fs into provided dir + if err := GetFileSystemForImage(img, path, nil); err != nil { + return Image{ + FSPath: path, + Layers: layers, + }, errors.Wrap(err, "getting filesystem for image") + } + return Image{ + Image: img, + Source: imageName, + FSPath: path, + Digest: imageDigest, + Layers: layers, + }, nil +} + +func getExtractPathForName(name string, cacheDir string) (string, error) { + path := cacheDir + var err error + if cacheDir != "" { + // if cachedir doesn't exist, create it + if _, err := os.Stat(cacheDir); err != nil && os.IsNotExist(err) { + err = os.MkdirAll(cacheDir, 0700) + if err != nil { + return "", err + } + logrus.Infof("caching filesystem at %s", cacheDir) + } + } else { + // otherwise, create tempdir + logrus.Infof("skipping caching") + path, err = ioutil.TempDir("", strings.Replace(name, "/", "", -1)) + if err != nil { + return "", err + } + } + return path, nil +} + +func getImageDigest(image v1.Image) (digest v1.Hash, err error) { + start := time.Now() + digest, err = image.Digest() + if err != nil { + return digest, err + } + elapsed := time.Now().Sub(start) + logrus.Infof("time elapsed retrieving image digest: %fs", elapsed.Seconds()) + return digest, nil +} + func CleanupImage(image Image) { if image.FSPath != "" { logrus.Infof("Removing image filesystem directory %s from system", image.FSPath)