Add filesystem caching

Author: nkubala

Gopkg.lock

@@ -21,6 +21,7 @@ go_library(
         "//vendor/github.com/google/go-containerregistry/v1/daemon:go_default_library",
         "//vendor/github.com/google/go-containerregistry/v1/remote:go_default_library",
         "//vendor/github.com/google/go-containerregistry/v1/tarball:go_default_library",
+        "//vendor/github.com/mitchellh/go-homedir:go_default_library",
         "//vendor/github.com/sirupsen/logrus:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

cmd/BUILD.bazel

@@ -64,7 +64,7 @@ func analyzeImage(imageName string, analyzerArgs []string) error {
 		return err
 	}
 
-	if !save {
+	if noCache && !save {
 		defer pkgutil.CleanupImage(image)
 	}
 	if err != nil {
@@ -82,7 +82,7 @@ func analyzeImage(imageName string, analyzerArgs []string) error {
 	output.PrintToStdErr("Retrieving analyses\n")
 	outputResults(analyses)
 
-	if save {
+	if noCache && save {
 		logrus.Infof("Image was saved at %s", image.FSPath)
 	}
 

cmd/analyze.go

@@ -97,7 +97,7 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
 	}
 	wg.Wait()
 
-	if !save {
+	if noCache && !save {
 		defer pkgutil.CleanupImage(*imageMap[image1Arg])
 		defer pkgutil.CleanupImage(*imageMap[image2Arg])
 	}
@@ -121,7 +121,7 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
 		}
 	}
 
-	if save {
+	if noCache && save {
 		logrus.Infof("Images were saved at %s and %s", imageMap[image1Arg].FSPath,
 			imageMap[image2Arg].FSPath)
 	}

cmd/diff.go

@@ -22,6 +22,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -35,14 +36,17 @@ import (
 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
 	"github.com/GoogleContainerTools/container-diff/util"
 	"github.com/google/go-containerregistry/v1"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
 
 var json bool
+
 var save bool
 var types diffTypes
+var noCache bool
 
 var LogLevel string
 var format string
@@ -164,13 +168,29 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 			return pkgutil.Image{}, err
 		}
 	}
-	// TODO(nkubala): implement caching
 
-	// create tempdir and extract fs into it
-	path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
-	if err != nil {
-		return pkgutil.Image{}, err
+	var path string
+	if !noCache {
+		path, err = cacheDir(imageName)
+		if err != nil {
+			return pkgutil.Image{}, err
+		}
+		// if cachedir doesn't exist, create it
+		if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
+			err = os.MkdirAll(path, 0700)
+			if err != nil {
+				return pkgutil.Image{}, err
+			}
+			logrus.Infof("Image fs cached at %s", path)
+		}
+	} else {
+		// otherwise, create tempdir
+		path, err = ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
+		if err != nil {
+			return pkgutil.Image{}, err
+		}
 	}
+	// extract fs into provided dir
 	if err := pkgutil.GetFileSystemForImage(img, path, nil); err != nil {
 		return pkgutil.Image{
 			FSPath: path,
@@ -183,6 +203,16 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 	}, nil
 }
 
+func cacheDir(imageName string) (string, error) {
+	dir, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+	rootDir := filepath.Join(dir, ".container-diff", "cache")
+	imageName = strings.Replace(imageName, string(os.PathSeparator), "", -1)
+	return filepath.Join(rootDir, filepath.Clean(imageName)), nil
+}
+
 func init() {
 	RootCmd.PersistentFlags().StringVarP(&LogLevel, "verbosity", "v", "warning", "This flag controls the verbosity of container-diff.")
 	RootCmd.PersistentFlags().StringVarP(&format, "format", "", "", "Format to output diff in.")
@@ -220,4 +250,5 @@ func addSharedFlags(cmd *cobra.Command) {
 	cmd.Flags().VarP(&types, "type", "t", "This flag sets the list of analyzer types to use. Set it repeatedly to use multiple analyzers.")
 	cmd.Flags().BoolVarP(&save, "save", "s", false, "Set this flag to save rather than remove the final image filesystems on exit.")
 	cmd.Flags().BoolVarP(&util.SortSize, "order", "o", false, "Set this flag to sort any file/package results by descending size. Otherwise, they will be sorted by name.")
+	cmd.Flags().BoolVarP(&noCache, "no-cache", "n", false, "Set this to force retrieval of image filesystem on each run.")
 }

cmd/root.go

@@ -18,6 +18,7 @@ package util
 
 import (
 	"bytes"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -191,3 +192,18 @@ func HasFilepathPrefix(path, prefix string) bool {
 	}
 	return true
 }
+
+// given a path to a directory, check if it has any contents
+func DirIsEmpty(path string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	_, err = f.Readdir(1)
+	if err == io.EOF {
+		return true, nil
+	}
+	return false, err
+}

pkg/util/fs_utils.go

@@ -62,7 +62,16 @@ func SortMap(m map[string]struct{}) string {
 }
 
 // unpack image filesystem to local disk
+// if provided directory is not empty, do nothing
 func GetFileSystemForImage(image v1.Image, root string, whitelist []string) error {
+	empty, err := DirIsEmpty(root)
+	if err != nil {
+		return err
+	}
+	if !empty {
+		logrus.Infof("Using cached filesystem in %s", root)
+		return nil
+	}
 	if err := unpackTar(tar.NewReader(mutate.Extract(image)), root, whitelist); err != nil {
 		return err
 	}