build,cache: support pulling/pushing cache layers to/from remote sources

Following commit

* Initiates `cacheKey` or `layerKey` for intermediate images generated
  for layers.
* Allows end users to upload cached layers with `cacheKey` to remote
  sources using `--cache-to`. `--cache-to` is a optional flag to be used
with `buildah build` which publishes cached layers to remote sources.
* Allows end users to use cached layers from `remote` sources with
  `--cache-from`. `--cache-from` is a optional flag to be used with
`buildah build` and it pulls cached layers from remote sources in a step
by step manner only if is a valid cache hit.

Example
* Populate cache source or use cached layers if already present
```bash
buildah build -t test --layers --cache-to registry/myrepo/cache --cache-from registry/myrepo/cache .
```

Future:
* `cacheKey` or `layerKey` model is only being used when working with
  remote sources however local cache lookup can be also optimized if its
is altered to use `cacheKey` model instead of iterating through all the
images in local storage. As discussed here

References:
* Feature is quite similar to `kaniko`'s `--cache-repo`: https://github.com/GoogleContainerTools/kaniko#--cache-repo

Closes: issues#620

Signed-off-by: Aditya R <[email protected]>

Author: flouthoc

define/build.go

@@ -136,6 +136,12 @@ type BuildOptions struct {
 	RuntimeArgs []string
 	// TransientMounts is a list of mounts that won't be kept in the image.
 	TransientMounts []string
+	// CacheFrom specifies any remote repository which can be treated as
+	// potential cache source.
+	CacheFrom string
+	// CacheTo specifies any remote repository which can be treated as
+	// potential cache destination.
+	CacheTo string
 	// Compression specifies the type of compression which is applied to
 	// layer blobs.  The default is to not use compression, but
 	// archive.Gzip is recommended.

docs/buildah-build.1.md

@@ -96,7 +96,37 @@ The value of `[name]` is matched with the following priority order:
 
 **--cache-from**
 
-Images to utilise as potential cache sources. Buildah does not currently support --cache-from so this is a NOOP.
+Repository to utilise as potential cache source. When specified buildah will try to look for
+cache images in specified repository and will attempt to pull cache images instead of actually
+executing and computing STEPS. Buildah will only attempt to pull cache images if they are considered
+as valid cache hits otherwise not.
+
+Use `--cache-to` to populate a remote repository with cache content
+
+Example
+
+```bash
+# populate a cache and also consult it
+buildah build -t test --layers --cache-to registry/myrepo/cache --cache-from registry/myrepo/cache .
+```
+
+Note: option is ignored unless `--layers` is specified.
+
+**--cache-to**
+
+Set this flag to specify a remote repository that will be used to store cache images. Buildah will attempt to
+push newly built cache image to the remote repository.
+
+Note: Use `--cache-from` in order to use cache content in a remote repository.
+
+Example
+
+```bash
+# populate a cache and also consult it
+buildah build -t test --layers --cache-to registry/myrepo/cache --cache-from registry/myrepo/cache .
+```
+
+Note: option is ignored unless `--layers` is specified.
 
 **--cap-add**=*CAP\_xxx*
 

imagebuildah/executor.go

@@ -58,6 +58,8 @@ var builtinAllowedBuildArgs = map[string]bool{
 // interface.  It coordinates the entire build by using one or more
 // StageExecutors to handle each stage of the build.
 type Executor struct {
+	cacheFrom                      string
+	cacheTo                        string
 	containerSuffix                string
 	logger                         *logrus.Logger
 	stages                         map[string]*StageExecutor
@@ -212,6 +214,8 @@ func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, o
 	}
 
 	exec := Executor{
+		cacheFrom:                      options.CacheFrom,
+		cacheTo:                        options.CacheTo,
 		containerSuffix:                options.ContainerSuffix,
 		logger:                         logger,
 		stages:                         make(map[string]*StageExecutor),

imagebuildah/stage_executor.go

@@ -2,6 +2,7 @@ package imagebuildah
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"os"
@@ -26,6 +27,7 @@ import (
 	"github.com/containers/image/v5/manifest"
 	is "github.com/containers/image/v5/storage"
 	"github.com/containers/image/v5/transports"
+	"github.com/containers/image/v5/transports/alltransports"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/chrootarchive"
@@ -945,6 +947,18 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
 			s.log(commitMessage)
 		}
 	}
+	logCachePulled := func(src string) {
+		if !s.executor.quiet {
+			cacheHitMessage := "--> Cache pulled from remote"
+			fmt.Fprintf(s.executor.out, "%s %s\n", cacheHitMessage, src)
+		}
+	}
+	logCachePush := func(src string) {
+		if !s.executor.quiet {
+			cacheHitMessage := "--> Pushing cache"
+			fmt.Fprintf(s.executor.out, "%s %s\n", cacheHitMessage, src)
+		}
+	}
 	logCacheHit := func(cacheID string) {
 		if !s.executor.quiet {
 			cacheHitMessage := "--> Using cache"
@@ -1145,6 +1159,8 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
 		var (
 			commitName          string
 			cacheID             string
+			cacheKey            string
+			cachePulled         bool
 			err                 error
 			rebase              bool
 			addedContentSummary string
@@ -1156,6 +1172,15 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
 			commitName = s.output
 		}
 
+		// If --cache-from or --cache-to is specified make sure to populate
+		// cacheKey since it will be used either while pulling or pushing the
+		// cache images.
+		if s.executor.cacheFrom != "" || s.executor.cacheTo != "" {
+			cacheKey, err = s.generateCacheKey(ctx, node, addedContentSummary, s.stepRequiresLayer(step))
+			if err != nil {
+				return "", nil, fmt.Errorf("failed while generating cache key: %w", err)
+			}
+		}
 		// Check if there's already an image based on our parent that
 		// has the same change that we're about to make, so far as we
 		// can tell.
@@ -1168,6 +1193,24 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
 			if err != nil {
 				return "", nil, fmt.Errorf("error checking if cached image exists from a previous build: %w", err)
 			}
+			// All the best effort to find cache on localstorage have failed try pulling
+			// cache from remote repo if `--cache-from` was configured.
+			if cacheID == "" && s.executor.cacheFrom != "" {
+				src := fmt.Sprintf("%s:%s", s.executor.cacheFrom, cacheKey)
+				// only attempt to use cache again if pulling was successful
+				// otherwise do nothing and attempt to run the step, err != nil
+				// is ignored and will be automatically logged for --log-level debug
+				if id, err := s.pullCache(ctx, src); id != "" && err == nil {
+					logCachePulled(src)
+					cacheID, err = s.intermediateImageExists(ctx, node, addedContentSummary, s.stepRequiresLayer(step))
+					if err != nil {
+						return "", nil, fmt.Errorf("error checking if cached image exists from a previous build: %w", err)
+					}
+					if cacheID != "" {
+						cachePulled = true
+					}
+				}
+			}
 		}
 
 		// If we didn't find a cache entry, or we need to add content
@@ -1216,6 +1259,18 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
 			}
 		}
 
+		// Small wrapper around s.pushCache to prevent duplication of code
+		pushCache := func() error {
+			if s.executor.cacheTo != "" {
+				destSpec := fmt.Sprintf("%s:%s", s.executor.cacheTo, cacheKey)
+				logCachePush(destSpec)
+				if err = s.pushCache(ctx, imgID, destSpec); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+
 		// Note: If the build has squash, we must try to re-use as many layers as possible if cache is found.
 		// So only perform commit if its the lastInstruction of lastStage.
 		if cacheID != "" {
@@ -1231,6 +1286,14 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
 					return "", nil, err
 				}
 			}
+			if !cachePulled {
+				// Try to push this cache to remote repository only
+				// if cache was present on local storage and not
+				// pulled from remote source while processing this step.
+				if err = pushCache(); err != nil {
+					return "", nil, err
+				}
+			}
 		} else {
 			// We're not going to find any more cache hits, so we
 			// can stop looking for them.
@@ -1246,6 +1309,12 @@ func (s *StageExecutor) Execute(ctx context.Context, base string) (imgID string,
 			if err != nil {
 				return "", nil, fmt.Errorf("error committing container for step %+v: %w", *step, err)
 			}
+			// Following step is just built and was not used from cache so
+			// check if --cache-to was specified if yes then attempt pushing
+			// this cache to remote repo and fail accordingly
+			if err = pushCache(); err != nil {
+				return "", nil, err
+			}
 		}
 
 		// Create a squashed version of this image
@@ -1521,6 +1590,142 @@ func (s *StageExecutor) tagExistingImage(ctx context.Context, cacheID, output st
 	return img.ID, ref, nil
 }
 
+// generateCacheKey returns a computed digest for the current STEP
+// running its history and diff against a hash algorithm and this
+// generated CacheKey is further used by buildah to lock and decide
+// tag for the intermeidate image which can be pushed and pulled to/from
+// the remote repository.
+func (s *StageExecutor) generateCacheKey(ctx context.Context, currNode *parser.Node, addedContentDigest string, buildAddsLayer bool) (string, error) {
+	hash := sha256.New()
+	var baseHistory []v1.History
+	var diffIDs []digest.Digest
+	var manifestType string
+	var err error
+	if s.builder.FromImageID != "" {
+		manifestType, baseHistory, diffIDs, err = s.executor.getImageTypeAndHistoryAndDiffIDs(ctx, s.builder.FromImageID)
+		if err != nil {
+			return "", fmt.Errorf("error getting history of base image %q: %w", s.builder.FromImageID, err)
+		}
+		for i := 0; i < len(diffIDs); i++ {
+			fmt.Fprintln(hash, diffIDs[i].String())
+		}
+	}
+	createdBy := s.getCreatedBy(currNode, addedContentDigest)
+	fmt.Fprintf(hash, "%t", buildAddsLayer)
+	fmt.Fprintln(hash, createdBy)
+	fmt.Fprintln(hash, manifestType)
+	for _, element := range baseHistory {
+		fmt.Fprintln(hash, element.CreatedBy)
+		fmt.Fprintln(hash, element.Author)
+		fmt.Fprintln(hash, element.Comment)
+		fmt.Fprintln(hash, element.Created)
+		fmt.Fprintf(hash, "%t", element.EmptyLayer)
+		fmt.Fprintln(hash)
+	}
+	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+}
+
+// pushCache takes the image id of intermediate image and attempts
+// to perform push at the remote repository with cacheKey as the tag.
+// Returns error if fails otherwise returns nil.
+func (s *StageExecutor) pushCache(ctx context.Context, src, destSpec string) error {
+	logrus.Debugf("trying to push cache to dest: %+v from src:%+v", destSpec, src)
+	dest, err := alltransports.ParseImageName(destSpec)
+	// Add the docker:// transport to see if user neglected it while
+	// specifying --cache-to `<dest>`.
+	if err != nil {
+		destTransport := strings.Split(destSpec, ":")[0]
+		if t := transports.Get(destTransport); t != nil {
+			return err
+		}
+
+		if strings.Contains(destSpec, "://") {
+			return err
+		}
+
+		destSpec = "docker://" + destSpec
+		dest2, err2 := alltransports.ParseImageName(destSpec)
+		if err2 != nil {
+			return err
+		}
+		dest = dest2
+		logrus.Debugf("Assuming docker:// as the transport method for cache DESTINATION: %s", destSpec)
+	}
+	options := buildah.PushOptions{
+		Compression:         s.executor.compression,
+		SignaturePolicyPath: s.executor.signaturePolicyPath,
+		Store:               s.executor.store,
+		SystemContext:       s.executor.systemContext,
+		BlobDirectory:       s.executor.blobDirectory,
+		SignBy:              s.executor.signBy,
+		MaxRetries:          s.executor.maxPullPushRetries,
+		RetryDelay:          s.executor.retryPullPushDelay,
+	}
+	ref, digest, err := buildah.Push(ctx, src, dest, options)
+	if err != nil {
+		return fmt.Errorf("failed pushing cache to %q: %w", dest, err)
+	}
+	logrus.Debugf("successfully pushed cache to dest: %+v with ref:%+v and digest: %v", dest, ref, digest)
+	return nil
+}
+
+// pullCache takes the image source of the cache assuming tag
+// already points to the valid cacheKey and pulls the image to
+// local storage only if it was not already present on local storage
+// or a newer version of cache was found in the upstream repo. If new
+// image was pulled function returns image id otherwise returns empty
+// string "" or error if any error was encontered while pulling the cache.
+func (s *StageExecutor) pullCache(ctx context.Context, src string) (string, error) {
+	logrus.Debugf("trying to pull cache from remote repo: %+v", src)
+	options := buildah.PullOptions{
+		SignaturePolicyPath: s.executor.signaturePolicyPath,
+		Store:               s.executor.store,
+		SystemContext:       s.executor.systemContext,
+		BlobDirectory:       s.executor.blobDirectory,
+		MaxRetries:          s.executor.maxPullPushRetries,
+		RetryDelay:          s.executor.retryPullPushDelay,
+		AllTags:             false,
+		ReportWriter:        nil,
+		PullPolicy:          define.PullIfNewer,
+	}
+	// Get list of images in local storage before we pull image.
+	// We need this to ensure if we actually pulled any image or not
+	// and reason for this is explained below in the same function.
+	//
+	// NOTE, TODO: This logic can be removed once Pull API supports notifying
+	// back if image was actually pulled or not.
+	images, err := s.executor.store.Images()
+	if err != nil {
+		return "", fmt.Errorf("failed while getting images from local storage: %w", err)
+	}
+	id, err := buildah.Pull(ctx, src, options)
+	if err != nil {
+		logrus.Debugf("failed pulling cache from source %s: %v", src, err)
+		return "", fmt.Errorf("failed while pulling cache from %q: %w", src, err)
+	}
+	if id != "" {
+		// Reason for this logic:
+		//
+		// pullCache eventually returns pulled image ID back to the caller
+		// and here buildah will produce a log with image ID notifying to
+		// end-users if anything was actually pulled from remote repo in the
+		// the build output by calling `logCachePulled()`.
+		//
+		// Since following build output is part of stdout lets not return wrong
+		// info and further we are using build output to verify test-cases so
+		// don't return any id, since image was already present on local and buildah
+		// will eventually figure out to use valid cache source from local storage.
+		for _, image := range images {
+			if id == image.ID {
+				logrus.Debugf("cache was not pulled from:%s since id:%s is already present on local storage", src, id)
+				return "", nil
+			}
+		}
+	}
+	logrus.Debugf("successfully pulled cache from repo %s: %s", src, id)
+	return id, nil
+}
+
 // intermediateImageExists returns true if an intermediate image of currNode exists in the image store from a previous build.
 // It verifies this by checking the parent of the top layer of the image and the history.
 func (s *StageExecutor) intermediateImageExists(ctx context.Context, currNode *parser.Node, addedContentDigest string, buildAddsLayer bool) (string, error) {

pkg/cli/build.go

@@ -233,10 +233,6 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) (
 		return options, nil, nil, errors.New("'rm' and 'force-rm' can only be set with either 'layers' or 'no-cache'")
 	}
 
-	if c.Flag("cache-from").Changed {
-		logrus.Debugf("build --cache-from not enabled, has no effect")
-	}
-
 	if c.Flag("compress").Changed {
 		logrus.Debugf("--compress option specified but is ignored")
 	}
@@ -300,6 +296,8 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) (
 		Args:                    args,
 		BlobDirectory:           iopts.BlobCache,
 		BuildOutput:             iopts.BuildOutput,
+		CacheFrom:               iopts.CacheFrom,
+		CacheTo:                 iopts.CacheTo,
 		CNIConfigDir:            iopts.CNIConfigDir,
 		CNIPluginPath:           iopts.CNIPlugInPath,
 		CPPFlags:                iopts.CPPFlags,

pkg/cli/common.go

@@ -54,6 +54,7 @@ type BudResults struct {
 	BuildArg            []string
 	BuildContext        []string
 	CacheFrom           string
+	CacheTo             string
 	CertDir             string
 	Compress            bool
 	Creds               string
@@ -197,7 +198,8 @@ func GetBudFlags(flags *BudResults) pflag.FlagSet {
 	fs.StringArrayVar(&flags.OCIHooksDir, "hooks-dir", []string{}, "set the OCI hooks directory path (may be set multiple times)")
 	fs.StringArrayVar(&flags.BuildArg, "build-arg", []string{}, "`argument=value` to supply to the builder")
 	fs.StringArrayVar(&flags.BuildContext, "build-context", []string{}, "`argument=value` to supply additional build context to the builder")
-	fs.StringVar(&flags.CacheFrom, "cache-from", "", "images to utilise as potential cache sources. The build process does not currently support caching so this is a NOOP.")
+	fs.StringVar(&flags.CacheFrom, "cache-from", "", "remote repository to utilise as potential cache source.")
+	fs.StringVar(&flags.CacheTo, "cache-to", "", "remote repository to utilise as potential cache destination.")
 	fs.StringVar(&flags.CertDir, "cert-dir", "", "use certificates at the specified path to access the registry")
 	fs.BoolVar(&flags.Compress, "compress", false, "this is a legacy option, which has no effect on the image")
 	fs.StringArrayVar(&flags.CPPFlags, "cpp-flag", []string{}, "set additional flag to pass to C preprocessor (cpp)")
@@ -276,6 +278,7 @@ func GetBudFlagsCompletions() commonComp.FlagCompletions {
 	flagCompletion["build-arg"] = commonComp.AutocompleteNone
 	flagCompletion["build-context"] = commonComp.AutocompleteNone
 	flagCompletion["cache-from"] = commonComp.AutocompleteNone
+	flagCompletion["cache-to"] = commonComp.AutocompleteNone
 	flagCompletion["cert-dir"] = commonComp.AutocompleteDefault
 	flagCompletion["cpp-flag"] = commonComp.AutocompleteNone
 	flagCompletion["creds"] = commonComp.AutocompleteNone