RUN: only use an overlay for --mount=type=bind,rw on overlay

.cirrus.yml

@@ -294,7 +294,7 @@ integration_rootless_task:
         <<: *standardlogs
 
 in_podman_task:
-    name: "Containerized Integration"
+    name: "Containerized Integration w/ $NESTED_STORAGE_DRIVER over $STORAGE_DRIVER"
     alias: in_podman
     skip: *not_build_docs
     depends_on: *smoke_vendor
@@ -307,7 +307,16 @@ in_podman_task:
         # This is key, cause the scripts to re-execute themselves inside a container.
         IN_PODMAN: 'true'
         BUILDAH_ISOLATION: 'chroot'
-        STORAGE_DRIVER: 'vfs'
+    matrix:
+        - env:
+              STORAGE_DRIVER: 'vfs'
+        - env:
+              STORAGE_DRIVER: 'overlay'
+    matrix:
+        - env:
+              NESTED_STORAGE_DRIVER: 'vfs'
+        - env:
+              NESTED_STORAGE_DRIVER: 'overlay'
 
     # Separate scripts for separate outputs, makes debugging easier.
     setup_script: '${SCRIPT_BASE}/setup.sh |& ${_TIMESTAMP}'

contrib/cirrus/lib.sh

@@ -198,13 +198,13 @@ in_podman() {
                    --cgroupns=host \
                    "${envargs[@]}" \
                    -e BUILDAH_ISOLATION \
-                   -e STORAGE_DRIVER \
+                   -e STORAGE_DRIVER"${NESTED_STORAGE_DRIVER:+=${NESTED_STORAGE_DRIVER}}" \
                    -e "IN_PODMAN=false" \
                    -e "CONTAINER=podman" \
                    -e "CGROUP_MANAGER=cgroupfs" \
                    -v "$HOME/auth:$HOME/auth:ro" \
                    -v /sys/fs/cgroup:/sys/fs/cgroup:rw \
-                   -v /dev/fuse:/dev/fuse:rw \
+                   --device /dev/fuse \
                    -v "$GOSRC:$GOSRC:z" \
                    --workdir "$GOSRC" \
                    "$@"

imagebuildah/stage_executor.go

@@ -21,6 +21,7 @@ import (
 	"github.com/containers/buildah/internal"
 	"github.com/containers/buildah/internal/tmpdir"
 	internalUtil "github.com/containers/buildah/internal/util"
+	"github.com/containers/buildah/pkg/overlay"
 	"github.com/containers/buildah/pkg/parse"
 	"github.com/containers/buildah/pkg/rusage"
 	"github.com/containers/buildah/util"
@@ -313,12 +314,24 @@ func (s *StageExecutor) volumeCacheRestoreVFS() (err error) {
 // using it as a lower for an overlay mount in the same location, and then
 // discarding the upper.
 func (s *StageExecutor) volumeCacheSaveOverlay() (mounts []specs.Mount, err error) {
+	containerDir, err := s.executor.store.ContainerDirectory(s.builder.ContainerID)
+	if err != nil {
+		return nil, fmt.Errorf("unable to locate temporary directory for container")
+	}
+	volumeCacheDir := filepath.Join(containerDir, "volume-cache")
+	options := overlay.Options{
+		MountLabel: s.builder.MountLabel,
+		ForceMount: true,
+	}
 	for cachedPath := range s.volumeCache {
 		volumePath := filepath.Join(s.mountPoint, cachedPath)
-		mount := specs.Mount{
-			Source:      volumePath,
-			Destination: cachedPath,
-			Options:     []string{"O", "private"},
+		tmpdir, err := overlay.TempDir(volumeCacheDir, 0, 0)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create temporary directory for cache for volumes")
+		}
+		mount, err := overlay.MountWithOptions(tmpdir, volumePath, cachedPath, &options)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create temporary directory for cache for volumes")
 		}
 		mounts = append(mounts, mount)
 	}
@@ -327,7 +340,12 @@ func (s *StageExecutor) volumeCacheSaveOverlay() (mounts []specs.Mount, err erro
 
 // Reset the contents of each of the executor's list of volumes.
 func (s *StageExecutor) volumeCacheRestoreOverlay() error {
-	return nil
+	containerDir, err := s.executor.store.ContainerDirectory(s.builder.ContainerID)
+	if err != nil {
+		return fmt.Errorf("unable to locate temporary directory for container")
+	}
+	volumeCacheDir := filepath.Join(containerDir, "volume-cache")
+	return overlay.CleanupContent(volumeCacheDir)
 }
 
 // Save the contents of each of the executor's list of volumes for which we

internal/volumes/volumes.go

@@ -4,12 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/containers/buildah/copier"
 	"github.com/containers/buildah/define"
@@ -77,6 +79,11 @@ func mountIsReadWrite(m specs.Mount) bool {
 	return rw
 }
 
+// convertToOverlay accepts a specs.Mount that specifies a bind mount, and
+// returns a replacement for it that instead mounts an overlay at the desired
+// location, using the original source location as a "lower", so that writes to
+// it will be redirected to it to a temporary location which can later be
+// discarded, leaving the original location unmodified.
 func convertToOverlay(m specs.Mount, store storage.Store, mountLabel, tmpDir string, uid, gid int) (specs.Mount, string, error) {
 	overlayDir, err := overlay.TempDir(tmpDir, uid, gid)
 	if err != nil {
@@ -93,7 +100,7 @@ func convertToOverlay(m specs.Mount, store storage.Store, mountLabel, tmpDir str
 		// do the normal thing of mounting this directory as a lower with a temporary upper
 		mountThisInstead, err = overlay.MountWithOptions(overlayDir, m.Source, m.Destination, &options)
 		if err != nil {
-			return specs.Mount{}, "", fmt.Errorf("setting up overlay of %q: %w", m.Source, err)
+			return specs.Mount{}, "", fmt.Errorf("setting up overlay of directory %q: %w", m.Source, err)
 		}
 	} else {
 		// mount the parent directory as the lower with a temporary upper, and return a
@@ -103,7 +110,7 @@ func convertToOverlay(m specs.Mount, store storage.Store, mountLabel, tmpDir str
 		destination := m.Destination
 		mountedOverlay, err := overlay.MountWithOptions(overlayDir, sourceDir, destination, &options)
 		if err != nil {
-			return specs.Mount{}, "", fmt.Errorf("setting up overlay of %q: %w", sourceDir, err)
+			return specs.Mount{}, "", fmt.Errorf("setting up overlay of directory %q containing %q: %w", sourceDir, sourceBase, err)
 		}
 		if mountedOverlay.Type != define.TypeBind {
 			if err2 := overlay.RemoveTemp(overlayDir); err2 != nil {
@@ -118,6 +125,91 @@ func convertToOverlay(m specs.Mount, store storage.Store, mountLabel, tmpDir str
 	return mountThisInstead, overlayDir, nil
 }
 
+// makeTempCopy accepts a specs.Mount that specifies a bind mount, and returns
+// a replacement for it that instead bind mounts a temporary copy of the
+// original source location, so that writes to it will succeed without actually
+// modifying the original copy.
+func makeTempCopy(m specs.Mount, tmpDir string, uid, gid int) (specs.Mount, string, error) {
+	succeeded := false
+	overlayDir, err := overlay.TempDir(tmpDir, uid, gid)
+	if err != nil {
+		return specs.Mount{}, "", fmt.Errorf("setting up temporary copy of %q: %w", m.Destination, err)
+	}
+	defer func() {
+		if !succeeded {
+			if err := overlay.RemoveTemp(overlayDir); err != nil {
+				logrus.Warnf("%v", err)
+			}
+		}
+	}()
+	mergedDir := filepath.Join(overlayDir, "merge")
+	fileInfo, err := os.Stat(m.Source)
+	if err != nil {
+		return specs.Mount{}, "", fmt.Errorf("setting up temporary copy of %q: %w", m.Source, err)
+	}
+	copiedDir := filepath.Join(mergedDir, "copied")
+	if fileInfo.IsDir() {
+		if err = os.Mkdir(copiedDir, fileInfo.Mode()); err != nil {
+			return specs.Mount{}, "", fmt.Errorf("creating top-level directory for copy of %q: %w", m.Source, err)
+		}
+	} else {
+		if err = os.Mkdir(copiedDir, 0o755); err != nil {
+			return specs.Mount{}, "", fmt.Errorf("creating parent directory for copy of %q: %w", m.Source, err)
+		}
+	}
+	if err = os.Chown(copiedDir, uid, gid); err != nil {
+		return specs.Mount{}, "", fmt.Errorf("setting ownership of %q: %w", m.Source, err)
+	}
+	forceOwner := idtools.IDPair{UID: uid, GID: gid}
+	getOptions := copier.GetOptions{
+		ChownDirs:  &forceOwner,
+		ChownFiles: &forceOwner,
+	}
+	var wg sync.WaitGroup
+	var putErr, getErr error
+	copyReader, copyWriter := io.Pipe()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		putErr = copier.Put(copiedDir, copiedDir, copier.PutOptions{}, copyReader)
+		copyReader.Close()
+	}()
+	mountThisInstead := m
+	mountThisInstead.Options = append(slices.Clone(define.BindOptions), "z")
+	wg.Add(1)
+	if fileInfo.IsDir() {
+		// copy the directory to the temporary directory
+		go func() {
+			defer wg.Done()
+			getErr = copier.Get(m.Source, m.Source, getOptions, []string{"."}, copyWriter)
+			copyWriter.Close()
+		}()
+		// point the if-everything-works mount at the temporary driectory
+		mountThisInstead.Type = define.TypeBind
+		mountThisInstead.Source = copiedDir
+		mountThisInstead.Destination = m.Destination
+	} else {
+		// copy just the one thing to the temporary directory
+		sourceDir := filepath.Dir(m.Source)
+		sourceBase := filepath.Base(m.Source)
+		go func() {
+			defer wg.Done()
+			getErr = copier.Get(sourceDir, sourceDir, getOptions, []string{sourceBase}, copyWriter)
+			copyWriter.Close()
+		}()
+		// point the if-everything-works mount at the new copy of the item
+		mountThisInstead.Type = define.TypeBind
+		mountThisInstead.Source = filepath.Join(copiedDir, sourceBase)
+		mountThisInstead.Destination = m.Destination
+	}
+	wg.Wait()
+	if getErr != nil || putErr != nil {
+		return specs.Mount{}, "", fmt.Errorf("creating a temporary copy: %w", errors.Join(getErr, putErr))
+	}
+	succeeded = true
+	return mountThisInstead, overlayDir, nil
+}
+
 // FIXME: this code needs to be merged with pkg/parse/parse.go ValidateVolumeOpts
 //
 // GetBindMount parses a single bind mount entry from the --mount flag.
@@ -329,8 +421,15 @@ func GetBindMount(sys *types.SystemContext, args []string, contextDir string, st
 
 	overlayDir := ""
 	if mountedImage != "" || mountIsReadWrite(newMount) {
-		if newMount, overlayDir, err = convertToOverlay(newMount, store, mountLabel, tmpDir, 0, 0); err != nil {
-			return newMount, "", "", "", err
+		switch store.GraphDriverName() {
+		case "overlay":
+			if newMount, overlayDir, err = convertToOverlay(newMount, store, mountLabel, tmpDir, 0, 0); err != nil {
+				return newMount, "", "", "", err
+			}
+		default:
+			if newMount, overlayDir, err = makeTempCopy(newMount, tmpDir, 0, 0); err != nil {
+				return newMount, "", "", "", err
+			}
 		}
 	}
 
@@ -662,8 +761,15 @@ func GetCacheMount(sys *types.SystemContext, args []string, store storage.Store,
 
 	overlayDir := ""
 	if needToOverlay {
-		if newMount, overlayDir, err = convertToOverlay(newMount, store, mountLabel, tmpDir, 0, 0); err != nil {
-			return newMount, "", "", "", nil, err
+		switch store.GraphDriverName() {
+		case "overlay":
+			if newMount, overlayDir, err = convertToOverlay(newMount, store, mountLabel, tmpDir, 0, 0); err != nil {
+				return newMount, "", "", "", nil, err
+			}
+		default:
+			if newMount, overlayDir, err = makeTempCopy(newMount, tmpDir, 0, 0); err != nil {
+				return newMount, "", "", "", nil, err
+			}
 		}
 	}
 

pkg/overlay/overlay.go

@@ -57,8 +57,8 @@ type Options struct {
 
 // TempDir generates a uniquely-named directory under ${containerDir}/overlay
 // which can be used as a parent directory for the upper and working
-// directories for an overlay mount, creates "upper" and "work" directories
-// beneath it, and then returns the path of the new directory.
+// directories for an overlay mount, creates "upper", "work", and "merge"
+// directories beneath it, and then returns the path of the new directory.
 func TempDir(containerDir string, rootUID, rootGID int) (string, error) {
 	contentDir := filepath.Join(containerDir, "overlay")
 	if err := idtools.MkdirAllAs(contentDir, 0o700, rootUID, rootGID); err != nil {
@@ -126,12 +126,10 @@ func findMountProgram(graphOptions []string) string {
 	}
 
 	for _, i := range graphOptions {
-		s := strings.SplitN(i, "=", 2)
-		if len(s) != 2 {
+		key, val, ok := strings.Cut(i, "=")
+		if !ok {
 			continue
 		}
-		key := s[0]
-		val := s[1]
 		if _, has := mountMap[key]; has {
 			return val
 		}
@@ -146,7 +144,7 @@ func mountWithMountProgram(mountProgram, overlayOptions, mergeDir string) error
 	cmd := exec.Command(mountProgram, "-o", overlayOptions, mergeDir)
 
 	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("exec %s: %w", mountProgram, err)
+		return fmt.Errorf("exec %s -o %q %q: %w", mountProgram, overlayOptions, mergeDir, err)
 	}
 	return nil
 }

pkg/overlay/overlay_linux.go

@@ -1,15 +1,18 @@
 package overlay
 
 import (
+	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"syscall"
 
 	"github.com/containers/storage/pkg/unshare"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/selinux/go-selinux/label"
+	"golang.org/x/sys/unix"
 )
 
 // MountWithOptions creates ${contentDir}/merge, where ${contentDir} was
@@ -83,19 +86,39 @@ func MountWithOptions(contentDir, source, dest string, opts *Options) (mount spe
 		return mount, nil
 	}
 
+	// A mount_program is not specified: fallback to try mounting native overlay.
 	if unshare.IsRootless() {
-		// If a mount_program is not specified, fallback to try mounting native overlay.
 		overlayOptions = fmt.Sprintf("%s,userxattr", overlayOptions)
 	}
-
 	mount.Source = mergeDir
 	mount.Destination = dest
 	mount.Type = "overlay"
 	mount.Options = strings.Split(overlayOptions, ",")
 
 	if opts.ForceMount {
 		if err := mountNatively(overlayOptions, mergeDir); err != nil {
-			return mount, err
+			if opts.ReadOnly {
+				// We don't expect any kinds of error that we can respond to.
+				return mount, err
+			}
+			if errors.Is(err, syscall.EINVAL) {
+				// We couldn't do that with the kernel's built-in overlay
+				// filesystem; check if it was because the upper we made is
+				// already on an overlay filesystem, and if so, make a
+				// last desperate effort to find fuse-overlayfs, which doesn't
+				// mind us doing stuff like this.
+				upperDir := filepath.Join(contentDir, "upper")
+				var fs syscall.Statfs_t
+				if err2 := syscall.Statfs(upperDir, &fs); err2 == nil && fs.Type == unix.OVERLAYFS_SUPER_MAGIC {
+					if path, err3 := exec.LookPath("fuse-overlayfs"); err3 == nil {
+						mountProgram = path
+						err = mountWithMountProgram(mountProgram, overlayOptions, mergeDir)
+					}
+				}
+			}
+			if err != nil {
+				return mount, err
+			}
 		}
 
 		mount.Source = mergeDir

run_linux.go

@@ -1171,6 +1171,8 @@ func (b *Builder) runSetupVolumeMounts(mountLabel string, volumeMounts []string,
 				UpperDirOptionFragment: upperDir,
 				WorkDirOptionFragment:  workDir,
 				GraphOpts:              slices.Clone(b.store.GraphOptions()),
+				ForceMount:             true,
+				MountLabel:             b.MountLabel,
 			}
 
 			overlayMount, err := overlay.MountWithOptions(contentDir, host, container, &overlayOpts)

tests/bud/buildkit-mount/Containerfile5

@@ -3,3 +3,4 @@ ARG INPUTPATH_1=subdir
 RUN mkdir /test
 # use option z if selinux is enabled
 RUN --mount=type=bind,source=${INPUTPATH_1:?}/,target=/test,z cat /test/input_file
+RUN --mount=type=bind,source=${INPUTPATH_1:?}/,target=/test,z --mount=type=bind,source=${INPUTPATH_1:?}/,target=/test2,rw,z cmp /test/input_file /test2/input_file