Merge pull request #45339 from vvoland/c8d-prune-upstream-gc

c8d/prune: Remove gc.ref labels from configs of deleted images 

3 files changed, 118 insertions, 9 deletions

diff --git a/daemon/containerd/handlers.go b/daemon/containerd/handlers.go
index a129139e0f..b77b7b738b 100644
--- a/daemon/containerd/handlers.go
+++ b/daemon/containerd/handlers.go
@@ -9,6 +9,19 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+// walkPresentChildren is a simple wrapper for containerdimages.Walk with
+// presentChildrenHandler wrapping a simple handler that only operates on
+// walked Descriptor and doesn't return any errror.
+// This is only a convenient helper to reduce boilerplate.
+func (i *ImageService) walkPresentChildren(ctx context.Context, target ocispec.Descriptor, f func(context.Context, ocispec.Descriptor)) error {
+	store := i.client.ContentStore()
+	return containerdimages.Walk(ctx, presentChildrenHandler(store, containerdimages.HandlerFunc(
+		func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+			f(ctx, desc)
+			return nil, nil
+		})), target)
+}
+
 // presentChildrenHandler is a handler wrapper which traverses all children
 // descriptors that are present in the store and calls specified handler.
 func presentChildrenHandler(store content.Store, h containerdimages.HandlerFunc) containerdimages.HandlerFunc {
diff --git a/daemon/containerd/image_delete.go b/daemon/containerd/image_delete.go
index d3c252d591..1fd13cf420 100644
--- a/daemon/containerd/image_delete.go
+++ b/daemon/containerd/image_delete.go
@@ -6,6 +6,9 @@ import (
 	"github.com/containerd/containerd/images"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/sirupsen/logrus"
 )
 
 // ImageDelete deletes the image referenced by the given imageRef from this
@@ -56,11 +59,25 @@ func (i *ImageService) ImageDelete(ctx context.Context, imageRef string, force,
 		return nil, err
 	}
 
+	possiblyDeletedConfigs := map[digest.Digest]struct{}{}
+	if err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, d ocispec.Descriptor) {
+		if images.IsConfigType(d.MediaType) {
+			possiblyDeletedConfigs[d.Digest] = struct{}{}
+		}
+	}); err != nil {
+		return nil, err
+	}
+
 	err = i.client.ImageService().Delete(ctx, img.Name, images.SynchronousDelete())
 	if err != nil {
 		return nil, err
 	}
 
+	// Workaround for: https://github.com/moby/buildkit/issues/3797
+	if err := i.unleaseSnapshotsFromDeletedConfigs(context.Background(), possiblyDeletedConfigs); err != nil {
+		logrus.WithError(err).Warn("failed to unlease snapshots")
+	}
+
 	imgID := string(img.Target.Digest)
 	i.LogImageEvent(imgID, imgID, "untag")
 	i.LogImageEvent(imgID, imgID, "delete")
diff --git a/daemon/containerd/image_prune.go b/daemon/containerd/image_prune.go
index e333e4c6bf..6da0d2c013 100644
--- a/daemon/containerd/image_prune.go
+++ b/daemon/containerd/image_prune.go
@@ -10,6 +10,7 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/errdefs"
 	"github.com/hashicorp/go-multierror"
+	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -78,7 +79,11 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu
 	// Apply filters
 	for name, img := range imagesToPrune {
 		filteredOut := !filterFunc(img)
-		logrus.WithField("image", name).WithField("filteredOut", filteredOut).Debug("filtering image")
+		logrus.WithFields(logrus.Fields{
+			"image":       name,
+			"filteredOut": filteredOut,
+		}).Debug("filtering image")
+
 		if filteredOut {
 			delete(imagesToPrune, name)
 		}
@@ -106,25 +111,39 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu
 		}
 	}
 
-	logrus.WithField("images", imagesToPrune).Debug("pruning")
+	possiblyDeletedConfigs := map[digest.Digest]struct{}{}
+
+	// Workaround for https://github.com/moby/buildkit/issues/3797
+	defer func() {
+		if err := i.unleaseSnapshotsFromDeletedConfigs(context.Background(), possiblyDeletedConfigs); err != nil {
+			errs = multierror.Append(errs, err)
+		}
+	}()
 
 	for _, img := range imagesToPrune {
-		blobs := []ocispec.Descriptor{}
+		logrus.WithField("image", img).Debug("pruning image")
 
-		err = containerdimages.Walk(ctx, presentChildrenHandler(store, containerdimages.HandlerFunc(
-			func(_ context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
-				blobs = append(blobs, desc)
-				return nil, nil
-			})),
-			img.Target)
+		blobs := []ocispec.Descriptor{}
 
+		err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) {
+			blobs = append(blobs, desc)
+			if containerdimages.IsConfigType(desc.MediaType) {
+				possiblyDeletedConfigs[desc.Digest] = struct{}{}
+			}
+		})
 		if err != nil {
 			errs = multierror.Append(errs, err)
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return &report, errs
+			}
 			continue
 		}
 		err = is.Delete(ctx, img.Name, containerdimages.SynchronousDelete())
 		if err != nil && !cerrdefs.IsNotFound(err) {
 			errs = multierror.Append(errs, err)
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return &report, errs
+			}
 			continue
 		}
 
@@ -148,5 +167,65 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu
 			}
 		}
 	}
+
 	return &report, errs
 }
+
+// unleaseSnapshotsFromDeletedConfigs removes gc.ref.snapshot content label from configs that are not
+// referenced by any of the existing images.
+// This is a temporary solution to the rootfs snapshot not being deleted when there's a buildkit history
+// item referencing an image config.
+func (i *ImageService) unleaseSnapshotsFromDeletedConfigs(ctx context.Context, possiblyDeletedConfigs map[digest.Digest]struct{}) error {
+	is := i.client.ImageService()
+	store := i.client.ContentStore()
+
+	all, err := is.List(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to list images during snapshot lease removal")
+	}
+
+	var errs error
+	for _, img := range all {
+		err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) {
+			if containerdimages.IsConfigType(desc.MediaType) {
+				delete(possiblyDeletedConfigs, desc.Digest)
+			}
+		})
+		if err != nil {
+			errs = multierror.Append(errs, err)
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return errs
+			}
+			continue
+		}
+	}
+
+	// At this point, all configs that are used by any image has been removed from the slice
+	for cfgDigest := range possiblyDeletedConfigs {
+		info, err := store.Info(ctx, cfgDigest)
+		if err != nil {
+			if cerrdefs.IsNotFound(err) {
+				logrus.WithField("config", cfgDigest).Debug("config already gone")
+			} else {
+				errs = multierror.Append(errs, err)
+				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+					return errs
+				}
+			}
+			continue
+		}
+
+		label := "containerd.io/gc.ref.snapshot." + i.StorageDriver()
+
+		delete(info.Labels, label)
+		_, err = store.Update(ctx, info, "labels."+label)
+		if err != nil {
+			errs = multierror.Append(errs, errors.Wrapf(err, "failed to remove gc.ref.snapshot label from %s", cfgDigest))
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return errs
+			}
+		}
+	}
+
+	return errs
+}