Allow manually remove invalid snapshots on restore

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

script/demo/config.stargz.toml

@@ -5,4 +5,6 @@ ipfs = true
 host = "registry2:5000"
 insecure = true
 [directory_cache]
-direct = true
+direct = true
+[snapshotter]
+no_restore_invalid = true

service/config.go

@@ -32,6 +32,9 @@ type Config struct {
 
 	// ResolverConfig is config for resolving registries.
 	ResolverConfig `toml:"resolver"`
+
+	// SnapshotterConfig is snapshotter-related config.
+	SnapshotterConfig `toml:"snapshotter"`
 }
 
 // KubeconfigKeychainConfig is config for kubeconfig-based keychain.
@@ -55,3 +58,11 @@ type CRIKeychainConfig struct {
 
 // ResolverConfig is config for resolving registries.
 type ResolverConfig resolver.Config
+
+// SnapshotterConfig is snapshotter-related config.
+type SnapshotterConfig struct {
+	// NoRestoreInvalid doesn't restore invalid snapshots.
+	// NOTE: snapshotter.Mount will fail to check this so containerd cannot use this snapshot anymore.
+	//       User needs to manually remove snapshot metadata from containerd's metadata store using ctr (e.g. `ctr i rm`).
+	NoRestoreInvalid bool `toml:"no_restore_invalid"`
+}

service/service.go

@@ -93,7 +93,12 @@ func NewStargzSnapshotterService(ctx context.Context, root string, config *Confi
 
 	var snapshotter snapshots.Snapshotter
 
-	snapshotter, err = snbase.NewSnapshotter(ctx, snapshotterRoot(root), fs, snbase.AsynchronousRemove)
+	snOpts := []snbase.Opt{snbase.AsynchronousRemove}
+	if config.SnapshotterConfig.NoRestoreInvalid {
+		snOpts = append(snOpts, snbase.NoRestoreInvalid)
+	}
+
+	snapshotter, err = snbase.NewSnapshotter(ctx, snapshotterRoot(root), fs, snOpts...)
 	if err != nil {
 		log.G(ctx).WithError(err).Fatalf("failed to create new snapshotter")
 	}

snapshot/snapshot.go

@@ -71,8 +71,9 @@ type FileSystem interface {
 
 // SnapshotterConfig is used to configure the remote snapshotter instance
 type SnapshotterConfig struct {
-	asyncRemove bool
-	noRestore   bool
+	asyncRemove      bool
+	noRestore        bool
+	noRestoreInvalid bool
 }
 
 // Opt is an option to configure the remote snapshotter
@@ -92,15 +93,21 @@ func NoRestore(config *SnapshotterConfig) error {
 	return nil
 }
 
+func NoRestoreInvalid(config *SnapshotterConfig) error {
+	config.noRestoreInvalid = true
+	return nil
+}
+
 type snapshotter struct {
 	root        string
 	ms          *storage.MetaStore
 	asyncRemove bool
 
 	// fs is a filesystem that this snapshotter recognizes.
-	fs        FileSystem
-	userxattr bool // whether to enable "userxattr" mount option
-	noRestore bool
+	fs               FileSystem
+	userxattr        bool // whether to enable "userxattr" mount option
+	noRestore        bool
+	noRestoreInvalid bool
 }
 
 // NewSnapshotter returns a Snapshotter which can use unpacked remote layers
@@ -144,12 +151,13 @@ func NewSnapshotter(ctx context.Context, root string, targetFs FileSystem, opts
 	}
 
 	o := &snapshotter{
-		root:        root,
-		ms:          ms,
-		asyncRemove: config.asyncRemove,
-		fs:          targetFs,
-		userxattr:   userxattr,
-		noRestore:   config.noRestore,
+		root:             root,
+		ms:               ms,
+		asyncRemove:      config.asyncRemove,
+		fs:               targetFs,
+		userxattr:        userxattr,
+		noRestore:        config.noRestore,
+		noRestoreInvalid: config.noRestoreInvalid,
 	}
 
 	if err := o.restoreRemoteSnapshot(ctx); err != nil {
@@ -741,6 +749,13 @@ func (o *snapshotter) restoreRemoteSnapshot(ctx context.Context) error {
 	}
 	for _, info := range task {
 		if err := o.prepareRemoteSnapshot(ctx, info.Name, info.Labels); err != nil {
+			if o.noRestoreInvalid {
+				logrus.WithError(err).Warnf("failed to restore remote snapshot %s; remove this snapshot manually", info.Name)
+				// This snapshot is invalid. Do not restore it.
+				// NOTE: snapshotter.Mount will fail to check this so containerd cannot use this snapshot anymore.
+				//       User needs to manually remove snapshot metadata from containerd's metadata store using ctr.
+				continue
+			}
 			return fmt.Errorf("failed to prepare remote snapshot: %s: %w", info.Name, err)
 		}
 	}