image: set up history array on exported image config

cache/manager.go

@@ -288,7 +288,7 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo)
 			createdAt:   getCreatedAt(cr.md),
 			usageCount:  usageCount,
 			lastUsedAt:  lastUsedAt,
-			description: getDescription(cr.md),
+			description: GetDescription(cr.md),
 		}
 		if cr.parent != nil {
 			c.parent = cr.parent.ID()

cache/metadata.go

@@ -111,7 +111,7 @@ func queueDescription(si *metadata.StorageItem, descr string) error {
 	return nil
 }
 
-func getDescription(si *metadata.StorageItem) string {
+func GetDescription(si *metadata.StorageItem) string {
 	v := si.Get(keyDescription)
 	if v == nil {
 		return ""

cache/refs.go

@@ -256,7 +256,7 @@ func (sr *mutableRef) commit(ctx context.Context) (ImmutableRef, error) {
 		md:           md,
 	}
 
-	if descr := getDescription(sr.md); descr != "" {
+	if descr := GetDescription(sr.md); descr != "" {
 		if err := queueDescription(md, descr); err != nil {
 			return nil, err
 		}

client/client_test.go

@@ -172,6 +172,7 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
 	}
 
 	run(`sh -c "mkdir -p foo/sub; echo -n first > foo/sub/bar; chmod 0741 foo;"`)
+	run(`true`) // this doesn't create a layer
 	run(`sh -c "echo -n second > foo/sub/baz"`)
 
 	def, err := st.Marshal()
@@ -268,6 +269,14 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
 		return false
 	})
 
+	require.Equal(t, 3, len(ociimg.History))
+	require.Contains(t, ociimg.History[0].CreatedBy, "foo/sub/bar")
+	require.Contains(t, ociimg.History[1].CreatedBy, "true")
+	require.Contains(t, ociimg.History[2].CreatedBy, "foo/sub/baz")
+	require.False(t, ociimg.History[0].EmptyLayer)
+	require.True(t, ociimg.History[1].EmptyLayer)
+	require.False(t, ociimg.History[2].EmptyLayer)
+
 	dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target().Digest)
 	require.NoError(t, err)
 

exporter/containerimage/export.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/containerd/containerd/content"
@@ -32,6 +33,8 @@ const (
 	keyPush             = "push"
 	keyInsecure         = "registry.insecure"
 	exporterImageConfig = "containerimage.config"
+
+	emptyGZLayer = digest.Digest("sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1")
 )
 
 type Opt struct {
@@ -62,7 +65,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 		case keyInsecure:
 			i.insecure = true
 		default:
-			logrus.Warnf("unknown exporter option %s", k)
+			logrus.Warnf("image exporter: unknown option %s", k)
 		}
 	}
 	return i, nil
@@ -83,44 +86,46 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
 	layersDone := oneOffProgress(ctx, "exporting layers")
 	diffPairs, err := blobs.GetDiffPairs(ctx, e.opt.ContentStore, e.opt.Snapshotter, e.opt.Differ, ref)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed calculaing diff pairs for exported snapshot")
 	}
 	layersDone(nil)
 
-	diffIDs := make([]digest.Digest, 0, len(diffPairs))
-	for _, dp := range diffPairs {
-		diffIDs = append(diffIDs, dp.DiffID)
-	}
-
-	var dt []byte
-	if config, ok := opt[exporterImageConfig]; ok {
-		dt, err = setDiffIDs(config, diffIDs)
+	config, ok := opt[exporterImageConfig]
+	if !ok {
+		config, err = emptyImageConfig()
 		if err != nil {
 			return err
 		}
-	} else {
-		dt, err = json.Marshal(imageConfig(diffIDs))
-		if err != nil {
-			return errors.Wrap(err, "failed to marshal image config")
-		}
+	}
+
+	history, err := parseHistoryFromConfig(config)
+	if err != nil {
+		return err
+	}
+
+	diffPairs, history = normalizeLayersAndHistory(diffPairs, history, ref)
+
+	config, err = patchImageConfig(config, diffPairs, history)
+	if err != nil {
+		return err
 	}
 
 	addAsRoot := content.WithLabels(map[string]string{
 		"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
 	})
 
-	dgst := digest.FromBytes(dt)
-	configDone := oneOffProgress(ctx, "exporting config "+dgst.String())
+	configDigest := digest.FromBytes(config)
+	configDone := oneOffProgress(ctx, "exporting config "+configDigest.String())
 
-	if err := content.WriteBlob(ctx, e.opt.ContentStore, dgst.String(), bytes.NewReader(dt), int64(len(dt)), dgst, addAsRoot); err != nil {
+	if err := content.WriteBlob(ctx, e.opt.ContentStore, configDigest.String(), bytes.NewReader(config), int64(len(config)), configDigest, addAsRoot); err != nil {
 		return configDone(errors.Wrap(err, "error writing config blob"))
 	}
 	configDone(nil)
 
 	mfst := schema2.Manifest{
 		Config: distribution.Descriptor{
-			Digest:    dgst,
-			Size:      int64(len(dt)),
+			Digest:    configDigest,
+			Size:      int64(len(config)),
 			MediaType: schema2.MediaTypeImageConfig,
 		},
 	}
@@ -130,7 +135,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
 	for _, dp := range diffPairs {
 		info, err := e.opt.ContentStore.Info(ctx, dp.Blobsum)
 		if err != nil {
-			return configDone(errors.Wrapf(err, "could not get blob %s", dp.Blobsum))
+			return configDone(errors.Wrapf(err, "could not find blob %s from contentstore", dp.Blobsum))
 		}
 		mfst.Layers = append(mfst.Layers, distribution.Descriptor{
 			Digest:    dp.Blobsum,
@@ -139,80 +144,166 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
 		})
 	}
 
-	dt, err = json.Marshal(mfst)
+	mfstJSON, err := json.Marshal(mfst)
 	if err != nil {
 		return errors.Wrap(err, "failed to marshal manifest")
 	}
 
-	dgst = digest.FromBytes(dt)
-	mfstDone := oneOffProgress(ctx, "exporting manifest "+dgst.String())
+	mfstDigest := digest.FromBytes(mfstJSON)
+	mfstDone := oneOffProgress(ctx, "exporting manifest "+mfstDigest.String())
 
-	if err := content.WriteBlob(ctx, e.opt.ContentStore, dgst.String(), bytes.NewReader(dt), int64(len(dt)), dgst, addAsRoot); err != nil {
-		return mfstDone(errors.Wrap(err, "error writing manifest blob"))
+	if err := content.WriteBlob(ctx, e.opt.ContentStore, mfstDigest.String(), bytes.NewReader(mfstJSON), int64(len(mfstJSON)), mfstDigest, addAsRoot); err != nil {
+		return mfstDone(errors.Wrapf(err, "error writing manifest blob %s", mfstDigest))
 	}
-
 	mfstDone(nil)
 
 	if e.targetName != "" {
 		if e.opt.Images != nil {
 			tagDone := oneOffProgress(ctx, "naming to "+e.targetName)
-			imgrec := images.Image{
+			img := images.Image{
 				Name: e.targetName,
 				Target: ocispec.Descriptor{
-					Digest:    dgst,
-					Size:      int64(len(dt)),
+					Digest:    mfstDigest,
+					Size:      int64(len(mfstJSON)),
 					MediaType: ocispec.MediaTypeImageManifest,
 				},
 				CreatedAt: time.Now(),
 			}
-			_, err := e.opt.Images.Update(ctx, imgrec)
-			if err != nil {
+
+			if _, err := e.opt.Images.Update(ctx, img); err != nil {
 				if !errdefs.IsNotFound(err) {
 					return tagDone(err)
 				}
 
-				_, err := e.opt.Images.Create(ctx, imgrec)
-				if err != nil {
+				if _, err := e.opt.Images.Create(ctx, img); err != nil {
 					return tagDone(err)
 				}
 			}
 			tagDone(nil)
 		}
 		if e.push {
-			return push.Push(ctx, e.opt.SessionManager, e.opt.ContentStore, dgst, e.targetName, e.insecure)
+			return push.Push(ctx, e.opt.SessionManager, e.opt.ContentStore, mfstDigest, e.targetName, e.insecure)
 		}
 	}
 
-	return err
+	return nil
 }
 
-// this is temporary: should move to dockerfile frontend
-func imageConfig(diffIDs []digest.Digest) ocispec.Image {
+func emptyImageConfig() ([]byte, error) {
 	img := ocispec.Image{
 		Architecture: runtime.GOARCH,
 		OS:           runtime.GOOS,
 	}
 	img.RootFS.Type = "layers"
-	img.RootFS.DiffIDs = diffIDs
 	img.Config.WorkingDir = "/"
 	img.Config.Env = []string{"PATH=" + system.DefaultPathEnv}
-	return img
+	dt, err := json.Marshal(img)
+	return dt, errors.Wrap(err, "failed to create empty image config")
+}
+
+func parseHistoryFromConfig(dt []byte) ([]ocispec.History, error) {
+	var config struct {
+		History []ocispec.History
+	}
+	if err := json.Unmarshal(dt, &config); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal history from config")
+	}
+	return config.History, nil
 }
 
-func setDiffIDs(config []byte, diffIDs []digest.Digest) ([]byte, error) {
-	mp := map[string]json.RawMessage{}
-	if err := json.Unmarshal(config, &mp); err != nil {
-		return nil, err
+func patchImageConfig(dt []byte, dps []blobs.DiffPair, history []ocispec.History) ([]byte, error) {
+	m := map[string]json.RawMessage{}
+	if err := json.Unmarshal(dt, &m); err != nil {
+		return nil, errors.Wrap(err, "failed to parse image config for patch")
 	}
+
 	var rootFS ocispec.RootFS
 	rootFS.Type = "layers"
-	rootFS.DiffIDs = diffIDs
+	for _, dp := range dps {
+		rootFS.DiffIDs = append(rootFS.DiffIDs, dp.DiffID)
+	}
 	dt, err := json.Marshal(rootFS)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed to marshal rootfs")
+	}
+	m["rootfs"] = dt
+
+	dt, err = json.Marshal(history)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to marshal history")
+	}
+	m["history"] = dt
+
+	dt, err = json.Marshal(m)
+	return dt, errors.Wrap(err, "failed to marshal config after patch")
+}
+
+func normalizeLayersAndHistory(diffs []blobs.DiffPair, history []ocispec.History, ref cache.ImmutableRef) ([]blobs.DiffPair, []ocispec.History) {
+	var historyLayers int
+	for _, h := range history {
+		if !h.EmptyLayer {
+			historyLayers += 1
+		}
+	}
+	if historyLayers > len(diffs) {
+		// this case shouldn't happen but if it does force set history layers empty
+		// from the bottom
+		logrus.Warn("invalid image config with unaccounted layers")
+		historyCopy := make([]ocispec.History, 0, len(history))
+		var l int
+		for _, h := range history {
+			if l >= len(diffs) {
+				h.EmptyLayer = true
+			}
+			if !h.EmptyLayer {
+				l++
+			}
+			historyCopy = append(historyCopy, h)
+		}
+		history = historyCopy
+	}
+
+	if len(diffs) > historyLayers {
+		// some history items are missing. add them based on the ref metadata
+		for _, msg := range getRefDesciptions(ref, len(diffs)-historyLayers) {
+			tm := time.Now().UTC()
+			history = append(history, ocispec.History{
+				Created:   &tm,
+				CreatedBy: msg,
+				Comment:   "buildkit.exporter.image.v0",
+			})
+		}
+	}
+
+	var layerIndex int
+	for i, h := range history {
+		if !h.EmptyLayer {
+			if diffs[layerIndex].Blobsum == emptyGZLayer {
+				h.EmptyLayer = true
+				diffs = append(diffs[:layerIndex], diffs[layerIndex+1:]...)
+			} else {
+				layerIndex++
+			}
+		}
+		history[i] = h
+	}
+
+	return diffs, history
+}
+
+func getRefDesciptions(ref cache.ImmutableRef, limit int) []string {
+	if limit <= 0 {
+		return nil
+	}
+	defaultMsg := "created by buildkit" // shouldn't happen but don't fail build
+	if ref == nil {
+		strings.Repeat(defaultMsg, limit)
+	}
+	descr := cache.GetDescription(ref.Metadata())
+	if descr == "" {
+		descr = defaultMsg
 	}
-	mp["rootfs"] = dt
-	return json.Marshal(mp)
+	return append(getRefDesciptions(ref.Parent(), limit-1), descr)
 }
 
 func oneOffProgress(ctx context.Context, id string) func(err error) error {