feature: add label ai.tensorchord.envd.build.manifestBytecodeHash to …

…image for cache robust (#661)

* WIP: feature support robus tag and label for cache

Signed-off-by: Qi Chen <aseaday@hotmail.com>

* WIP: feature support robus tag and label for cache

Signed-off-by: Qi Chen <aseaday@hotmail.com>

* WIP: feature support robus tag and label for cache

Signed-off-by: Qi Chen <aseaday@hotmail.com>

* WIP: feature support robus tag and label for cache

Signed-off-by: Qi Chen <aseaday@hotmail.com>

* WIP: feature support robus tag and label for cache

Signed-off-by: Qi Chen <aseaday@hotmail.com>

* WIP: fix lint error

Signed-off-by: Qi Chen <aseaday@hotmail.com>

* Modify the label name

Signed-off-by: Qi Chen <aseaday@hotmail.com>
Co-authored-by: Keming <kemingyang@tensorchord.ai>

* Fix the return type of envdProgramHash

Signed-off-by: Qi Chen <aseaday@hotmail.com>
Co-authored-by: Keming <kemingyang@tensorchord.ai>

* Fix ci

Signed-off-by: Qi Chen <aseaday@hotmail.com>

Co-authored-by: Keming <kemingyang@tensorchord.ai>

e2e/bytecode_hash_test.go

@@ -0,0 +1,59 @@
+// Copyright 2022 The envd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"context"
+	"os"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func appendSomeToFile(path string) {
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		panic(err)
+	}
+	blank := "\n\n"
+	_, err = f.Write([]byte(blank))
+	if err != nil {
+		panic(err)
+	}
+	if err := f.Close(); err != nil {
+		panic(err)
+	}
+}
+
+var _ = Describe("bytecode hash cache target", func() {
+	exampleName := "quick-start"
+	It("add some blank to build.envd", func() {
+		testcase := "add-blank"
+		e := NewExample(exampleName, testcase)
+		ctx := context.TODO()
+		e.BuildImage(false)()
+		dockerClient := GetDockerClient(ctx)
+		imageSum, err := dockerClient.GetImage(ctx, e.Tag)
+		Expect(err).NotTo(HaveOccurred())
+		oldCreated := imageSum.Created
+		appendSomeToFile("testdata/" + exampleName + "/build.envd")
+		e.BuildImage(false)()
+		imageSum, err = dockerClient.GetImage(ctx, e.Tag)
+		Expect(err).NotTo(HaveOccurred())
+		newCreated := imageSum.Created
+		Expect(oldCreated).To(Equal(newCreated))
+		e.RemoveImage()()
+	})
+})

e2e/e2e_helper.go

@@ -26,107 +26,98 @@ import (
 	sshconfig "github.com/tensorchord/envd/pkg/ssh/config"
 )
 
-func BuildImage(exampleName string) func() {
+func (e *Example) BuildImage(force bool) func() {
 	return func() {
 		logrus.Info("building quick-start image")
-		err := BuildExampleImage(exampleName, app.New())
+		buildContext := "testdata/" + e.Name
+		args := []string{
+			"envd.test", "--debug", "build", "--path", buildContext, "--tag", e.Tag,
+		}
+		if force {
+			args = append(args, "--force")
+		}
+		err := e.app.Run(args)
 		if err != nil {
 			panic(err)
 		}
 	}
 }
 
-func RemoveImage(exampleName string) func() {
+func (e *Example) RemoveImage() func() {
 	return func() {
-		err := RemoveExampleImage(exampleName)
+		ctx := context.TODO()
+		dockerClient, err := docker.NewClient(ctx)
 		if err != nil {
 			panic(err)
 		}
-	}
-}
-
-func RunContainer(exampleName string) func() {
-	return func() {
-		err := RunExampleContainer(exampleName, app.New())
+		err = dockerClient.RemoveImage(ctx, e.Tag)
 		if err != nil {
 			panic(err)
 		}
 	}
 }
 
-func DestoryContainer(exampleName string) func() {
-	return func() {
-		err := DestroyExampleContainer(exampleName, app.New())
-		if err != nil {
-			panic(err)
-		}
+func GetDockerClient(ctx context.Context) docker.Client {
+	dockerClient, err := docker.NewClient(ctx)
+	if err != nil {
+		panic(err)
 	}
+	return dockerClient
 }
 
 type Example struct {
 	Name string
+	Tag  string
+	app  app.EnvdApp
 }
 
-func example(name string) *Example {
+func NewExample(name string, testcaseAbbr string) *Example {
+	tag := name + ":" + testcaseAbbr
 	return &Example{
 		Name: name,
+		Tag:  tag,
+		app:  app.New(),
 	}
 }
 
 func (e *Example) Exec(cmd string) string {
-	sshClient := getSSHClient(e.Name)
+	sshClient := e.getSSHClient()
 	ret, err := sshClient.ExecWithOutput(cmd)
 	if err != nil {
 		panic(err)
 	}
 	return strings.Trim(string(ret), "\n")
 }
 
-func BuildExampleImage(exampleName string, app app.EnvdApp) error {
-	buildContext := "testdata/" + exampleName
-	tag := exampleName + ":e2etest"
-	args := []string{
-		"envd.test", "--debug", "build", "--path", buildContext, "--tag", tag, "--force",
-	}
-	err := app.Run(args)
-	return err
-}
-
-func RemoveExampleImage(exampleName string) error {
-	ctx := context.TODO()
-	dockerClient, err := docker.NewClient(ctx)
-	if err != nil {
-		return err
-	}
-	err = dockerClient.RemoveImage(ctx, exampleName+":e2etest")
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func RunExampleContainer(exampleName string, app app.EnvdApp) error {
-	buildContext := "testdata/" + exampleName
-	tag := exampleName + ":e2etest"
-	args := []string{
-		"envd.test", "--debug", "up", "--path", buildContext, "--tag", tag, "--detach", "--force",
+func (e *Example) RunContainer() func() {
+	return func() {
+		buildContext := "testdata/" + e.Name
+		args := []string{
+			"envd.test", "--debug", "up", "--path", buildContext, "--tag", e.Tag, "--detach", "--force",
+		}
+		err := e.app.Run(args)
+		if err != nil {
+			panic(err)
+		}
 	}
-	err := app.Run(args)
-	return err
 }
 
-func DestroyExampleContainer(exampleName string, app app.EnvdApp) error {
-	buildContext := "testdata/" + exampleName
-	args := []string{
-		"envd.test", "--debug", "destroy", "--path", buildContext,
+func (e *Example) DestroyContainer() func() {
+	return func() {
+		buildContext := "testdata/" + e.Name
+		args := []string{
+			"envd.test", "--debug", "destroy", "--path", buildContext,
+		}
+		err := e.app.Run(args)
+		if err != nil {
+			panic(err)
+		}
 	}
-	err := app.Run(args)
-	return err
 }
 
-func getSSHClient(exampleName string) ssh.Client {
+func (e *Example) getSSHClient() ssh.Client {
 	localhost := "127.0.0.1"
-	port, err := sshconfig.GetPort(exampleName)
+	port, err := sshconfig.GetPort(e.Name)
 	if err != nil {
 		panic(err)
 	}

e2e/quick_start_test.go

@@ -21,11 +21,13 @@ import (
 
 var _ = Describe("e2e quickstart", Ordered, func() {
 	exampleName := "quick-start"
-	BeforeAll(BuildImage(exampleName))
-	BeforeEach(RunContainer(exampleName))
+	testcase := "e2e"
+	e := NewExample(exampleName, testcase)
+	BeforeAll(e.BuildImage(true))
+	BeforeEach(e.RunContainer())
 	It("execute python demo.py", func() {
-		Expect(example(exampleName).Exec("python demo.py")).To(Equal("[2 3 4]"))
+		Expect(e.Exec("python demo.py")).To(Equal("[2 3 4]"))
 	})
-	AfterEach(DestoryContainer(exampleName))
-	AfterAll(RemoveImage(exampleName))
+	AfterEach(e.DestroyContainer())
+	AfterAll(e.RemoveImage())
 })

pkg/builder/builder.go

@@ -71,7 +71,8 @@ type Options struct {
 
 type generalBuilder struct {
 	Options
-	entries []client.ExportEntry
+	manifestCodeHash string
+	entries          []client.ExportEntry
 
 	logger *logrus.Entry
 	starlark.Interpreter
@@ -96,9 +97,15 @@ func New(ctx context.Context, opt Options) (Builder, error) {
 		return nil, errors.New("only one output type is supported")
 	}
 
+	manifestHash, err := starlark.GetEnvdProgramHash(opt.ManifestFilePath)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to compile manifest file")
+	}
+
 	b := &generalBuilder{
-		Options: opt,
-		entries: entries,
+		Options:          opt,
+		manifestCodeHash: manifestHash,
+		entries:          entries,
 		logger: logrus.WithFields(logrus.Fields{
 			"tag": opt.Tag,
 		}),
@@ -167,11 +174,17 @@ func (b generalBuilder) compile(ctx context.Context) (*llb.Definition, error) {
 	return def, nil
 }
 
+func (b generalBuilder) addBuilderTag(labels *map[string]string) {
+	(*labels)[types.ImageLabelCacheHash] = b.manifestCodeHash
+}
+
 func (b generalBuilder) imageConfig(ctx context.Context) (string, error) {
 	labels, err := ir.Labels()
 	if err != nil {
 		return "", errors.Wrap(err, "failed to get labels")
 	}
+	b.addBuilderTag(&labels)
+
 	ports, err := ir.ExposedPorts()
 	if err != nil {
 		return "", errors.Wrap(err, "failed to get expose ports")
@@ -303,9 +316,8 @@ func (b generalBuilder) checkIfNeedBuild(ctx context.Context) bool {
 	depsFiles := []string{
 		b.PubKeyPath,
 		b.ConfigFilePath,
-		b.ManifestFilePath,
 	}
-	isUpdated, err := b.checkDepsFileUpdate(ctx, b.Tag, depsFiles)
+	isUpdated, err := b.checkDepsFileUpdate(ctx, b.Tag, b.ManifestFilePath, depsFiles)
 	if err != nil {
 		b.logger.Debugf("failed to check manifest update: %s", err)
 	}
@@ -317,12 +329,13 @@ func (b generalBuilder) checkIfNeedBuild(ctx context.Context) bool {
 }
 
 // Always return updated when met error
-func (b generalBuilder) checkDepsFileUpdate(ctx context.Context, tag string, deps []string) (bool, error) {
+func (b generalBuilder) checkDepsFileUpdate(ctx context.Context, tag string, manifest string, deps []string) (bool, error) {
 	dockerClient, err := docker.NewClient(ctx)
 	if err != nil {
 		return true, err
 	}
-	image, err := dockerClient.GetImage(ctx, tag)
+
+	image, err := dockerClient.GetImageWithCacheHashLabel(ctx, tag, b.manifestCodeHash)
 	if err != nil {
 		return true, err
 	}

pkg/docker/docker.go

@@ -75,6 +75,7 @@ type Client interface {
 
 	ListImage(ctx context.Context) ([]types.ImageSummary, error)
 	GetImage(ctx context.Context, image string) (types.ImageSummary, error)
+	GetImageWithCacheHashLabel(ctx context.Context, image string, hash string) (types.ImageSummary, error)
 	RemoveImage(ctx context.Context, image string) error
 
 	GetInfo(ctx context.Context) (types.Info, error)
@@ -204,6 +205,19 @@ func (c generalClient) GetImage(ctx context.Context, image string) (types.ImageS
 	return images[0], nil
 }
 
+func (c generalClient) GetImageWithCacheHashLabel(ctx context.Context, image string, hash string) (types.ImageSummary, error) {
+	images, err := c.ImageList(ctx, types.ImageListOptions{
+		Filters: dockerFiltersWithCacheLabel(image, hash),
+	})
+	if err != nil {
+		return types.ImageSummary{}, err
+	}
+	if len(images) == 0 {
+		return types.ImageSummary{}, errors.Errorf("image with hash %s not found", hash)
+	}
+	return images[0], nil
+}
+
 func (c generalClient) ListContainer(ctx context.Context) ([]types.Container, error) {
 	return c.ContainerList(ctx, types.ContainerListOptions{
 		Filters: dockerFilters(false),

pkg/docker/label.go

@@ -55,3 +55,10 @@ func dockerFiltersWithName(name string) filters.Args {
 	f.Add("reference", name)
 	return f
 }
+
+func dockerFiltersWithCacheLabel(name string, hash string) filters.Args {
+	f := filters.NewArgs()
+	f.Add("reference", name)
+	f.Add("label", fmt.Sprintf("%s=%s", types.ImageLabelCacheHash, hash))
+	return f
+}

pkg/lang/frontend/starlark/interpreter.go

@@ -15,6 +15,11 @@
 package starlark
 
 import (
+	"bytes"
+	"hash/fnv"
+	"io/ioutil"
+	"strconv"
+
 	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
 	"go.starlark.net/repl"
@@ -53,6 +58,30 @@ func NewInterpreter(buildContextDir string) Interpreter {
 	}
 }
 
+func GetEnvdProgramHash(filename string) (string, error) {
+	envdSrc, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return "", err
+	}
+	// No Check builtin or predeclared for now
+	funcAlwaysHas := func(x string) bool {
+		return true
+	}
+	_, prog, err := starlark.SourceProgram(filename, envdSrc, funcAlwaysHas)
+	if err != nil {
+		return "", err
+	}
+	buf := new(bytes.Buffer)
+	err = prog.Write(buf)
+	if err != nil {
+		return "", err
+	}
+	h := fnv.New64a()
+	h.Write(buf.Bytes())
+	hashsum := h.Sum64()
+	return strconv.FormatUint(hashsum, 16), nil
+}
+
 func (s generalInterpreter) ExecFile(filename string, funcname string) (interface{}, error) {
 	logrus.WithField("filename", filename).Debug("interprete the file")
 	var src interface{}