Skip to content

Commit

Permalink
Merge pull request #141 from paketo-buildpacks/cache-layer-fix
Browse files Browse the repository at this point in the history
Rewrite LayerContributor
  • Loading branch information
pivotal-david-osullivan authored Apr 26, 2022
2 parents f02533e + 6d6ee18 commit 49b5c1d
Show file tree
Hide file tree
Showing 5 changed files with 367 additions and 31 deletions.
84 changes: 68 additions & 16 deletions layer.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package libpak

import (
"fmt"
"io/fs"
"os"
"path/filepath"
"reflect"
Expand Down Expand Up @@ -64,34 +65,31 @@ type LayerFunc func() (libcnb.Layer, error)

// Contribute is the function to call when implementing your libcnb.LayerContributor.
func (l *LayerContributor) Contribute(layer libcnb.Layer, f LayerFunc) (libcnb.Layer, error) {
raw, err := toml.Marshal(l.ExpectedMetadata)
layerRestored, err := l.checkIfLayerRestored(layer)
if err != nil {
return libcnb.Layer{}, fmt.Errorf("unable to encode metadata\n%w", err)
return libcnb.Layer{}, fmt.Errorf("unable to check metadata\n%w", err)
}

expected := map[string]interface{}{}
if err := toml.Unmarshal(raw, &expected); err != nil {
return libcnb.Layer{}, fmt.Errorf("unable to decode metadata\n%w", err)
expected, cached, err := l.checkIfMetadataMatches(layer)
if err != nil {
return libcnb.Layer{}, fmt.Errorf("unable to check metadata\n%w", err)
}

l.Logger.Debugf("Expected metadata: %+v", expected)
l.Logger.Debugf("Actual metadata: %+v", layer.Metadata)

// TODO: compare entire layer not just metadata (in case build, launch, or cache have changed)
if reflect.DeepEqual(expected, layer.Metadata) {
if cached && layerRestored {
l.Logger.Headerf("%s: %s cached layer", color.BlueString(l.Name), color.GreenString("Reusing"))
layer.LayerTypes = l.ExpectedTypes
return layer, nil
}

l.Logger.Headerf("%s: %s to layer", color.BlueString(l.Name), color.YellowString("Contributing"))

if err := os.RemoveAll(layer.Path); err != nil {
return libcnb.Layer{}, fmt.Errorf("unable to remove existing layer directory %s\n%w", layer.Path, err)
if !layerRestored {
l.Logger.Headerf("%s: %s cached layer", color.BlueString(l.Name), color.RedString("Reloading"))
} else {
l.Logger.Headerf("%s: %s to layer", color.BlueString(l.Name), color.YellowString("Contributing"))
}

if err := os.MkdirAll(layer.Path, 0755); err != nil {
return libcnb.Layer{}, fmt.Errorf("unable to create layer directory %s\n%w", layer.Path, err)
err = l.reset(layer)
if err != nil {
return libcnb.Layer{}, fmt.Errorf("unable to reset\n%w", err)
}

layer, err = f()
Expand All @@ -105,6 +103,60 @@ func (l *LayerContributor) Contribute(layer libcnb.Layer, f LayerFunc) (libcnb.L
return layer, nil
}

func (l *LayerContributor) checkIfMetadataMatches(layer libcnb.Layer) (map[string]interface{}, bool, error) {
raw, err := toml.Marshal(l.ExpectedMetadata)
if err != nil {
return map[string]interface{}{}, false, fmt.Errorf("unable to encode metadata\n%w", err)
}

expected := map[string]interface{}{}
if err := toml.Unmarshal(raw, &expected); err != nil {
return map[string]interface{}{}, false, fmt.Errorf("unable to decode metadata\n%w", err)
}

l.Logger.Debugf("Expected metadata: %+v", expected)
l.Logger.Debugf("Actual metadata: %+v", layer.Metadata)

return expected, reflect.DeepEqual(expected, layer.Metadata), nil
}

func (l *LayerContributor) checkIfLayerRestored(layer libcnb.Layer) (bool, error) {
layerTOML := fmt.Sprintf("%s.toml", layer.Path)
tomlExists, err := sherpa.FileExists(layerTOML)
if err != nil {
return false, fmt.Errorf("unable to check if layer toml exists %s\n%w", layerTOML, err)
}

layerDirExists, err := sherpa.DirExists(layer.Path)
if err != nil {
return false, fmt.Errorf("unable to check if layer directory exists %s\n%w", layer.Path, err)
}

var dirContents []fs.DirEntry
if layerDirExists {
dirContents, err = os.ReadDir(layer.Path)
if err != nil {
return false, fmt.Errorf("unable to read directory %s\n%w", layer.Path, err)
}
}

l.Logger.Debugf("Check If Layer Restored -> tomlExists: %s, layerDirExists: %s, dirContents: %s, cache: %s, build: %s",
tomlExists, layerDirExists, dirContents, l.ExpectedTypes.Cache, l.ExpectedTypes.Build)
return !(tomlExists && (!layerDirExists || len(dirContents) == 0) && (l.ExpectedTypes.Cache || l.ExpectedTypes.Build)), nil
}

func (l *LayerContributor) reset(layer libcnb.Layer) error {
if err := os.RemoveAll(layer.Path); err != nil {
return fmt.Errorf("unable to remove existing layer directory %s\n%w", layer.Path, err)
}

if err := os.MkdirAll(layer.Path, 0755); err != nil {
return fmt.Errorf("unable to create layer directory %s\n%w", layer.Path, err)
}

return nil
}

// DependencyLayerContributor is a helper for implementing a libcnb.LayerContributor for a BuildpackDependency in order
// to get consistent logging and avoidance.
type DependencyLayerContributor struct {
Expand Down
107 changes: 92 additions & 15 deletions layer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,22 +37,24 @@ func testLayer(t *testing.T, context spec.G, it spec.S) {
var (
Expect = NewWithT(t).Expect

layer libcnb.Layer
layersDir string
layer libcnb.Layer
)

it.Before(func() {
var err error

layer.Path, err = ioutil.TempDir("", "layer")
layersDir, err = ioutil.TempDir("", "layer")
Expect(err).NotTo(HaveOccurred())
layer.Path = filepath.Join(layersDir, "test-layer")

layer.Exec.Path = layer.Path
layer.Metadata = map[string]interface{}{}
layer.Profile = libcnb.Profile{}
})

it.After(func() {
Expect(os.RemoveAll(layer.Path)).To(Succeed())
Expect(os.RemoveAll(layersDir)).To(Succeed())
})

context("LayerContributor", func() {
Expand Down Expand Up @@ -96,6 +98,90 @@ func testLayer(t *testing.T, context spec.G, it spec.S) {
Expect(called).To(BeTrue())
})

context("reloads layers not restored", func() {
var called bool

it.Before(func() {
layer.Metadata = map[string]interface{}{
"alpha": "test-alpha",
"bravo": map[string]interface{}{
"bravo-1": "test-bravo-1",
"bravo-2": "test-bravo-2",
},
}
})

it("calls function with matching metadata but no layer directory on cache layer", func() {
Expect(ioutil.WriteFile(fmt.Sprintf("%s.toml", layer.Path), []byte{}, 0644)).To(Succeed())
Expect(os.RemoveAll(layer.Path)).To(Succeed())
lc.ExpectedTypes.Cache = true

_, err := lc.Contribute(layer, func() (libcnb.Layer, error) {
called = true
return layer, nil
})
Expect(err).NotTo(HaveOccurred())

Expect(called).To(BeTrue())
})

it("calls function with matching metadata but no layer directory on build layer", func() {
Expect(ioutil.WriteFile(fmt.Sprintf("%s.toml", layer.Path), []byte{}, 0644)).To(Succeed())
Expect(os.RemoveAll(layer.Path)).To(Succeed())
lc.ExpectedTypes.Build = true

_, err := lc.Contribute(layer, func() (libcnb.Layer, error) {
called = true
return layer, nil
})
Expect(err).NotTo(HaveOccurred())

Expect(called).To(BeTrue())
})

it("calls function with matching metadata but an empty layer directory on build layer", func() {
Expect(ioutil.WriteFile(fmt.Sprintf("%s.toml", layer.Path), []byte{}, 0644)).To(Succeed())
Expect(os.MkdirAll(layer.Path, 0755)).To(Succeed())
lc.ExpectedTypes.Build = true

_, err := lc.Contribute(layer, func() (libcnb.Layer, error) {
called = true
return layer, nil
})
Expect(err).NotTo(HaveOccurred())

Expect(called).To(BeTrue())
})

it("does not call function with matching metadata when layer directory exists and has a file in it", func() {
Expect(ioutil.WriteFile(fmt.Sprintf("%s.toml", layer.Path), []byte{}, 0644)).To(Succeed())
Expect(os.MkdirAll(layer.Path, 0755)).To(Succeed())
Expect(ioutil.WriteFile(filepath.Join(layer.Path, "foo"), []byte{}, 0644)).To(Succeed())
lc.ExpectedTypes.Build = true

_, err := lc.Contribute(layer, func() (libcnb.Layer, error) {
called = true
return layer, nil
})
Expect(err).NotTo(HaveOccurred())

Expect(called).To(BeFalse())
})

it("does not call function with matching metadata when layer TOML missing", func() {
Expect(os.MkdirAll(layer.Path, 0755)).To(Succeed())
layer.Build = true

_, err := lc.Contribute(layer, func() (libcnb.Layer, error) {
called = true
return layer, nil
})
Expect(err).NotTo(HaveOccurred())

Expect(called).To(BeFalse())
})
})

it("does not call function with matching metadata", func() {
layer.Metadata = map[string]interface{}{
"alpha": "test-alpha",
Expand Down Expand Up @@ -675,22 +761,13 @@ func testLayer(t *testing.T, context spec.G, it spec.S) {
})

it("adds expected Syft SBOM file", func() {
layer.Metadata = map[string]interface{}{
"id": buildpack.Info.ID,
"name": buildpack.Info.Name,
"version": buildpack.Info.Version,
"homepage": buildpack.Info.Homepage,
"clear-env": buildpack.Info.ClearEnvironment,
"description": "",
"sbom-formats": []interface{}{},
"keywords": []interface{}{},
}
layer.Metadata = map[string]interface{}{}

_, err := hlc.Contribute(layer)
Expect(err).NotTo(HaveOccurred())

Expect(filepath.Join(layer.Exec.FilePath("test-name-1"))).NotTo(BeAnExistingFile())
Expect(filepath.Join(layer.Exec.FilePath("test-name-2"))).NotTo(BeAnExistingFile())
Expect(filepath.Join(layer.Exec.FilePath("test-name-1"))).To(BeAnExistingFile())
Expect(filepath.Join(layer.Exec.FilePath("test-name-2"))).To(BeAnExistingFile())

outputFile := layer.SBOMPath(libcnb.SyftJSON)
Expect(outputFile).To(BeARegularFile())
Expand Down
63 changes: 63 additions & 0 deletions sherpa/exists.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
* Copyright 2018-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package sherpa

import "os"

// Exists returns true if the path exists.
func Exists(path string) (bool, error) {
if _, err := os.Stat(path); err == nil {
return true, nil
} else if os.IsNotExist(err) {
return false, nil
} else {
return false, err
}
}

// FileExists returns true if the path exists and is a regular file.
func FileExists(path string) (bool, error) {
if stat, err := os.Stat(path); err == nil {
return stat.Mode().IsRegular(), nil
} else if os.IsNotExist(err) {
return false, nil
} else {
return false, err
}
}

// DirExists returns true if the path exists and is a directory.
func DirExists(path string) (bool, error) {
if stat, err := os.Stat(path); err == nil {
return stat.IsDir(), nil
} else if os.IsNotExist(err) {
return false, nil
} else {
return false, err
}
}

// SymlinkExists returns true if the path exists and is a symlink.
func SymlinkExists(path string) (bool, error) {
if stat, err := os.Lstat(path); err == nil {
return stat.Mode()&os.ModeSymlink != 0, nil
} else if os.IsNotExist(err) {
return false, nil
} else {
return false, err
}
}
Loading

0 comments on commit 49b5c1d

Please sign in to comment.