Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for processing files with prepended bytes before the zip archive #428

Merged
merged 4 commits into from
Jun 4, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 36 additions & 52 deletions internal/file/zip_file_helpers_test.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package file

import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"syscall"
"testing"

"github.com/stretchr/testify/assert"
)

var expectedZipArchiveEntries = []string{
Expand All @@ -18,35 +19,21 @@ var expectedZipArchiveEntries = []string{
"nested.zip",
}

// fatalIfError calls the supplied function. If the function returns a non-nil error, t.Fatal(err) is called.
func fatalIfError(t *testing.T, fn func() error) {
t.Helper()

if fn == nil {
return
}

err := fn()
if err != nil {
t.Fatal(err)
}
}

// createZipArchive creates a new ZIP archive file at destinationArchivePath based on the directory found at
// sourceDirPath.
func createZipArchive(t *testing.T, sourceDirPath, destinationArchivePath string) error {
func createZipArchive(t testing.TB, sourceDirPath, destinationArchivePath string) {
t.Helper()

cwd, err := os.Getwd()
if err != nil {
return fmt.Errorf("unable to get cwd: %+v", err)
t.Fatalf("unable to get cwd: %+v", err)
}

cmd := exec.Command("./generate-zip-fixture.sh", destinationArchivePath, path.Base(sourceDirPath))
cmd := exec.Command("./generate-zip-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath))
cmd.Dir = filepath.Join(cwd, "test-fixtures")

if err := cmd.Start(); err != nil {
return fmt.Errorf("unable to start generate zip fixture script: %+v", err)
t.Fatalf("unable to start generate zip fixture script: %+v", err)
}

if err := cmd.Wait(); err != nil {
Expand All @@ -59,73 +46,70 @@ func createZipArchive(t *testing.T, sourceDirPath, destinationArchivePath string
// an ExitStatus() method with the same signature.
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
if status.ExitStatus() != 0 {
return fmt.Errorf("failed to generate fixture: rc=%d", status.ExitStatus())
t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
}
}
} else {
return fmt.Errorf("unable to get generate fixture script result: %+v", err)
t.Fatalf("unable to get generate fixture script result: %+v", err)
}
}

return nil
}

func assertNoError(t testing.TB, fn func() error) func() {
return func() {
assert.NoError(t, fn())
}
}

// setupZipFileTest encapsulates common test setup work for zip file tests. It returns a cleanup function,
// which should be called (typically deferred) by the caller, the path of the created zip archive, and an error,
// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil
// (even if there's an error), and it should always be called.
func setupZipFileTest(t *testing.T, sourceDirPath string) (func() error, string, error) {
func setupZipFileTest(t testing.TB, sourceDirPath string) string {
t.Helper()

// Keep track of any needed cleanup work as we go
var cleanupFns []func() error
cleanup := func(fns []func() error) func() error {
return func() error {
for _, fn := range fns {
err := fn()
if err != nil {
return err
}
}

return nil
}
}

archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
if err != nil {
return cleanup(cleanupFns), "", fmt.Errorf("unable to create tempfile: %+v", err)
t.Fatalf("unable to create tempfile: %+v", err)
}
cleanupFns = append(cleanupFns, func() error { return os.Remove(archivePrefix.Name()) })

t.Cleanup(
wagoodman marked this conversation as resolved.
Show resolved Hide resolved
assertNoError(t,
func() error {
return os.Remove(archivePrefix.Name())
},
),
)

destinationArchiveFilePath := archivePrefix.Name() + ".zip"
t.Logf("archive path: %s", destinationArchiveFilePath)
err = createZipArchive(t, sourceDirPath, destinationArchiveFilePath)
cleanupFns = append(cleanupFns, func() error { return os.Remove(destinationArchiveFilePath) })
if err != nil {
return cleanup(cleanupFns), "", err
}
createZipArchive(t, sourceDirPath, destinationArchiveFilePath)

t.Cleanup(
assertNoError(t,
func() error {
return os.Remove(destinationArchiveFilePath)
},
),
)

cwd, err := os.Getwd()
if err != nil {
return cleanup(cleanupFns), "", fmt.Errorf("unable to get cwd: %+v", err)
t.Fatalf("unable to get cwd: %+v", err)
}

t.Logf("running from: %s", cwd)

return cleanup(cleanupFns), destinationArchiveFilePath, nil
return destinationArchiveFilePath
}

// TODO: Consider moving any non-git asset generation to a task (e.g. make) that's run ahead of running go tests.
func ensureNestedZipExists(t *testing.T, sourceDirPath string) error {
t.Helper()

nestedArchiveFilePath := path.Join(sourceDirPath, "nested.zip")
err := createZipArchive(t, sourceDirPath, nestedArchiveFilePath)

if err != nil {
return fmt.Errorf("unable to create nested archive for test fixture: %+v", err)
}
createZipArchive(t, sourceDirPath, nestedArchiveFilePath)

return nil
}
43 changes: 18 additions & 25 deletions internal/file/zip_file_manifest.go
Original file line number Diff line number Diff line change
@@ -1,23 +1,36 @@
package file

import (
"archive/zip"
"fmt"
"os"
"sort"
"strings"

"github.com/anchore/syft/internal"

"github.com/anchore/syft/internal/log"
)

// ZipFileManifest is a collection of paths and their file metadata.
type ZipFileManifest map[string]os.FileInfo

// newZipManifest creates an empty ZipFileManifest.
func newZipManifest() ZipFileManifest {
return make(ZipFileManifest)
// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path.
func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
zipReader, err := OpenZip(archivePath)
manifest := make(ZipFileManifest)
if err != nil {
return manifest, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
}
defer func() {
err = zipReader.Close()
if err != nil {
log.Warnf("unable to close zip archive (%s): %+v", archivePath, err)
}
}()

for _, file := range zipReader.Reader.File {
manifest.Add(file.Name, file.FileInfo())
}
return manifest, nil
}

// Add a new path and it's file metadata to the collection.
Expand Down Expand Up @@ -47,26 +60,6 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
return results
}

// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path.
func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
zipReader, err := zip.OpenReader(archivePath)
manifest := newZipManifest()
if err != nil {
return manifest, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
}
defer func() {
err = zipReader.Close()
if err != nil {
log.Errorf("unable to close zip archive (%s): %+v", archivePath, err)
}
}()

for _, file := range zipReader.Reader.File {
manifest.Add(file.Name, file.FileInfo())
}
return manifest, nil
}

// normalizeZipEntryName takes the given path entry and ensures it is prefixed with "/".
func normalizeZipEntryName(entry string) string {
if !strings.HasPrefix(entry, "/") {
Expand Down
13 changes: 2 additions & 11 deletions internal/file/zip_file_manifest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,7 @@ func TestNewZipFileManifest(t *testing.T) {
t.Fatal(err)
}

cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
defer fatalIfError(t, cleanup)
if err != nil {
t.Fatal(err)
}
archiveFilePath := setupZipFileTest(t, sourceDirPath)

actual, err := NewZipFileManifest(archiveFilePath)
if err != nil {
Expand Down Expand Up @@ -63,12 +59,7 @@ func TestZipFileManifest_GlobMatch(t *testing.T) {
t.Fatal(err)
}

cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
//goland:noinspection GoNilness
defer fatalIfError(t, cleanup)
if err != nil {
t.Fatal(err)
}
archiveFilePath := setupZipFileTest(t, sourceDirPath)

z, err := NewZipFileManifest(archiveFilePath)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/file/zip_file_traversal.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func newZipTraverseRequest(paths ...string) zipTraversalRequest {
func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths ...string) error {
request := newZipTraverseRequest(paths...)

zipReader, err := zip.OpenReader(archivePath)
zipReader, err := OpenZip(archivePath)
if err != nil {
return fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
}
Expand Down
Loading