Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow scanning unpacked container filesystems #1485

Merged
merged 3 commits into from
Jan 30, 2023
Merged

Allow scanning unpacked container filesystems #1485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b9cacf0
source: avoid second-step of symlink resolution in directory resolver
jedevc Jan 20, 2023
533d60b
source: add base parameter to directory resolver
jedevc Jan 20, 2023
d1d5fa1
source: add tests for new base parameter
jedevc Jan 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 46 additions & 17 deletions syft/source/directory_resolver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type pathFilterFn func(string, os.FileInfo) bool
// directoryResolver implements path and content access for the directory data source.
type directoryResolver struct {
path string
base string
currentWdRelativeToRoot string
currentWd string
fileTree *filetree.FileTree
Expand All @@ -47,7 +48,7 @@ type directoryResolver struct {
errPaths map[string]error
}

func newDirectoryResolver(root string, pathFilters ...pathFilterFn) (*directoryResolver, error) {
func newDirectoryResolver(root string, base string, pathFilters ...pathFilterFn) (*directoryResolver, error) {
currentWD, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("could not get CWD: %w", err)
Expand All @@ -64,6 +65,18 @@ func newDirectoryResolver(root string, pathFilters ...pathFilterFn) (*directoryR
return nil, fmt.Errorf("could not evaluate root=%q symlinks: %w", root, err)
}

cleanBase := ""
if base != "" {
cleanBase, err = filepath.EvalSymlinks(base)
if err != nil {
return nil, fmt.Errorf("could not evaluate base=%q symlinks: %w", base, err)
}
cleanBase, err = filepath.Abs(cleanBase)
if err != nil {
return nil, err
}
}

var currentWdRelRoot string
if path.IsAbs(cleanRoot) {
currentWdRelRoot, err = filepath.Rel(cleanCWD, cleanRoot)
Expand All @@ -76,6 +89,7 @@ func newDirectoryResolver(root string, pathFilters ...pathFilterFn) (*directoryR

resolver := directoryResolver{
path: cleanRoot,
base: cleanBase,
currentWd: cleanCWD,
currentWdRelativeToRoot: currentWdRelRoot,
fileTree: filetree.NewFileTree(),
Expand Down Expand Up @@ -244,10 +258,25 @@ func (r directoryResolver) addSymlinkToIndex(p string, info os.FileInfo) (string
return "", fmt.Errorf("unable to readlink for path=%q: %w", p, err)
}

// note: if the link is not absolute (e.g, /dev/stderr -> fd/2 ) we need to resolve it relative to the directory
// in question (e.g. resolve to /dev/fd/2)
if !filepath.IsAbs(linkTarget) {
linkTarget = filepath.Join(filepath.Dir(p), linkTarget)
if filepath.IsAbs(linkTarget) {
// if the link is absolute (e.g, /bin/ls -> /bin/busybox) we need to
// resolve relative to the root of the base directory
linkTarget = filepath.Join(r.base, filepath.Clean(linkTarget))
} else {
// if the link is not absolute (e.g, /dev/stderr -> fd/2 ) we need to
// resolve it relative to the directory in question (e.g. resolve to
// /dev/fd/2)
if r.base == "" {
linkTarget = filepath.Join(filepath.Dir(p), linkTarget)
} else {
// if the base is set, then we first need to resolve the link,
// before finding it's location in the base
dir, err := filepath.Rel(r.base, filepath.Dir(p))
if err != nil {
return "", fmt.Errorf("unable to resolve relative path for path=%q: %w", p, err)
}
linkTarget = filepath.Join(r.base, filepath.Clean(filepath.Join("/", dir, linkTarget)))
}
}

ref, err := r.fileTree.AddSymLink(file.Path(p), file.Path(linkTarget))
Expand Down Expand Up @@ -336,14 +365,17 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
}

// we should be resolving symlinks and preserving this information as a VirtualPath to the real file
evaluatedPath, err := filepath.EvalSymlinks(userStrPath)
exists, ref, err := r.fileTree.File(file.Path(userStrPath), filetree.FollowBasenameLinks)
if err != nil {
log.Debugf("directory resolver unable to evaluate symlink for path=%q : %+v", userPath, err)
continue
}
if !exists {
continue
}

// TODO: why not use stored metadata?
fileMeta, err := os.Stat(evaluatedPath)
fileMeta, err := os.Stat(string(ref.RealPath))
if errors.Is(err, os.ErrNotExist) {
// note: there are other kinds of errors other than os.ErrNotExist that may be given that is platform
// specific, but essentially hints at the same overall problem (that the path does not exist). Such an
Expand All @@ -354,7 +386,7 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
// invalid paths. This logging statement is meant to raise IO or permissions related problems.
var pathErr *os.PathError
if !errors.As(err, &pathErr) {
log.Warnf("path is not valid (%s): %+v", evaluatedPath, err)
log.Warnf("path is not valid (%s): %+v", ref.RealPath, err)
}
continue
}
Expand All @@ -368,15 +400,12 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
userStrPath = windowsToPosix(userStrPath)
}

exists, ref, err := r.fileTree.File(file.Path(userStrPath), filetree.FollowBasenameLinks)
if err == nil && exists {
loc := NewVirtualLocationFromDirectory(
r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
*ref,
)
references = append(references, loc)
}
loc := NewVirtualLocationFromDirectory(
r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
*ref,
)
references = append(references, loc)
}

return references, nil
Expand Down
111 changes: 92 additions & 19 deletions syft/source/directory_resolver_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver(c.relativeRoot)
resolver, err := newDirectoryResolver(c.relativeRoot, "")
assert.NoError(t, err)

refs, err := resolver.FilesByPath(c.input)
Expand Down Expand Up @@ -111,7 +111,7 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
absRoot, err := filepath.Abs(c.relativeRoot)
require.NoError(t, err)

resolver, err := newDirectoryResolver(absRoot)
resolver, err := newDirectoryResolver(absRoot, "")
assert.NoError(t, err)

refs, err := resolver.FilesByPath(c.input)
Expand Down Expand Up @@ -172,7 +172,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver(c.root)
resolver, err := newDirectoryResolver(c.root, "")
assert.NoError(t, err)

hasPath := resolver.HasPath(c.input)
Expand Down Expand Up @@ -220,7 +220,7 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures")
resolver, err := newDirectoryResolver("./test-fixtures", "")
assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input...)
assert.NoError(t, err)
Expand All @@ -233,7 +233,7 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
}

func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures")
resolver, err := newDirectoryResolver("./test-fixtures", "")
assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/image-symlinks/file*")
assert.NoError(t, err)
Expand All @@ -242,15 +242,15 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
}

func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/image-symlinks")
resolver, err := newDirectoryResolver("./test-fixtures/image-symlinks", "")
assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err)
assert.Len(t, refs, 6)
}

func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures")
resolver, err := newDirectoryResolver("./test-fixtures", "")
assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt")
assert.NoError(t, err)
Expand All @@ -277,7 +277,7 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple")
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", "")
assert.NoError(t, err)

refs, err := resolver.FilesByPath(test.fixture)
Expand All @@ -300,7 +300,7 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {

func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
// let's make certain that "dev/place" is not ignored, since it is not "/dev/place"
resolver, err := newDirectoryResolver("test-fixtures/system_paths/target")
resolver, err := newDirectoryResolver("test-fixtures/system_paths/target", "")
assert.NoError(t, err)
// ensure the correct filter function is wired up by default
expectedFn := reflect.ValueOf(isUnallowableFileType)
Expand Down Expand Up @@ -431,7 +431,7 @@ func Test_isUnallowableFileType(t *testing.T) {

func Test_directoryResolver_index(t *testing.T) {
// note: this test is testing the effects from newDirectoryResolver, indexTree, and addPathToIndex
r, err := newDirectoryResolver("test-fixtures/system_paths/target")
r, err := newDirectoryResolver("test-fixtures/system_paths/target", "")
if err != nil {
t.Fatalf("unable to get indexed dir resolver: %+v", err)
}
Expand Down Expand Up @@ -608,7 +608,7 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
}
for _, test := range tests {
t.Run(test.fixturePath, func(t *testing.T) {
resolver, err := newDirectoryResolver(test.fixturePath)
resolver, err := newDirectoryResolver(test.fixturePath, "")
assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType(test.mimeType)
assert.NoError(t, err)
Expand All @@ -621,7 +621,7 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
}

func Test_IndexingNestedSymLinks(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple")
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", "")
require.NoError(t, err)

// check that we can get the real path
Expand Down Expand Up @@ -674,7 +674,7 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
return strings.HasSuffix(path, string(filepath.Separator)+"readme")
}

resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", filterFn)
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", "", filterFn)
require.NoError(t, err)

// the path to the real file is PRUNED from the index, so we should NOT expect a location returned
Expand All @@ -694,7 +694,7 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
}

func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-multiple-roots/root")
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-multiple-roots/root", "")
require.NoError(t, err)

// check that we can get the real path
Expand All @@ -712,7 +712,7 @@ func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
}

func Test_RootViaSymlink(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinked-root/nested/link-root")
resolver, err := newDirectoryResolver("./test-fixtures/symlinked-root/nested/link-root", "")
require.NoError(t, err)

locations, err := resolver.FilesByPath("./file1.txt")
Expand Down Expand Up @@ -753,7 +753,7 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
r, err := newDirectoryResolver(".")
r, err := newDirectoryResolver(".", "")
require.NoError(t, err)

actual, err := r.FileContentsByLocation(test.location)
Expand Down Expand Up @@ -819,7 +819,7 @@ func Test_isUnixSystemRuntimePath(t *testing.T) {

func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
test := func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-loop")
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-loop", "")
require.NoError(t, err)

locations, err := resolver.FilesByGlob("**/file.target")
Expand Down Expand Up @@ -853,7 +853,7 @@ func Test_IncludeRootPathInIndex(t *testing.T) {
return path != "/"
}

resolver, err := newDirectoryResolver("/", filterFn)
resolver, err := newDirectoryResolver("/", "", filterFn)
require.NoError(t, err)

exists, ref, err := resolver.fileTree.File(file.Path("/"))
Expand All @@ -870,7 +870,7 @@ func TestDirectoryResolver_indexPath(t *testing.T) {
tempFile, err := os.CreateTemp("", "")
require.NoError(t, err)

resolver, err := newDirectoryResolver(tempFile.Name())
resolver, err := newDirectoryResolver(tempFile.Name(), "")
require.NoError(t, err)

t.Run("filtering path with nil os.FileInfo", func(t *testing.T) {
Expand All @@ -885,3 +885,76 @@ func TestDirectoryResolver_indexPath(t *testing.T) {
})
})
}

func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
cases := []struct {
name string
root string
input string
expected []string
}{
{
name: "should find the base file",
root: "./test-fixtures/symlinks-base/",
input: "./base",
expected: []string{
"base",
},
},
{
name: "should follow a link with a pivoted root",
root: "./test-fixtures/symlinks-base/",
input: "./foo",
expected: []string{
"base",
},
},
{
name: "should follow a relative link with extra parents",
root: "./test-fixtures/symlinks-base/",
input: "./bar",
expected: []string{
"base",
},
},
{
name: "should follow an absolute link with extra parents",
root: "./test-fixtures/symlinks-base/",
input: "./baz",
expected: []string{
"base",
},
},
{
name: "should follow an absolute link with extra parents",
root: "./test-fixtures/symlinks-base/",
input: "./sub/link",
expected: []string{
"sub/item",
},
},
{
name: "should follow chained pivoted link",
root: "./test-fixtures/symlinks-base/",
input: "./chain",
expected: []string{
"base",
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver(c.root, c.root)
assert.NoError(t, err)

refs, err := resolver.FilesByPath(c.input)
require.NoError(t, err)
assert.Len(t, refs, len(c.expected))
s := strset.New()
for _, actual := range refs {
s.Add(actual.RealPath)
}
assert.ElementsMatch(t, c.expected, s.List())
})
}
}
1 change: 1 addition & 0 deletions syft/source/metadata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ type Metadata struct {
Scheme Scheme // the source data scheme type (directory or image)
ImageMetadata ImageMetadata // all image info (image only)
Path string // the root path to be cataloged (directory only)
Base string // the base path to be cataloged (directory only)
Name string
}
Loading