Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Node properties: support for leaf nodes #475

Merged
merged 5 commits into from
Jul 6, 2023
Merged

Node properties: support for leaf nodes #475

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1cd9885
Node properties: support for leaf nodes
ola-rozenfeld Jun 18, 2023
eb57ce3
fixing test flakiness
ola-rozenfeld Jul 5, 2023
131ca1f
fixing lint errors (unwrapped errors)
ola-rozenfeld Jul 5, 2023
1ab18ae
Addressing comments
ola-rozenfeld Jul 5, 2023
9ef6dd4
addressing more comments
ola-rozenfeld Jul 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions go/api/command/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,18 @@ proto_library(
name = "command_proto",
srcs = ["command.proto"],
visibility = ["//visibility:public"],
deps = ["@com_google_protobuf//:timestamp_proto"],
deps = [
"@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto",
"@com_google_protobuf//:timestamp_proto",
],
)

go_proto_library(
name = "command_go_proto",
importpath = "github.com/bazelbuild/remote-apis-sdks/go/api/command",
proto = ":cmd_proto",
proto = ":command_proto",
visibility = ["//visibility:public"],
deps = ["@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto"],
)

go_library(
Expand All @@ -22,10 +26,3 @@ go_library(
importpath = "github.com/bazelbuild/remote-apis-sdks/go/api/command",
visibility = ["//visibility:public"],
)

proto_library(
ola-rozenfeld marked this conversation as resolved.
Show resolved Hide resolved
name = "cmd_proto",
srcs = ["command.proto"],
visibility = ["//visibility:public"],
deps = ["@com_google_protobuf//:timestamp_proto"],
)
312 changes: 171 additions & 141 deletions go/api/command/command.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions go/api/command/command.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ syntax = "proto3";

package cmd;

import "build/bazel/remote/execution/v2/remote_execution.proto";
import "google/protobuf/timestamp.proto";

// A command to execute locally or remotely.
Expand Down Expand Up @@ -123,6 +124,9 @@ message InputSpec {

// Determines how symlinks should be treated when constructing the input tree.
SymlinkBehaviorType.Value symlink_behavior = 6;

// Node properties of inputs.
map<string,build.bazel.remote.execution.v2.NodeProperties> input_node_properties = 7;
}

message OutputSpec {
Expand Down
44 changes: 27 additions & 17 deletions go/pkg/client/tree.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ type fileSysNode struct {
file *fileNode
emptyDirectoryMarker bool
symlink *symlinkNode
nodeProperties *repb.NodeProperties
}

// TreeStats contains various stats/metadata of the constructed Merkle tree.
Expand Down Expand Up @@ -164,7 +165,7 @@ func getExecRootRelPaths(absPath, execRoot, workingDir, remoteWorkingDir string)

// loadFiles reads all files specified by the given InputSpec (descending into subdirectories
// recursively), and loads their contents into the provided map.
func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*command.InputExclusion, filesToProcess []string, fs map[string]*fileSysNode, cache filemetadata.Cache, opts *TreeSymlinkOpts) error {
func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*command.InputExclusion, filesToProcess []string, fs map[string]*fileSysNode, cache filemetadata.Cache, opts *TreeSymlinkOpts, nodeProperties map[string]*repb.NodeProperties) error {
if opts == nil {
opts = DefaultTreeSymlinkOpts()
}
Expand All @@ -181,6 +182,7 @@ func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*comma
if err != nil {
return err
}
np := nodeProperties[remoteNormPath]
meta := cache.Get(absPath)

// An implication of this is that, if a path is a symlink to a
Expand Down Expand Up @@ -213,7 +215,8 @@ func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*comma
// an absolute path. Since the remote worker will map the exec root
// to a different directory, we must strip away the local exec root.
// See https://github.com/bazelbuild/remote-apis-sdks/pull/229#discussion_r524830458
symlink: &symlinkNode{target: targetSymDir},
symlink: &symlinkNode{target: targetSymDir},
nodeProperties: np,
}

if !meta.Symlink.IsDangling && opts.FollowsTarget {
Expand Down Expand Up @@ -248,7 +251,7 @@ func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*comma

if len(files) == 0 {
if normPath != "." {
fs[remoteNormPath] = &fileSysNode{emptyDirectoryMarker: true}
fs[remoteNormPath] = &fileSysNode{emptyDirectoryMarker: true, nodeProperties: np}
}
continue
}
Expand All @@ -267,6 +270,7 @@ func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*comma
ue: uploadinfo.EntryFromFile(meta.Digest, absPath),
isExecutable: meta.IsExecutable,
},
nodeProperties: np,
}
}
}
Expand All @@ -286,9 +290,10 @@ func (c *Client) ComputeMerkleTree(execRoot, workingDir, remoteWorkingDir string
if err != nil {
return digest.Empty, nil, nil, err
}
np := is.InputNodeProperties[remoteNormPath]
if i.IsEmptyDirectory {
if normPath != "." {
fs[remoteNormPath] = &fileSysNode{emptyDirectoryMarker: true}
fs[remoteNormPath] = &fileSysNode{emptyDirectoryMarker: true, nodeProperties: np}
}
continue
}
Expand All @@ -297,9 +302,10 @@ func (c *Client) ComputeMerkleTree(execRoot, workingDir, remoteWorkingDir string
ue: uploadinfo.EntryFromBlob(i.Contents),
isExecutable: i.IsExecutable,
},
nodeProperties: np,
}
}
if err := loadFiles(execRoot, workingDir, remoteWorkingDir, is.InputExclusions, is.Inputs, fs, cache, treeSymlinkOpts(c.TreeSymlinkOpts, is.SymlinkBehavior)); err != nil {
if err := loadFiles(execRoot, workingDir, remoteWorkingDir, is.InputExclusions, is.Inputs, fs, cache, treeSymlinkOpts(c.TreeSymlinkOpts, is.SymlinkBehavior), is.InputNodeProperties); err != nil {
return digest.Empty, nil, nil, err
}
ft, err := buildTree(fs)
Expand Down Expand Up @@ -374,14 +380,14 @@ func packageTree(t *treeNode, stats *TreeStats) (root digest.Digest, blobs map[d
// A node can have exactly one of file/symlink/emptyDirectoryMarker.
if n.file != nil {
dg := n.file.ue.Digest
dir.Files = append(dir.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable})
dir.Files = append(dir.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable, NodeProperties: n.nodeProperties})
blobs[dg] = n.file.ue
stats.InputFiles++
stats.TotalInputBytes += dg.Size
continue
}
if n.symlink != nil {
dir.Symlinks = append(dir.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target})
dir.Symlinks = append(dir.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target, NodeProperties: n.nodeProperties})
stats.InputSymlinks++
}
}
Expand All @@ -407,6 +413,7 @@ type TreeOutput struct {
IsExecutable bool
IsEmptyDirectory bool
SymlinkTarget string
NodeProperties *repb.NodeProperties
}

// FlattenTree takes a Tree message and calculates the relative paths of all the files to
Expand Down Expand Up @@ -456,24 +463,27 @@ func flattenTree(root digest.Digest, rootPath string, dirs map[digest.Digest]*re
Path: flatDir.p,
Digest: digest.Empty,
IsEmptyDirectory: true,
NodeProperties: dir.NodeProperties,
}
continue
}
// Add files to the set to return
for _, file := range dir.Files {
out := &TreeOutput{
Path: filepath.Join(flatDir.p, file.Name),
Digest: digest.NewFromProtoUnvalidated(file.Digest),
IsExecutable: file.IsExecutable,
Path: filepath.Join(flatDir.p, file.Name),
Digest: digest.NewFromProtoUnvalidated(file.Digest),
IsExecutable: file.IsExecutable,
NodeProperties: file.NodeProperties,
}
flatFiles[out.Path] = out
}

// Add symlinks to the set to return
for _, sm := range dir.Symlinks {
out := &TreeOutput{
Path: filepath.Join(flatDir.p, sm.Name),
SymlinkTarget: sm.Target,
Path: filepath.Join(flatDir.p, sm.Name),
SymlinkTarget: sm.Target,
NodeProperties: sm.NodeProperties,
}
flatFiles[out.Path] = out
}
Expand Down Expand Up @@ -523,12 +533,12 @@ func packageDirectories(t *treeNode) (root *repb.Directory, files map[digest.Dig
// A node can have exactly one of file/symlink/emptyDirectoryMarker.
if n.file != nil {
dg := n.file.ue.Digest
root.Files = append(root.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable})
root.Files = append(root.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable, NodeProperties: n.nodeProperties})
files[dg] = n.file.ue
continue
}
if n.symlink != nil {
root.Symlinks = append(root.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target})
root.Symlinks = append(root.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target, NodeProperties: n.nodeProperties})
}
}
sort.Slice(root.Files, func(i, j int) bool { return root.Files[i].Name < root.Files[j].Name })
Expand All @@ -540,7 +550,7 @@ func packageDirectories(t *treeNode) (root *repb.Directory, files map[digest.Dig
// ComputeOutputsToUpload transforms the provided local output paths into uploadable Chunkers.
// The paths have to be relative to execRoot.
// It also populates the remote ActionResult, packaging output directories as trees where required.
func (c *Client) ComputeOutputsToUpload(execRoot, workingDir string, paths []string, cache filemetadata.Cache, sb command.SymlinkBehaviorType) (map[digest.Digest]*uploadinfo.Entry, *repb.ActionResult, error) {
func (c *Client) ComputeOutputsToUpload(execRoot, workingDir string, paths []string, cache filemetadata.Cache, sb command.SymlinkBehaviorType, nodeProperties map[string]*repb.NodeProperties) (map[digest.Digest]*uploadinfo.Entry, *repb.ActionResult, error) {
outs := make(map[digest.Digest]*uploadinfo.Entry)
resPb := &repb.ActionResult{}
for _, path := range paths {
Expand All @@ -563,12 +573,12 @@ func (c *Client) ComputeOutputsToUpload(execRoot, workingDir string, paths []str
// A regular file.
ue := uploadinfo.EntryFromFile(meta.Digest, absPath)
outs[meta.Digest] = ue
resPb.OutputFiles = append(resPb.OutputFiles, &repb.OutputFile{Path: normPath, Digest: meta.Digest.ToProto(), IsExecutable: meta.IsExecutable})
resPb.OutputFiles = append(resPb.OutputFiles, &repb.OutputFile{Path: normPath, Digest: meta.Digest.ToProto(), IsExecutable: meta.IsExecutable, NodeProperties: nodeProperties[normPath]})
continue
}
// A directory.
fs := make(map[string]*fileSysNode)
if e := loadFiles(absPath, "", "", nil, []string{"."}, fs, cache, treeSymlinkOpts(c.TreeSymlinkOpts, sb)); e != nil {
if e := loadFiles(absPath, "", "", nil, []string{"."}, fs, cache, treeSymlinkOpts(c.TreeSymlinkOpts, sb), nodeProperties); e != nil {
return nil, nil, e
}
ft, err := buildTree(fs)
Expand Down
Loading