Skip to content

Commit

Permalink
Node properties: support for leaf nodes
Browse files Browse the repository at this point in the history
  • Loading branch information
@ola-rozenfeld
ola-rozenfeld committed Jul 2, 2023
1 parent 01f4f61 commit 97c66f9
Show file tree
Hide file tree
Showing 10 changed files with 441 additions and 256 deletions.
15 changes: 6 additions & 9 deletions go/api/command/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,18 @@ proto_library(
name = "command_proto",
srcs = ["command.proto"],
visibility = ["//visibility:public"],
deps = ["@com_google_protobuf//:timestamp_proto"],
deps = [
"@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto",
"@com_google_protobuf//:timestamp_proto",
],
)

go_proto_library(
name = "command_go_proto",
importpath = "github.com/bazelbuild/remote-apis-sdks/go/api/command",
proto = ":cmd_proto",
proto = ":command_proto",
visibility = ["//visibility:public"],
deps = ["@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto"],
)

go_library(
Expand All @@ -22,10 +26,3 @@ go_library(
importpath = "github.com/bazelbuild/remote-apis-sdks/go/api/command",
visibility = ["//visibility:public"],
)

proto_library(
name = "cmd_proto",
srcs = ["command.proto"],
visibility = ["//visibility:public"],
deps = ["@com_google_protobuf//:timestamp_proto"],
)
312 changes: 171 additions & 141 deletions go/api/command/command.pb.go
View file

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions go/api/command/command.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ syntax = "proto3";

package cmd;

import "build/bazel/remote/execution/v2/remote_execution.proto";
import "google/protobuf/timestamp.proto";

// A command to execute locally or remotely.
Expand Down Expand Up @@ -123,6 +124,9 @@ message InputSpec {

// Determines how symlinks should be treated when constructing the input tree.
SymlinkBehaviorType.Value symlink_behavior = 6;

// Node properties of inputs.
map<string,build.bazel.remote.execution.v2.NodeProperties> input_node_properties = 7;
}

message OutputSpec {
Expand Down
46 changes: 27 additions & 19 deletions go/pkg/client/tree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ type fileSysNode struct {
file *fileNode
emptyDirectoryMarker bool
symlink *symlinkNode
nodeProperties *repb.NodeProperties
}

// TreeStats contains various stats/metadata of the constructed Merkle tree.
Expand Down Expand Up @@ -164,7 +165,7 @@ func getExecRootRelPaths(absPath, execRoot, workingDir, remoteWorkingDir string)

// loadFiles reads all files specified by the given InputSpec (descending into subdirectories
// recursively), and loads their contents into the provided map.
func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*command.InputExclusion, filesToProcess []string, fs map[string]*fileSysNode, cache filemetadata.Cache, opts *TreeSymlinkOpts) error {
func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*command.InputExclusion, filesToProcess []string, fs map[string]*fileSysNode, cache filemetadata.Cache, opts *TreeSymlinkOpts, nodeProperties map[string]*repb.NodeProperties) error {
if opts == nil {
opts = DefaultTreeSymlinkOpts()
}
Expand All @@ -181,6 +182,7 @@ func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*comma
if err != nil {
return err
}
np := nodeProperties[remoteNormPath]
meta := cache.Get(absPath)

// An implication of this is that, if a path is a symlink to a
Expand Down Expand Up @@ -213,7 +215,8 @@ func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*comma
// an absolute path. Since the remote worker will map the exec root
// to a different directory, we must strip away the local exec root.
// See https://github.com/bazelbuild/remote-apis-sdks/pull/229#discussion_r524830458
symlink: &symlinkNode{target: targetSymDir},
symlink: &symlinkNode{target: targetSymDir},
nodeProperties: np,
}

if !meta.Symlink.IsDangling && opts.FollowsTarget {
Expand Down Expand Up @@ -248,7 +251,7 @@ func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*comma

if len(files) == 0 {
if normPath != "." {
fs[remoteNormPath] = &fileSysNode{emptyDirectoryMarker: true}
fs[remoteNormPath] = &fileSysNode{emptyDirectoryMarker: true, nodeProperties: np}
}
continue
}
Expand All @@ -267,6 +270,7 @@ func loadFiles(execRoot, localWorkingDir, remoteWorkingDir string, excl []*comma
ue: uploadinfo.EntryFromFile(meta.Digest, absPath),
isExecutable: meta.IsExecutable,
},
nodeProperties: np,
}
}
}
Expand All @@ -286,9 +290,10 @@ func (c *Client) ComputeMerkleTree(execRoot, workingDir, remoteWorkingDir string
if err != nil {
return digest.Empty, nil, nil, err
}
np := is.InputNodeProperties[remoteNormPath]
if i.IsEmptyDirectory {
if normPath != "." {
fs[remoteNormPath] = &fileSysNode{emptyDirectoryMarker: true}
fs[remoteNormPath] = &fileSysNode{emptyDirectoryMarker: true, nodeProperties: np}
}
continue
}
Expand All @@ -297,9 +302,10 @@ func (c *Client) ComputeMerkleTree(execRoot, workingDir, remoteWorkingDir string
ue: uploadinfo.EntryFromBlob(i.Contents),
isExecutable: i.IsExecutable,
},
nodeProperties: np,
}
}
if err := loadFiles(execRoot, workingDir, remoteWorkingDir, is.InputExclusions, is.Inputs, fs, cache, treeSymlinkOpts(c.TreeSymlinkOpts, is.SymlinkBehavior)); err != nil {
if err := loadFiles(execRoot, workingDir, remoteWorkingDir, is.InputExclusions, is.Inputs, fs, cache, treeSymlinkOpts(c.TreeSymlinkOpts, is.SymlinkBehavior), is.InputNodeProperties); err != nil {
return digest.Empty, nil, nil, err
}
ft, err := buildTree(fs)
Expand Down Expand Up @@ -371,17 +377,16 @@ func packageTree(t *treeNode, stats *TreeStats) (root digest.Digest, blobs map[d
sort.Slice(dir.Directories, func(i, j int) bool { return dir.Directories[i].Name < dir.Directories[j].Name })

for name, n := range t.leaves {
// A node can have exactly one of file/symlink/emptyDirectoryMarker.
if n.file != nil {
dg := n.file.ue.Digest
dir.Files = append(dir.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable})
dir.Files = append(dir.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable, NodeProperties: n.nodeProperties})
blobs[dg] = n.file.ue
stats.InputFiles++
stats.TotalInputBytes += dg.Size
continue
}
if n.symlink != nil {
dir.Symlinks = append(dir.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target})
dir.Symlinks = append(dir.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target, NodeProperties: n.nodeProperties})
stats.InputSymlinks++
}
}
Expand All @@ -407,6 +412,7 @@ type TreeOutput struct {
IsExecutable bool
IsEmptyDirectory bool
SymlinkTarget string
NodeProperties *repb.NodeProperties
}

// FlattenTree takes a Tree message and calculates the relative paths of all the files to
Expand Down Expand Up @@ -456,24 +462,27 @@ func flattenTree(root digest.Digest, rootPath string, dirs map[digest.Digest]*re
Path: flatDir.p,
Digest: digest.Empty,
IsEmptyDirectory: true,
NodeProperties: dir.NodeProperties,
}
continue
}
// Add files to the set to return
for _, file := range dir.Files {
out := &TreeOutput{
Path: filepath.Join(flatDir.p, file.Name),
Digest: digest.NewFromProtoUnvalidated(file.Digest),
IsExecutable: file.IsExecutable,
Path: filepath.Join(flatDir.p, file.Name),
Digest: digest.NewFromProtoUnvalidated(file.Digest),
IsExecutable: file.IsExecutable,
NodeProperties: file.NodeProperties,
}
flatFiles[out.Path] = out
}

// Add symlinks to the set to return
for _, sm := range dir.Symlinks {
out := &TreeOutput{
Path: filepath.Join(flatDir.p, sm.Name),
SymlinkTarget: sm.Target,
Path: filepath.Join(flatDir.p, sm.Name),
SymlinkTarget: sm.Target,
NodeProperties: sm.NodeProperties,
}
flatFiles[out.Path] = out
}
Expand Down Expand Up @@ -520,15 +529,14 @@ func packageDirectories(t *treeNode) (root *repb.Directory, files map[digest.Dig
sort.Slice(root.Directories, func(i, j int) bool { return root.Directories[i].Name < root.Directories[j].Name })

for name, n := range t.leaves {
// A node can have exactly one of file/symlink/emptyDirectoryMarker.
if n.file != nil {
dg := n.file.ue.Digest
root.Files = append(root.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable})
root.Files = append(root.Files, &repb.FileNode{Name: name, Digest: dg.ToProto(), IsExecutable: n.file.isExecutable, NodeProperties: n.nodeProperties})
files[dg] = n.file.ue
continue
}
if n.symlink != nil {
root.Symlinks = append(root.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target})
root.Symlinks = append(root.Symlinks, &repb.SymlinkNode{Name: name, Target: n.symlink.target, NodeProperties: n.nodeProperties})
}
}
sort.Slice(root.Files, func(i, j int) bool { return root.Files[i].Name < root.Files[j].Name })
Expand All @@ -540,7 +548,7 @@ func packageDirectories(t *treeNode) (root *repb.Directory, files map[digest.Dig
// ComputeOutputsToUpload transforms the provided local output paths into uploadable Chunkers.
// The paths have to be relative to execRoot.
// It also populates the remote ActionResult, packaging output directories as trees where required.
func (c *Client) ComputeOutputsToUpload(execRoot, workingDir string, paths []string, cache filemetadata.Cache, sb command.SymlinkBehaviorType) (map[digest.Digest]*uploadinfo.Entry, *repb.ActionResult, error) {
func (c *Client) ComputeOutputsToUpload(execRoot, workingDir string, paths []string, cache filemetadata.Cache, sb command.SymlinkBehaviorType, nodeProperties map[string]*repb.NodeProperties) (map[digest.Digest]*uploadinfo.Entry, *repb.ActionResult, error) {
outs := make(map[digest.Digest]*uploadinfo.Entry)
resPb := &repb.ActionResult{}
for _, path := range paths {
Expand All @@ -563,12 +571,12 @@ func (c *Client) ComputeOutputsToUpload(execRoot, workingDir string, paths []str
// A regular file.
ue := uploadinfo.EntryFromFile(meta.Digest, absPath)
outs[meta.Digest] = ue
resPb.OutputFiles = append(resPb.OutputFiles, &repb.OutputFile{Path: normPath, Digest: meta.Digest.ToProto(), IsExecutable: meta.IsExecutable})
resPb.OutputFiles = append(resPb.OutputFiles, &repb.OutputFile{Path: normPath, Digest: meta.Digest.ToProto(), IsExecutable: meta.IsExecutable, NodeProperties: nodeProperties[normPath]})
continue
}
// A directory.
fs := make(map[string]*fileSysNode)
if e := loadFiles(absPath, "", "", nil, []string{"."}, fs, cache, treeSymlinkOpts(c.TreeSymlinkOpts, sb)); e != nil {
if e := loadFiles(absPath, "", "", nil, []string{"."}, fs, cache, treeSymlinkOpts(c.TreeSymlinkOpts, sb), nodeProperties); e != nil {
return nil, nil, e
}
ft, err := buildTree(fs)
Expand Down
Loading

0 comments on commit 97c66f9

Please sign in to comment.