diff --git a/src/cmd/go/internal/cache/prog.go b/src/cmd/go/internal/cache/prog.go index b657497417b3d0..124128d172f107 100644 --- a/src/cmd/go/internal/cache/prog.go +++ b/src/cmd/go/internal/cache/prog.go @@ -7,6 +7,7 @@ package cache import ( "bufio" "cmd/go/internal/base" + "cmd/go/internal/cacheprog" "cmd/internal/quoted" "context" "crypto/sha256" @@ -38,7 +39,7 @@ type ProgCache struct { // can are the commands that the child process declared that it supports. // This is effectively the versioning mechanism. - can map[ProgCmd]bool + can map[cacheprog.ProgCmd]bool // fuzzDirCache is another Cache implementation to use for the FuzzDir // method. In practice this is the default GOCACHE disk-based @@ -55,7 +56,7 @@ type ProgCache struct { mu sync.Mutex // guards following fields nextID int64 - inFlight map[int64]chan<- *ProgResponse + inFlight map[int64]chan<- *cacheprog.ProgResponse outputFile map[OutputID]string // object => abs path on disk // writeMu serializes writing to the child process. @@ -63,134 +64,6 @@ type ProgCache struct { writeMu sync.Mutex } -// The following types define the protocol for a GOCACHEPROG program. -// -// By default, the go command manages a build cache stored in the file system -// itself. GOCACHEPROG can be set to the name of a command (with optional -// space-separated flags) that implements the go command build cache externally. -// This permits defining a different cache policy. -// -// The go command will start the GOCACHEPROG as a subprocess and communicate -// with it via JSON messages over stdin/stdout. The subprocess's stderr will be -// connected to the go command's stderr. -// -// The subprocess should immediately send a [ProgResponse] with its capabilities. -// After that, the go command will send a stream of [ProgRequest] messages and the -// subprocess should reply to each [ProgRequest] with a [ProgResponse] message. - -// ProgCmd is a command that can be issued to a child process. -// -// If the interface needs to grow, the go command can add new commands or new -// versioned commands like "get2" in the future. The initial [ProgResponse] from -// the child process indicates which commands it supports. -type ProgCmd string - -const ( - // cmdPut tells the cache program to store an object in the cache. - // - // [ProgRequest.ActionID] is the cache key of this object. The cache should - // store [ProgRequest.OutputID] and [ProgRequest.Body] under this key for a - // later "get" request. It must also store the Body in a file in the local - // file system and return the path to that file in [ProgResponse.DiskPath], - // which must exist at least until a "close" request. - cmdPut = ProgCmd("put") - - // cmdGet tells the cache program to retrieve an object from the cache. - // - // [ProgRequest.ActionID] specifies the key of the object to get. If the - // cache does not contain this object, it should set [ProgResponse.Miss] to - // true. Otherwise, it should populate the fields of [ProgResponse], - // including setting [ProgResponse.OutputID] to the OutputID of the original - // "put" request and [ProgResponse.DiskPath] to the path of a local file - // containing the Body of the original "put" request. That file must - // continue to exist at least until a "close" request. - cmdGet = ProgCmd("get") - - // cmdClose requests that the cache program exit gracefully. - // - // The cache program should reply to this request and then exit - // (thus closing its stdout). - cmdClose = ProgCmd("close") -) - -// ProgRequest is the JSON-encoded message that's sent from the go command to -// the GOCACHEPROG child process over stdin. Each JSON object is on its own -// line. A ProgRequest of Type "put" with BodySize > 0 will be followed by a -// line containing a base64-encoded JSON string literal of the body. -type ProgRequest struct { - // ID is a unique number per process across all requests. - // It must be echoed in the ProgResponse from the child. - ID int64 - - // Command is the type of request. - // The go command will only send commands that were declared - // as supported by the child. - Command ProgCmd - - // ActionID is the cache key for "put" and "get" requests. - ActionID []byte `json:",omitempty"` // or nil if not used - - // OutputID is stored with the body for "put" requests. - // - // Prior to Go 1.24, when GOCACHEPROG was still an experiment, this was - // accidentally named ObjectID. It was renamed to OutputID in Go 1.24. - OutputID []byte `json:",omitempty"` // or nil if not used - - // Body is the body for "put" requests. It's sent after the JSON object - // as a base64-encoded JSON string when BodySize is non-zero. - // It's sent as a separate JSON value instead of being a struct field - // send in this JSON object so large values can be streamed in both directions. - // The base64 string body of a ProgRequest will always be written - // immediately after the JSON object and a newline. - Body io.Reader `json:"-"` - - // BodySize is the number of bytes of Body. If zero, the body isn't written. - BodySize int64 `json:",omitempty"` - - // ObjectID is the accidental spelling of OutputID that was used prior to Go - // 1.24. - // - // Deprecated: use OutputID. This field is only populated temporarily for - // backwards compatibility with Go 1.23 and earlier when - // GOEXPERIMENT=gocacheprog is set. It will be removed in Go 1.25. - ObjectID []byte `json:",omitempty"` -} - -// ProgResponse is the JSON response from the child process to the go command. -// -// With the exception of the first protocol message that the child writes to its -// stdout with ID==0 and KnownCommands populated, these are only sent in -// response to a ProgRequest from the go command. -// -// ProgResponses can be sent in any order. The ID must match the request they're -// replying to. -type ProgResponse struct { - ID int64 // that corresponds to ProgRequest; they can be answered out of order - Err string `json:",omitempty"` // if non-empty, the error - - // KnownCommands is included in the first message that cache helper program - // writes to stdout on startup (with ID==0). It includes the - // ProgRequest.Command types that are supported by the program. - // - // This lets the go command extend the protocol gracefully over time (adding - // "get2", etc), or fail gracefully when needed. It also lets the go command - // verify the program wants to be a cache helper. - KnownCommands []ProgCmd `json:",omitempty"` - - // For "get" requests. - - Miss bool `json:",omitempty"` // cache miss - OutputID []byte `json:",omitempty"` // the ObjectID stored with the body - Size int64 `json:",omitempty"` // body size in bytes - Time *time.Time `json:",omitempty"` // when the object was put in the cache (optional; used for cache expiration) - - // For "get" and "put" requests. - - // DiskPath is the absolute path on disk of the body corresponding to a - // "get" (on cache hit) or "put" request's ActionID. - DiskPath string `json:",omitempty"` -} - // startCacheProg starts the prog binary (with optional space-separated flags) // and returns a Cache implementation that talks to it. // @@ -238,14 +111,14 @@ func startCacheProg(progAndArgs string, fuzzDirCache Cache) Cache { stdout: out, stdin: in, bw: bufio.NewWriter(in), - inFlight: make(map[int64]chan<- *ProgResponse), + inFlight: make(map[int64]chan<- *cacheprog.ProgResponse), outputFile: make(map[OutputID]string), readLoopDone: make(chan struct{}), } // Register our interest in the initial protocol message from the child to // us, saying what it can do. - capResc := make(chan *ProgResponse, 1) + capResc := make(chan *cacheprog.ProgResponse, 1) pc.inFlight[0] = capResc pc.jenc = json.NewEncoder(pc.bw) @@ -260,7 +133,7 @@ func startCacheProg(progAndArgs string, fuzzDirCache Cache) Cache { case <-timer.C: log.Printf("# still waiting for GOCACHEPROG %v ...", prog) case capRes := <-capResc: - can := map[ProgCmd]bool{} + can := map[cacheprog.ProgCmd]bool{} for _, cmd := range capRes.KnownCommands { can[cmd] = true } @@ -277,7 +150,7 @@ func (c *ProgCache) readLoop(readLoopDone chan<- struct{}) { defer close(readLoopDone) jd := json.NewDecoder(c.stdout) for { - res := new(ProgResponse) + res := new(cacheprog.ProgResponse) if err := jd.Decode(res); err != nil { if c.closing.Load() { return // quietly @@ -302,8 +175,8 @@ func (c *ProgCache) readLoop(readLoopDone chan<- struct{}) { } } -func (c *ProgCache) send(ctx context.Context, req *ProgRequest) (*ProgResponse, error) { - resc := make(chan *ProgResponse, 1) +func (c *ProgCache) send(ctx context.Context, req *cacheprog.ProgRequest) (*cacheprog.ProgResponse, error) { + resc := make(chan *cacheprog.ProgResponse, 1) if err := c.writeToChild(req, resc); err != nil { return nil, err } @@ -318,7 +191,7 @@ func (c *ProgCache) send(ctx context.Context, req *ProgRequest) (*ProgResponse, } } -func (c *ProgCache) writeToChild(req *ProgRequest, resc chan<- *ProgResponse) (err error) { +func (c *ProgCache) writeToChild(req *cacheprog.ProgRequest, resc chan<- *cacheprog.ProgResponse) (err error) { c.mu.Lock() c.nextID++ req.ID = c.nextID @@ -369,7 +242,7 @@ func (c *ProgCache) writeToChild(req *ProgRequest, resc chan<- *ProgResponse) (e } func (c *ProgCache) Get(a ActionID) (Entry, error) { - if !c.can[cmdGet] { + if !c.can[cacheprog.CmdGet] { // They can't do a "get". Maybe they're a write-only cache. // // TODO(bradfitz,bcmills): figure out the proper error type here. Maybe @@ -379,8 +252,8 @@ func (c *ProgCache) Get(a ActionID) (Entry, error) { // error types on the Cache interface. return Entry{}, &entryNotFoundError{} } - res, err := c.send(c.ctx, &ProgRequest{ - Command: cmdGet, + res, err := c.send(c.ctx, &cacheprog.ProgRequest{ + Command: cacheprog.CmdGet, ActionID: a[:], }) if err != nil { @@ -436,7 +309,7 @@ func (c *ProgCache) Put(a ActionID, file io.ReadSeeker) (_ OutputID, size int64, return OutputID{}, 0, err } - if !c.can[cmdPut] { + if !c.can[cacheprog.CmdPut] { // Child is a read-only cache. Do nothing. return out, size, nil } @@ -448,8 +321,8 @@ func (c *ProgCache) Put(a ActionID, file io.ReadSeeker) (_ OutputID, size int64, deprecatedValue = out[:] } - res, err := c.send(c.ctx, &ProgRequest{ - Command: cmdPut, + res, err := c.send(c.ctx, &cacheprog.ProgRequest{ + Command: cacheprog.CmdPut, ActionID: a[:], OutputID: out[:], ObjectID: deprecatedValue, // TODO(bradfitz): remove in Go 1.25 @@ -473,8 +346,8 @@ func (c *ProgCache) Close() error { // First write a "close" message to the child so it can exit nicely // and clean up if it wants. Only after that exchange do we cancel // the context that kills the process. - if c.can[cmdClose] { - _, err = c.send(c.ctx, &ProgRequest{Command: cmdClose}) + if c.can[cacheprog.CmdClose] { + _, err = c.send(c.ctx, &cacheprog.ProgRequest{Command: cacheprog.CmdClose}) } // Cancel the context, which will close the helper's stdin. c.ctxCancel() diff --git a/src/cmd/go/internal/cacheprog/cacheprog.go b/src/cmd/go/internal/cacheprog/cacheprog.go new file mode 100644 index 00000000000000..41b1b0d79f94fe --- /dev/null +++ b/src/cmd/go/internal/cacheprog/cacheprog.go @@ -0,0 +1,137 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cacheprog defines the protocol for a GOCACHEPROG program. +// +// By default, the go command manages a build cache stored in the file system +// itself. GOCACHEPROG can be set to the name of a command (with optional +// space-separated flags) that implements the go command build cache externally. +// This permits defining a different cache policy. +// +// The go command will start the GOCACHEPROG as a subprocess and communicate +// with it via JSON messages over stdin/stdout. The subprocess's stderr will be +// connected to the go command's stderr. +// +// The subprocess should immediately send a [ProgResponse] with its capabilities. +// After that, the go command will send a stream of [ProgRequest] messages and the +// subprocess should reply to each [ProgRequest] with a [ProgResponse] message. +package cacheprog + +import ( + "io" + "time" +) + +// ProgCmd is a command that can be issued to a child process. +// +// If the interface needs to grow, the go command can add new commands or new +// versioned commands like "get2" in the future. The initial [ProgResponse] from +// the child process indicates which commands it supports. +type ProgCmd string + +const ( + // CmdPut tells the cache program to store an object in the cache. + // + // [ProgRequest.ActionID] is the cache key of this object. The cache should + // store [ProgRequest.OutputID] and [ProgRequest.Body] under this key for a + // later "get" request. It must also store the Body in a file in the local + // file system and return the path to that file in [ProgResponse.DiskPath], + // which must exist at least until a "close" request. + CmdPut = ProgCmd("put") + + // CmdGet tells the cache program to retrieve an object from the cache. + // + // [ProgRequest.ActionID] specifies the key of the object to get. If the + // cache does not contain this object, it should set [ProgResponse.Miss] to + // true. Otherwise, it should populate the fields of [ProgResponse], + // including setting [ProgResponse.OutputID] to the OutputID of the original + // "put" request and [ProgResponse.DiskPath] to the path of a local file + // containing the Body of the original "put" request. That file must + // continue to exist at least until a "close" request. + CmdGet = ProgCmd("get") + + // CmdClose requests that the cache program exit gracefully. + // + // The cache program should reply to this request and then exit + // (thus closing its stdout). + CmdClose = ProgCmd("close") +) + +// ProgRequest is the JSON-encoded message that's sent from the go command to +// the GOCACHEPROG child process over stdin. Each JSON object is on its own +// line. A ProgRequest of Type "put" with BodySize > 0 will be followed by a +// line containing a base64-encoded JSON string literal of the body. +type ProgRequest struct { + // ID is a unique number per process across all requests. + // It must be echoed in the ProgResponse from the child. + ID int64 + + // Command is the type of request. + // The go command will only send commands that were declared + // as supported by the child. + Command ProgCmd + + // ActionID is the cache key for "put" and "get" requests. + ActionID []byte `json:",omitempty"` // or nil if not used + + // OutputID is stored with the body for "put" requests. + // + // Prior to Go 1.24, when GOCACHEPROG was still an experiment, this was + // accidentally named ObjectID. It was renamed to OutputID in Go 1.24. + OutputID []byte `json:",omitempty"` // or nil if not used + + // Body is the body for "put" requests. It's sent after the JSON object + // as a base64-encoded JSON string when BodySize is non-zero. + // It's sent as a separate JSON value instead of being a struct field + // send in this JSON object so large values can be streamed in both directions. + // The base64 string body of a ProgRequest will always be written + // immediately after the JSON object and a newline. + Body io.Reader `json:"-"` + + // BodySize is the number of bytes of Body. If zero, the body isn't written. + BodySize int64 `json:",omitempty"` + + // ObjectID is the accidental spelling of OutputID that was used prior to Go + // 1.24. + // + // Deprecated: use OutputID. This field is only populated temporarily for + // backwards compatibility with Go 1.23 and earlier when + // GOEXPERIMENT=gocacheprog is set. It will be removed in Go 1.25. + ObjectID []byte `json:",omitempty"` +} + +// ProgResponse is the JSON response from the child process to the go command. +// +// With the exception of the first protocol message that the child writes to its +// stdout with ID==0 and KnownCommands populated, these are only sent in +// response to a ProgRequest from the go command. +// +// ProgResponses can be sent in any order. The ID must match the request they're +// replying to. +type ProgResponse struct { + ID int64 // that corresponds to ProgRequest; they can be answered out of order + Err string `json:",omitempty"` // if non-empty, the error + + // KnownCommands is included in the first message that cache helper program + // writes to stdout on startup (with ID==0). It includes the + // ProgRequest.Command types that are supported by the program. + // + // This lets the go command extend the protocol gracefully over time (adding + // "get2", etc), or fail gracefully when needed. It also lets the go command + // verify the program wants to be a cache helper. + KnownCommands []ProgCmd `json:",omitempty"` + + // For "get" requests. + + Miss bool `json:",omitempty"` // cache miss + OutputID []byte `json:",omitempty"` // the ObjectID stored with the body + Size int64 `json:",omitempty"` // body size in bytes + Time *time.Time `json:",omitempty"` // when the object was put in the cache (optional; used for cache expiration) + + // For "get" and "put" requests. + + // DiskPath is the absolute path on disk of the body corresponding to a + // "get" (on cache hit) or "put" request's ActionID. + DiskPath string `json:",omitempty"` +}