diff --git a/README.md b/README.md index 2950350ba..6ab4da002 100644 --- a/README.md +++ b/README.md @@ -309,10 +309,13 @@ OPTIONS: --azblob.prefix value The Azure blob storage object prefix to use when using azblob proxy backend. [$BAZEL_REMOTE_AZBLOB_PREFIX] + --azblob.update_timestamps Whether to update timestamps of object on cache + hit. (default: false) [$BAZEL_REMOTE_AZBLOB_UPDATE_TIMESTAMPS] + --azblob.auth_method value The Azure blob storage authentication method. This argument is required when an azblob proxy backend is used. Allowed values: client_certificate, client_secret, environment_credential, - default. [$BAZEL_REMOTE_AZBLOB_AUTH_METHOD] + shared_key, default. [$BAZEL_REMOTE_AZBLOB_AUTH_METHOD] --azblob.shared_key value The Azure blob storage account access key to use when using azblob proxy backend. Applies to AzBlob auth method(s): diff --git a/cache/azblobproxy/azblobproxy.go b/cache/azblobproxy/azblobproxy.go index 7b6c80375..ef6cf8c2c 100644 --- a/cache/azblobproxy/azblobproxy.go +++ b/cache/azblobproxy/azblobproxy.go @@ -7,6 +7,7 @@ import ( "io" "log" "path" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" @@ -35,15 +36,16 @@ type uploadReq struct { } type azBlobCache struct { - containerClient *azblob.ContainerClient - storageAccount string - container string - prefix string - v2mode bool - uploadQueue chan<- uploadReq - accessLogger cache.Logger - errorLogger cache.Logger - objectKey func(hash string, kind cache.EntryKind) string + containerClient *azblob.ContainerClient + storageAccount string + container string + prefix string + v2mode bool + uploadQueue chan<- uploadReq + accessLogger cache.Logger + errorLogger cache.Logger + objectKey func(hash string, kind cache.EntryKind) string + updateTimestamps bool } func (c *azBlobCache) Put(ctx context.Context, kind cache.EntryKind, hash string, size int64, rc io.ReadCloser) { @@ -87,6 +89,10 @@ func (c *azBlobCache) Get(ctx context.Context, kind cache.EntryKind, hash string } cacheHits.Inc() + if c.updateTimestamps { + c.UpdateModificationTimestamp(ctx, key) + } + logResponse(c.accessLogger, "DOWNLOAD", c.storageAccount, c.container, key, err) if kind == cache.CAS && c.v2mode { @@ -137,6 +143,7 @@ func New( prefix string, creds azcore.TokenCredential, sharedKey string, + UpdateTimestamps bool, storageMode string, accessLogger cache.Logger, errorLogger cache.Logger, numUploaders, maxQueuedUploads int, ) cache.Proxy { @@ -170,13 +177,14 @@ func New( } c := &azBlobCache{ - containerClient: containerClient, - prefix: prefix, - storageAccount: storageAccount, - container: containerName, - accessLogger: accessLogger, - errorLogger: errorLogger, - v2mode: storageMode == "zstd", + containerClient: containerClient, + prefix: prefix, + storageAccount: storageAccount, + container: containerName, + accessLogger: accessLogger, + errorLogger: errorLogger, + v2mode: storageMode == "zstd", + updateTimestamps: UpdateTimestamps, } if c.v2mode { @@ -223,6 +231,18 @@ func (c *azBlobCache) uploadFile(item uploadReq) { logResponse(c.accessLogger, "UPLOAD", c.storageAccount, c.container, key, err) } +func (c *azBlobCache) UpdateModificationTimestamp(ctx context.Context, key string) { + client, err := c.containerClient.NewBlockBlobClient(key) + if err != nil { + logResponse(c.accessLogger, "UPDATE_TIMESTAMPS", c.storageAccount, c.container, key, err) + } + metadata := map[string]string{ + "LastModified": time.Now().Format("1/2/2006, 03:05 PM"), + } + _, err = client.SetMetadata(ctx, metadata, nil) + logResponse(c.accessLogger, "UPDATE_TIMESTAMPS", c.storageAccount, c.container, key, err) +} + func objectKeyV2(prefix string, hash string, kind cache.EntryKind) string { var baseKey string if kind == cache.CAS { diff --git a/config/azblob.go b/config/azblob.go index 39b26783d..fa23734e2 100644 --- a/config/azblob.go +++ b/config/azblob.go @@ -11,15 +11,16 @@ import ( ) type AzBlobStorageConfig struct { - StorageAccount string `yaml:"storage_account"` - ContainerName string `yaml:"container_name"` - Prefix string `yaml:"prefix"` - AuthMethod string `yaml:"auth_method"` - TenantID string `yaml:"tenant_id"` - ClientID string `yaml:"client_id"` - ClientSecret string `yaml:"client_secret"` - CertPath string `yaml:"cert_path"` - SharedKey string `yaml:"shared_key"` + StorageAccount string `yaml:"storage_account"` + ContainerName string `yaml:"container_name"` + Prefix string `yaml:"prefix"` + AuthMethod string `yaml:"auth_method"` + TenantID string `yaml:"tenant_id"` + ClientID string `yaml:"client_id"` + ClientSecret string `yaml:"client_secret"` + CertPath string `yaml:"cert_path"` + SharedKey string `yaml:"shared_key"` + UpdateTimestamps bool `yaml:"update_timestamps"` } func (azblobc AzBlobStorageConfig) GetCredentials() (azcore.TokenCredential, error) { diff --git a/config/config.go b/config/config.go index 91c3a876e..54fd0afcc 100644 --- a/config/config.go +++ b/config/config.go @@ -463,15 +463,16 @@ func get(ctx *cli.Context) (*Config, error) { var azblob *AzBlobStorageConfig if ctx.String("azblob.tenant_id") != "" { azblob = &AzBlobStorageConfig{ - TenantID: ctx.String("azblob.tenant_id"), - StorageAccount: ctx.String("azblob.storage_account"), - ContainerName: ctx.String("azblob.container_name"), - Prefix: ctx.String("azblob.prefix"), - AuthMethod: ctx.String("azblob.auth_method"), - ClientID: ctx.String("azblob.client_id"), - ClientSecret: ctx.String("azblob.client_secret"), - CertPath: ctx.String("azblob.cert_path"), - SharedKey: ctx.String("azblob.shared_key"), + TenantID: ctx.String("azblob.tenant_id"), + StorageAccount: ctx.String("azblob.storage_account"), + ContainerName: ctx.String("azblob.container_name"), + Prefix: ctx.String("azblob.prefix"), + AuthMethod: ctx.String("azblob.auth_method"), + ClientID: ctx.String("azblob.client_id"), + ClientSecret: ctx.String("azblob.client_secret"), + CertPath: ctx.String("azblob.cert_path"), + SharedKey: ctx.String("azblob.shared_key"), + UpdateTimestamps: ctx.Bool("azblob.update_timestamps"), } } diff --git a/config/proxy.go b/config/proxy.go index a83c66787..24945564b 100644 --- a/config/proxy.go +++ b/config/proxy.go @@ -70,6 +70,7 @@ func (c *Config) setProxy() error { c.AzBlobConfig.Prefix, creds, c.AzBlobConfig.SharedKey, + c.AzBlobConfig.UpdateTimestamps, c.StorageMode, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads, ) return nil diff --git a/utils/flags/flags.go b/utils/flags/flags.go index 3344a3fcf..166bd2f4f 100644 --- a/utils/flags/flags.go +++ b/utils/flags/flags.go @@ -308,6 +308,12 @@ func GetCliFlags() []cli.Flag { Usage: "The Azure blob storage object prefix to use when using azblob proxy backend.", EnvVars: []string{"BAZEL_REMOTE_AZBLOB_PREFIX"}, }, + &cli.BoolFlag{ + Name: "azblob.update_timestamps", + Usage: "Whether to update timestamps of object on cache hit.", + DefaultText: "false", + EnvVars: []string{"BAZEL_REMOTE_AZBLOB_UPDATE_TIMESTAMPS"}, + }, &cli.StringFlag{ Name: "azblob.auth_method", Value: "",