Skip to content

Commit

Permalink
feat: Add UpdateTimestamps option
Browse files Browse the repository at this point in the history
  • Loading branch information
bakjos committed Jul 4, 2022
1 parent 0b18808 commit c11b8f1
Show file tree
Hide file tree
Showing 6 changed files with 68 additions and 36 deletions.
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -309,10 +309,13 @@ OPTIONS:
--azblob.prefix value The Azure blob storage object prefix to use when
using azblob proxy backend. [$BAZEL_REMOTE_AZBLOB_PREFIX]
--azblob.update_timestamps Whether to update timestamps of object on cache
hit. (default: false) [$BAZEL_REMOTE_AZBLOB_UPDATE_TIMESTAMPS]
--azblob.auth_method value The Azure blob storage authentication method.
This argument is required when an azblob proxy backend is used. Allowed
values: client_certificate, client_secret, environment_credential,
default. [$BAZEL_REMOTE_AZBLOB_AUTH_METHOD]
shared_key, default. [$BAZEL_REMOTE_AZBLOB_AUTH_METHOD]
--azblob.shared_key value The Azure blob storage account access key to use
when using azblob proxy backend. Applies to AzBlob auth method(s):
Expand Down
54 changes: 37 additions & 17 deletions cache/azblobproxy/azblobproxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"log"
"path"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
Expand Down Expand Up @@ -35,15 +36,16 @@ type uploadReq struct {
}

type azBlobCache struct {
containerClient *azblob.ContainerClient
storageAccount string
container string
prefix string
v2mode bool
uploadQueue chan<- uploadReq
accessLogger cache.Logger
errorLogger cache.Logger
objectKey func(hash string, kind cache.EntryKind) string
containerClient *azblob.ContainerClient
storageAccount string
container string
prefix string
v2mode bool
uploadQueue chan<- uploadReq
accessLogger cache.Logger
errorLogger cache.Logger
objectKey func(hash string, kind cache.EntryKind) string
updateTimestamps bool
}

func (c *azBlobCache) Put(ctx context.Context, kind cache.EntryKind, hash string, size int64, rc io.ReadCloser) {
Expand Down Expand Up @@ -87,6 +89,10 @@ func (c *azBlobCache) Get(ctx context.Context, kind cache.EntryKind, hash string
}
cacheHits.Inc()

if c.updateTimestamps {
c.UpdateModificationTimestamp(ctx, key)
}

logResponse(c.accessLogger, "DOWNLOAD", c.storageAccount, c.container, key, err)

if kind == cache.CAS && c.v2mode {
Expand Down Expand Up @@ -137,6 +143,7 @@ func New(
prefix string,
creds azcore.TokenCredential,
sharedKey string,
UpdateTimestamps bool,
storageMode string, accessLogger cache.Logger,
errorLogger cache.Logger, numUploaders, maxQueuedUploads int,
) cache.Proxy {
Expand All @@ -149,7 +156,7 @@ func New(
if e != nil {
log.Fatalln(e)
}
serviceClient, e = azblob.NewServiceClientWithSharedKey(url, cred, nil)
serviceClient, err = azblob.NewServiceClientWithSharedKey(url, cred, nil)

} else {
serviceClient, err = azblob.NewServiceClient(url, creds, nil)
Expand All @@ -170,13 +177,14 @@ func New(
}

c := &azBlobCache{
containerClient: containerClient,
prefix: prefix,
storageAccount: storageAccount,
container: containerName,
accessLogger: accessLogger,
errorLogger: errorLogger,
v2mode: storageMode == "zstd",
containerClient: containerClient,
prefix: prefix,
storageAccount: storageAccount,
container: containerName,
accessLogger: accessLogger,
errorLogger: errorLogger,
v2mode: storageMode == "zstd",
updateTimestamps: UpdateTimestamps,
}

if c.v2mode {
Expand Down Expand Up @@ -223,6 +231,18 @@ func (c *azBlobCache) uploadFile(item uploadReq) {
logResponse(c.accessLogger, "UPLOAD", c.storageAccount, c.container, key, err)
}

func (c *azBlobCache) UpdateModificationTimestamp(ctx context.Context, key string) {
client, err := c.containerClient.NewBlockBlobClient(key)
if err != nil {
logResponse(c.accessLogger, "UPDATE_TIMESTAMPS", c.storageAccount, c.container, key, err)
}
metadata := map[string]string{
"LastModified": time.Now().Format("1/2/2006, 03:05 PM"),
}
_, err = client.SetMetadata(ctx, metadata, nil)
logResponse(c.accessLogger, "UPDATE_TIMESTAMPS", c.storageAccount, c.container, key, err)
}

func objectKeyV2(prefix string, hash string, kind cache.EntryKind) string {
var baseKey string
if kind == cache.CAS {
Expand Down
19 changes: 10 additions & 9 deletions config/azblob.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,16 @@ import (
)

type AzBlobStorageConfig struct {
StorageAccount string `yaml:"storage_account"`
ContainerName string `yaml:"container_name"`
Prefix string `yaml:"prefix"`
AuthMethod string `yaml:"auth_method"`
TenantID string `yaml:"tenant_id"`
ClientID string `yaml:"client_id"`
ClientSecret string `yaml:"client_secret"`
CertPath string `yaml:"cert_path"`
SharedKey string `yaml:"shared_key"`
StorageAccount string `yaml:"storage_account"`
ContainerName string `yaml:"container_name"`
Prefix string `yaml:"prefix"`
AuthMethod string `yaml:"auth_method"`
TenantID string `yaml:"tenant_id"`
ClientID string `yaml:"client_id"`
ClientSecret string `yaml:"client_secret"`
CertPath string `yaml:"cert_path"`
SharedKey string `yaml:"shared_key"`
UpdateTimestamps bool `yaml:"update_timestamps"`
}

func (azblobc AzBlobStorageConfig) GetCredentials() (azcore.TokenCredential, error) {
Expand Down
19 changes: 10 additions & 9 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -463,15 +463,16 @@ func get(ctx *cli.Context) (*Config, error) {
var azblob *AzBlobStorageConfig
if ctx.String("azblob.tenant_id") != "" {
azblob = &AzBlobStorageConfig{
TenantID: ctx.String("azblob.tenant_id"),
StorageAccount: ctx.String("azblob.storage_account"),
ContainerName: ctx.String("azblob.container_name"),
Prefix: ctx.String("azblob.prefix"),
AuthMethod: ctx.String("azblob.auth_method"),
ClientID: ctx.String("azblob.client_id"),
ClientSecret: ctx.String("azblob.client_secret"),
CertPath: ctx.String("azblob.cert_path"),
SharedKey: ctx.String("azblob.shared_key"),
TenantID: ctx.String("azblob.tenant_id"),
StorageAccount: ctx.String("azblob.storage_account"),
ContainerName: ctx.String("azblob.container_name"),
Prefix: ctx.String("azblob.prefix"),
AuthMethod: ctx.String("azblob.auth_method"),
ClientID: ctx.String("azblob.client_id"),
ClientSecret: ctx.String("azblob.client_secret"),
CertPath: ctx.String("azblob.cert_path"),
SharedKey: ctx.String("azblob.shared_key"),
UpdateTimestamps: ctx.Bool("azblob.update_timestamps"),
}
}

Expand Down
1 change: 1 addition & 0 deletions config/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ func (c *Config) setProxy() error {
c.AzBlobConfig.Prefix,
creds,
c.AzBlobConfig.SharedKey,
c.AzBlobConfig.UpdateTimestamps,
c.StorageMode, c.AccessLogger, c.ErrorLogger, c.NumUploaders, c.MaxQueuedUploads,
)
return nil
Expand Down
6 changes: 6 additions & 0 deletions utils/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,12 @@ func GetCliFlags() []cli.Flag {
Usage: "The Azure blob storage object prefix to use when using azblob proxy backend.",
EnvVars: []string{"BAZEL_REMOTE_AZBLOB_PREFIX"},
},
&cli.BoolFlag{
Name: "azblob.update_timestamps",
Usage: "Whether to update timestamps of object on cache hit.",
DefaultText: "false",
EnvVars: []string{"BAZEL_REMOTE_AZBLOB_UPDATE_TIMESTAMPS"},
},
&cli.StringFlag{
Name: "azblob.auth_method",
Value: "",
Expand Down

0 comments on commit c11b8f1

Please sign in to comment.