Add support for more digest functions

cache/BUILD.bazel

@@ -5,4 +5,5 @@ go_library(
     srcs = ["cache.go"],
     importpath = "github.com/buchgr/bazel-remote/v2/cache",
     visibility = ["//visibility:public"],
+    deps = ["//cache/hashing:go_default_library"],
 )

cache/azblobproxy/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     deps = [
         "//cache:go_default_library",
         "//cache/disk/casblob:go_default_library",
+        "//cache/hashing:go_default_library",
         "//utils/backendproxy:go_default_library",
         "@com_github_azure_azure_sdk_for_go_sdk_azcore//:go_default_library",
         "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:go_default_library",

cache/azblobproxy/azblobproxy.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/buchgr/bazel-remote/v2/cache"
 	"github.com/buchgr/bazel-remote/v2/cache/disk/casblob"
+	"github.com/buchgr/bazel-remote/v2/cache/hashing"
 	"github.com/buchgr/bazel-remote/v2/utils/backendproxy"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
@@ -40,18 +41,19 @@ type azBlobCache struct {
 	uploadQueue      chan<- backendproxy.UploadReq
 	accessLogger     cache.Logger
 	errorLogger      cache.Logger
-	objectKey        func(hash string, kind cache.EntryKind) string
+	objectKey        func(hasher hashing.Hasher, hash string, kind cache.EntryKind) string
 	updateTimestamps bool
 }
 
-func (c *azBlobCache) Put(ctx context.Context, kind cache.EntryKind, hash string, logicalSize int64, sizeOnDisk int64, rc io.ReadCloser) {
+func (c *azBlobCache) Put(ctx context.Context, kind cache.EntryKind, hasher hashing.Hasher, hash string, logicalSize int64, sizeOnDisk int64, rc io.ReadCloser) {
 	if c.uploadQueue == nil {
 		rc.Close()
 		return
 	}
 
 	select {
 	case c.uploadQueue <- backendproxy.UploadReq{
+		Hasher:      hasher,
 		Hash:        hash,
 		LogicalSize: logicalSize,
 		SizeOnDisk:  sizeOnDisk,
@@ -64,8 +66,8 @@ func (c *azBlobCache) Put(ctx context.Context, kind cache.EntryKind, hash string
 	}
 }
 
-func (c *azBlobCache) Get(ctx context.Context, kind cache.EntryKind, hash string, _ int64) (rc io.ReadCloser, size int64, err error) {
-	key := c.objectKey(hash, kind)
+func (c *azBlobCache) Get(ctx context.Context, kind cache.EntryKind, hasher hashing.Hasher, hash string, _ int64) (rc io.ReadCloser, size int64, err error) {
+	key := c.objectKey(hasher, hash, kind)
 	if c.prefix != "" {
 		key = c.prefix + "/" + key
 	}
@@ -111,8 +113,8 @@ func (c *azBlobCache) Get(ctx context.Context, kind cache.EntryKind, hash string
 
 var errNotFound = errors.New("NOT FOUND")
 
-func (c *azBlobCache) Contains(ctx context.Context, kind cache.EntryKind, hash string, _ int64) (bool, int64) {
-	key := c.objectKey(hash, kind)
+func (c *azBlobCache) Contains(ctx context.Context, kind cache.EntryKind, hasher hashing.Hasher, hash string, _ int64) (bool, int64) {
+	key := c.objectKey(hasher, hash, kind)
 	if c.prefix != "" {
 		key = c.prefix + "/" + key
 	}
@@ -193,12 +195,12 @@ func New(
 	}
 
 	if c.v2mode {
-		c.objectKey = func(hash string, kind cache.EntryKind) string {
-			return objectKeyV2(c.prefix, hash, kind)
+		c.objectKey = func(hasher hashing.Hasher, hash string, kind cache.EntryKind) string {
+			return objectKeyV2(c.prefix, hasher, hash, kind)
 		}
 	} else {
-		c.objectKey = func(hash string, kind cache.EntryKind) string {
-			return objectKeyV1(c.prefix, hash, kind)
+		c.objectKey = func(hasher hashing.Hasher, hash string, kind cache.EntryKind) string {
+			return objectKeyV1(c.prefix, hasher, hash, kind)
 		}
 	}
 
@@ -210,7 +212,7 @@ func New(
 func (c *azBlobCache) UploadFile(item backendproxy.UploadReq) {
 	defer item.Rc.Close()
 
-	key := c.objectKey(item.Hash, item.Kind)
+	key := c.objectKey(item.Hasher, item.Hash, item.Kind)
 	if c.prefix != "" {
 		key = c.prefix + "/" + key
 	}
@@ -237,28 +239,17 @@ func (c *azBlobCache) UpdateModificationTimestamp(ctx context.Context, key strin
 	logResponse(c.accessLogger, "UPDATE_TIMESTAMPS", c.storageAccount, c.container, key, err)
 }
 
-func objectKeyV2(prefix string, hash string, kind cache.EntryKind) string {
-	var baseKey string
+func objectKeyV2(prefix string, hasher hashing.Hasher, hash string, kind cache.EntryKind) string {
+	baseFolder := kind.String()
 	if kind == cache.CAS {
 		// Use "cas.v2" to distinguish new from old format blobs.
-		baseKey = path.Join("cas.v2", hash[:2], hash)
-	} else {
-		baseKey = path.Join(kind.String(), hash[:2], hash)
-	}
-
-	if prefix == "" {
-		return baseKey
+		baseFolder = "cas.v2"
 	}
-
-	return path.Join(prefix, baseKey)
+	return path.Join(prefix, baseFolder, hasher.Dir(), hash[:2], hash)
 }
 
-func objectKeyV1(prefix string, hash string, kind cache.EntryKind) string {
-	if prefix == "" {
-		return path.Join(kind.String(), hash[:2], hash)
-	}
-
-	return path.Join(prefix, kind.String(), hash[:2], hash)
+func objectKeyV1(prefix string, hasher hashing.Hasher, hash string, kind cache.EntryKind) string {
+	return path.Join(prefix, kind.String(), hasher.Dir(), hash[:2], hash)
 }
 
 // Helper function for logging responses

cache/cache.go

@@ -2,9 +2,11 @@ package cache
 
 import (
 	"context"
-	"crypto/sha256"
 	"encoding/hex"
 	"io"
+	"path"
+
+	"github.com/buchgr/bazel-remote/v2/cache/hashing"
 )
 
 // EntryKind describes the kind of cache entry
@@ -71,29 +73,29 @@ type Proxy interface {
 	// format as used by the disk.Cache instance.
 	//
 	// This is allowed to fail silently (for example when under heavy load).
-	Put(ctx context.Context, kind EntryKind, hash string, logicalSize int64, sizeOnDisk int64, rc io.ReadCloser)
+	Put(ctx context.Context, kind EntryKind, hasher hashing.Hasher, hash string, logicalSize int64, sizeOnDisk int64, rc io.ReadCloser)
 
 	// Get returns an io.ReadCloser from which the cache item identified by
 	// `hash` can be read, its logical size, and an error if something went
 	// wrong. The data available from `rc` is in the same format as used by
 	// the disk.Cache instance.
-	Get(ctx context.Context, kind EntryKind, hash string, size int64) (io.ReadCloser, int64, error)
+	Get(ctx context.Context, kind EntryKind, hasher hashing.Hasher, hash string, size int64) (io.ReadCloser, int64, error)
 
 	// Contains returns whether or not the cache item exists on the
 	// remote end, and the size if it exists (and -1 if the size is
 	// unknown).
-	Contains(ctx context.Context, kind EntryKind, hash string, size int64) (bool, int64)
+	Contains(ctx context.Context, kind EntryKind, hasher hashing.Hasher, hash string, size int64) (bool, int64)
 }
 
 // TransformActionCacheKey takes an ActionCache key and an instance name
 // and returns a new ActionCache key to use instead. If the instance name
 // is empty, then the original key is returned unchanged.
-func TransformActionCacheKey(key, instance string, logger Logger) string {
+func TransformActionCacheKey(hasher hashing.Hasher, key, instance string, logger Logger) string {
 	if instance == "" {
 		return key
 	}
 
-	h := sha256.New()
+	h := hasher.New()
 	h.Write([]byte(key))
 	h.Write([]byte(instance))
 	b := h.Sum(nil)
@@ -104,6 +106,6 @@ func TransformActionCacheKey(key, instance string, logger Logger) string {
 	return newKey
 }
 
-func LookupKey(kind EntryKind, hash string) string {
-	return kind.String() + "/" + hash
+func LookupKey(kind EntryKind, hasher hashing.Hasher, hash string) string {
+	return path.Join(kind.String(), hasher.Dir(), hash)
 }

cache/disk/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "//cache:go_default_library",
         "//cache/disk/casblob:go_default_library",
         "//cache/disk/zstdimpl:go_default_library",
+        "//cache/hashing:go_default_library",
         "//genproto/build/bazel/remote/execution/v2:go_default_library",
         "//utils/annotate:go_default_library",
         "//utils/tempfile:go_default_library",
@@ -42,6 +43,7 @@ go_test(
         "//cache:go_default_library",
         "//cache/disk/casblob:go_default_library",
         "//cache/disk/zstdimpl:go_default_library",
+        "//cache/hashing:go_default_library",
         "//cache/httpproxy:go_default_library",
         "//genproto/build/bazel/remote/execution/v2:go_default_library",
         "//utils:go_default_library",

cache/disk/casblob/BUILD.bazel

@@ -5,7 +5,10 @@ go_library(
     srcs = ["casblob.go"],
     importpath = "github.com/buchgr/bazel-remote/v2/cache/disk/casblob",
     visibility = ["//visibility:public"],
-    deps = ["//cache/disk/zstdimpl:go_default_library"],
+    deps = [
+        "//cache/disk/zstdimpl:go_default_library",
+        "//cache/hashing:go_default_library",
+    ],
 )
 
 go_test(
@@ -14,6 +17,7 @@ go_test(
     deps = [
         ":go_default_library",
         "//cache/disk/zstdimpl:go_default_library",
+        "//cache/hashing:go_default_library",
         "//utils:go_default_library",
     ],
 )

cache/disk/casblob/casblob.go

@@ -2,7 +2,6 @@ package casblob
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
@@ -12,6 +11,7 @@ import (
 	"os"
 
 	"github.com/buchgr/bazel-remote/v2/cache/disk/zstdimpl"
+	"github.com/buchgr/bazel-remote/v2/cache/hashing"
 )
 
 type CompressionType uint8
@@ -510,7 +510,7 @@ func (b *readCloserWrapper) Close() error {
 
 // Read from r and write to f, using CompressionType t.
 // Return the size on disk or an error if something went wrong.
-func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t CompressionType, hash string, size int64) (int64, error) {
+func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t CompressionType, hh hashing.Hasher, hash string, size int64) (int64, error) {
 	var err error
 	defer f.Close()
 
@@ -550,7 +550,7 @@ func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t Compressio
 	var n int64
 
 	if t == Identity {
-		hasher := sha256.New()
+		hasher := hh.New()
 
 		n, err = io.Copy(io.MultiWriter(f, hasher), r)
 		if err != nil {
@@ -579,7 +579,7 @@ func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t Compressio
 
 	uncompressedChunk := make([]byte, chunkSize)
 
-	hasher := sha256.New()
+	hasher := hh.New()
 
 	for nextChunk < len(h.chunkOffsets)-1 {
 		h.chunkOffsets[nextChunk] = fileOffset

cache/disk/casblob/casblob_test.go

@@ -2,8 +2,6 @@ package casblob_test
 
 import (
 	"bytes"
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
@@ -12,6 +10,7 @@ import (
 
 	"github.com/buchgr/bazel-remote/v2/cache/disk/casblob"
 	"github.com/buchgr/bazel-remote/v2/cache/disk/zstdimpl"
+	"github.com/buchgr/bazel-remote/v2/cache/hashing"
 	testutils "github.com/buchgr/bazel-remote/v2/utils"
 )
 
@@ -35,7 +34,7 @@ func TestZstdFromLegacy(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	data, hash := testutils.RandomDataAndHash(int64(size))
+	data, hash := testutils.RandomDataAndHash(int64(size), hashing.DefaultHasher)
 	dir := testutils.TempDir(t)
 	filename := fmt.Sprintf("%s/%s", dir, hash)
 	file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0664)
@@ -73,8 +72,7 @@ func TestZstdFromLegacy(t *testing.T) {
 		t.Fatalf("Unexpected buf size %d, expected %d", buf.Len(), size)
 	}
 
-	h := sha256.Sum256(data)
-	hs := hex.EncodeToString(h[:])
+	hs := hashing.DefaultHasher.Hash(data)
 	if hs != hash {
 		t.Fatalf("Unexpected content sha %s, expected %s", hs, hash)
 	}