From a03dad613ef224e2f761783396188ac43031ae7c Mon Sep 17 00:00:00 2001 From: Pedro Saraiva Date: Thu, 3 Jun 2021 21:27:40 +0100 Subject: [PATCH 1/5] fix(azure): append method on azure backend --- tempodb/backend/azure/azure.go | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tempodb/backend/azure/azure.go b/tempodb/backend/azure/azure.go index 000c898e960..8e1fc6b1250 100644 --- a/tempodb/backend/azure/azure.go +++ b/tempodb/backend/azure/azure.go @@ -4,6 +4,8 @@ import ( "bufio" "bytes" "context" + "encoding/base64" + "encoding/binary" "encoding/json" "io" "strings" @@ -223,14 +225,37 @@ func (rw *readerWriter) writeAll(ctx context.Context, name string, b []byte) err } func (rw *readerWriter) append(ctx context.Context, src []byte, name string) (string, error) { - appendBlobURL := rw.containerURL.NewAppendBlobURL(name) + appendBlobURL := rw.containerURL.NewBlockBlobURL(name) + + // These helper functions convert a binary block ID to a base-64 string and vice versa + // NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length + blockIDBinaryToBase64 := func(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) } + l, err := appendBlobURL.GetBlockList(ctx, blob.BlockListAll, blob.LeaseAccessConditions{}) + lenth := len(l.CommittedBlocks[0].Name) + blockIDIntToBase64 := func(blockID int) string { + binaryBlockID := (&[64]byte{})[:] // All block IDs are 4 bytes long + binary.LittleEndian.PutUint32(binaryBlockID, uint32(blockID)) + return blockIDBinaryToBase64(binaryBlockID) + } + + _, err = appendBlobURL.StageBlock(ctx, blockIDIntToBase64(lenth+1), bytes.NewReader(src), blob.LeaseAccessConditions{}, nil) + if err != nil { + return "", err + } - resp, err := appendBlobURL.AppendBlock(ctx, bytes.NewReader(src), blob.AppendBlobAccessConditions{}, nil) + base64BlockIDs := make([]string, len(l.CommittedBlocks)+1) + for i := 0; i < len(l.CommittedBlocks); i++ { + base64BlockIDs[i] = l.CommittedBlocks[i].Name + } + + base64BlockIDs[len(l.CommittedBlocks)] = blockIDIntToBase64(lenth + 1) + // After all the blocks are uploaded, atomically commit them to the blob. + _, err = appendBlobURL.CommitBlockList(ctx, base64BlockIDs, blob.BlobHTTPHeaders{}, blob.Metadata{}, blob.BlobAccessConditions{}) if err != nil { return "", err } - return resp.RequestID(), nil + return "", nil } From 7d12a6ddaaab226e3df3bfadffe89128ffede915 Mon Sep 17 00:00:00 2001 From: Pedro Saraiva Date: Thu, 3 Jun 2021 22:07:06 +0100 Subject: [PATCH 2/5] fix: lint issues --- tempodb/backend/azure/azure.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tempodb/backend/azure/azure.go b/tempodb/backend/azure/azure.go index 8e1fc6b1250..6eff7816686 100644 --- a/tempodb/backend/azure/azure.go +++ b/tempodb/backend/azure/azure.go @@ -93,7 +93,7 @@ func (rw *readerWriter) Append(ctx context.Context, name string, blockID uuid.UU } else { a = tracker.(appendTracker) - _, err := rw.append(ctx, buffer, a.Name) + err := rw.append(ctx, buffer, a.Name) if err != nil { return nil, err } @@ -224,9 +224,9 @@ func (rw *readerWriter) writeAll(ctx context.Context, name string, b []byte) err return nil } -func (rw *readerWriter) append(ctx context.Context, src []byte, name string) (string, error) { +func (rw *readerWriter) append(ctx context.Context, src []byte, name string) error { appendBlobURL := rw.containerURL.NewBlockBlobURL(name) - + // These helper functions convert a binary block ID to a base-64 string and vice versa // NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length blockIDBinaryToBase64 := func(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) } @@ -240,23 +240,22 @@ func (rw *readerWriter) append(ctx context.Context, src []byte, name string) (st _, err = appendBlobURL.StageBlock(ctx, blockIDIntToBase64(lenth+1), bytes.NewReader(src), blob.LeaseAccessConditions{}, nil) if err != nil { - return "", err + return err } base64BlockIDs := make([]string, len(l.CommittedBlocks)+1) for i := 0; i < len(l.CommittedBlocks); i++ { base64BlockIDs[i] = l.CommittedBlocks[i].Name } - + base64BlockIDs[len(l.CommittedBlocks)] = blockIDIntToBase64(lenth + 1) // After all the blocks are uploaded, atomically commit them to the blob. _, err = appendBlobURL.CommitBlockList(ctx, base64BlockIDs, blob.BlobHTTPHeaders{}, blob.Metadata{}, blob.BlobAccessConditions{}) if err != nil { - return "", err + return err } - return "", nil - + return nil } func (rw *readerWriter) writer(ctx context.Context, src io.Reader, name string) error { From 7510c416a43d3c11cad6a771c1e3d023065ff111 Mon Sep 17 00:00:00 2001 From: Pedro Saraiva Date: Thu, 3 Jun 2021 22:07:06 +0100 Subject: [PATCH 3/5] fix: lint issues --- tempodb/backend/azure/azure.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tempodb/backend/azure/azure.go b/tempodb/backend/azure/azure.go index 6eff7816686..02a5ab2de5f 100644 --- a/tempodb/backend/azure/azure.go +++ b/tempodb/backend/azure/azure.go @@ -230,15 +230,22 @@ func (rw *readerWriter) append(ctx context.Context, src []byte, name string) err // These helper functions convert a binary block ID to a base-64 string and vice versa // NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length blockIDBinaryToBase64 := func(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) } - l, err := appendBlobURL.GetBlockList(ctx, blob.BlockListAll, blob.LeaseAccessConditions{}) - lenth := len(l.CommittedBlocks[0].Name) + blockIDIntToBase64 := func(blockID int) string { - binaryBlockID := (&[64]byte{})[:] // All block IDs are 4 bytes long + binaryBlockID := (&[64]byte{})[:] binary.LittleEndian.PutUint32(binaryBlockID, uint32(blockID)) return blockIDBinaryToBase64(binaryBlockID) } - _, err = appendBlobURL.StageBlock(ctx, blockIDIntToBase64(lenth+1), bytes.NewReader(src), blob.LeaseAccessConditions{}, nil) + l, err := appendBlobURL.GetBlockList(ctx, blob.BlockListAll, blob.LeaseAccessConditions{}) + if err != nil { + return err + } + + // generate the next block id + id := blockIDIntToBase64(len(l.CommittedBlocks) + 1) + + _, err = appendBlobURL.StageBlock(ctx, id, bytes.NewReader(src), blob.LeaseAccessConditions{}, nil) if err != nil { return err } @@ -248,7 +255,7 @@ func (rw *readerWriter) append(ctx context.Context, src []byte, name string) err base64BlockIDs[i] = l.CommittedBlocks[i].Name } - base64BlockIDs[len(l.CommittedBlocks)] = blockIDIntToBase64(lenth + 1) + base64BlockIDs[len(l.CommittedBlocks)] = id // After all the blocks are uploaded, atomically commit them to the blob. _, err = appendBlobURL.CommitBlockList(ctx, base64BlockIDs, blob.BlobHTTPHeaders{}, blob.Metadata{}, blob.BlobAccessConditions{}) From 68dc400d34bff20ab52689a239a98ba02d537868 Mon Sep 17 00:00:00 2001 From: Pedro Saraiva Date: Fri, 4 Jun 2021 00:44:22 +0100 Subject: [PATCH 4/5] chore: update ChangeLog.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1526f444512..1c2268d4ec6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * [BUGFIX] Fix Query Frontend grpc settings to avoid noisy error log. [#690](https://github.com/grafana/tempo/pull/690) * [BUGFIX] Zipkin Support - CombineTraces. [#688](https://github.com/grafana/tempo/pull/688) * [BUGFIX] Zipkin support - Dedupe span IDs based on span.Kind (client/server) in Query Frontend. [#687](https://github.com/grafana/tempo/pull/687) +* [BUGFIX] Azure Backend - Fix an issue with the append method on the Azure backend. [#736](https://github.com/grafana/tempo/pull/736) ## v0.7.0 From 054d37cbd3991e6d5e4653b9de9cf5c871249026 Mon Sep 17 00:00:00 2001 From: Pedro Saraiva Date: Fri, 4 Jun 2021 02:24:59 +0100 Subject: [PATCH 5/5] chore: update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c2268d4ec6..24b664fb15f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## main / unreleased +* [BUGFIX] Azure Backend - Fix an issue with the append method on the Azure backend. [#736](https://github.com/grafana/tempo/pull/736) + ## v1.0.0-rc.0 * [ENHANCEMENT] Performance: Improve Ingester Record Insertion. [#681](https://github.com/grafana/tempo/pull/681) @@ -15,7 +17,6 @@ * [BUGFIX] Fix Query Frontend grpc settings to avoid noisy error log. [#690](https://github.com/grafana/tempo/pull/690) * [BUGFIX] Zipkin Support - CombineTraces. [#688](https://github.com/grafana/tempo/pull/688) * [BUGFIX] Zipkin support - Dedupe span IDs based on span.Kind (client/server) in Query Frontend. [#687](https://github.com/grafana/tempo/pull/687) -* [BUGFIX] Azure Backend - Fix an issue with the append method on the Azure backend. [#736](https://github.com/grafana/tempo/pull/736) ## v0.7.0