[cli] Add command to generate index/bloom from a data file

cmd/tempo-cli/cmd-gen-bloom-index.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/google/uuid"
+
+	"github.com/grafana/tempo/tempodb/backend"
+	"github.com/grafana/tempo/tempodb/encoding"
+	"github.com/grafana/tempo/tempodb/encoding/common"
+)
+
+type genIndexCmd struct {
+	TenantID string `arg:"" help:"tenant-id within the bucket"`
+	BlockID  string `arg:"" help:"block ID to list"`
+	backendOptions
+}
+
+func ReplayBlockAndGetRecords(meta *backend.BlockMeta, filepath string) ([]common.Record, error, error) {
+	v, err := encoding.FromVersion(meta.Version)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var warning error
+	// replay file to extract records
+	f, err := os.OpenFile(filepath, os.O_RDONLY, 0644)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	dataReader, err := v.NewDataReader(backend.NewContextReaderWithAllReader(f), meta.Encoding)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer dataReader.Close()
+
+	var buffer []byte
+	var records []common.Record
+	objectReader := v.NewObjectReaderWriter()
+	currentOffset := uint64(0)
+	for {
+		buffer, pageLen, err := dataReader.NextPage(buffer)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			warning = err
+			break
+		}
+
+		reader := bytes.NewReader(buffer)
+		id, _, err := objectReader.UnmarshalObjectFromReader(reader)
+		if err != nil {
+			warning = err
+			break
+		}
+
+		// make a copy so we don't hold onto the iterator buffer
+		recordID := append([]byte(nil), id...)
+		records = append(records, common.Record{
+			ID:     recordID,
+			Start:  currentOffset,
+			Length: pageLen,
+		})
+		currentOffset += uint64(pageLen)
+	}
+
+	common.SortRecords(records)
+
+	return records, warning, nil
+}
+
+func (cmd *genIndexCmd) Run(ctx *globalOptions) error {
+	blockID, err := uuid.Parse(cmd.BlockID)
+	if err != nil {
+		return err
+	}
+
+	r, w, _, err := loadBackend(&cmd.backendOptions, ctx)
+	if err != nil {
+		return err
+	}
+
+	meta, err := r.BlockMeta(context.TODO(), blockID, cmd.TenantID)
+	if err != nil {
+		return err
+	}
+
+	// replay file to extract records
+	records, warning, err := ReplayBlockAndGetRecords(meta, "./cmd/tempo-cli/test-data/" + cmd.TenantID + "/" + cmd.BlockID + "/data")
+	if warning != nil || err != nil {
+		fmt.Println("error replaying block", warning, err)
+		return nil
+	}
+
+	// write using IndexWriter
+	v, err := encoding.FromVersion(meta.Version)
+	if err != nil {
+		fmt.Println("error creating versioned encoding", err)
+	}
+
+	indexWriter := v.NewIndexWriter(int(meta.IndexPageSize))
+	indexBytes, err := indexWriter.Write(records)
+	if err != nil {
+		fmt.Println("error writing records to indexWriter", err)
+	}
+
+	// write to the local backend
+	err = w.Write(context.TODO(), "index", blockID, cmd.TenantID, indexBytes, false)
+	if err != nil {
+		fmt.Println("error writing index to backend", err)
+	}
+
+	fmt.Println("index written to backend successfully")
+
+	return nil
+}

cmd/tempo-cli/cmd-list-block.go

@@ -25,7 +25,7 @@ type listBlockCmd struct {
 }
 
 func (cmd *listBlockCmd) Run(ctx *globalOptions) error {
-	r, c, err := loadBackend(&cmd.backendOptions, ctx)
+	r, _, c, err := loadBackend(&cmd.backendOptions, ctx)
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/cmd-list-blocks.go

@@ -17,7 +17,7 @@ type listBlocksCmd struct {
 }
 
 func (l *listBlocksCmd) Run(ctx *globalOptions) error {
-	r, c, err := loadBackend(&l.backendOptions, ctx)
+	r, _, c, err := loadBackend(&l.backendOptions, ctx)
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/cmd-list-cachesummary.go

@@ -14,7 +14,7 @@ type listCacheSummaryCmd struct {
 }
 
 func (l *listCacheSummaryCmd) Run(ctx *globalOptions) error {
-	r, c, err := loadBackend(&l.backendOptions, ctx)
+	r, _, c, err := loadBackend(&l.backendOptions, ctx)
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/cmd-list-compactionsummary.go

@@ -17,7 +17,7 @@ type listCompactionSummaryCmd struct {
 }
 
 func (l *listCompactionSummaryCmd) Run(ctx *globalOptions) error {
-	r, c, err := loadBackend(&l.backendOptions, ctx)
+	r, _, c, err := loadBackend(&l.backendOptions, ctx)
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/cmd-list-index.go

@@ -23,7 +23,7 @@ func (cmd *listIndexCmd) Run(ctx *globalOptions) error {
 		return err
 	}
 
-	r, _, err := loadBackend(&cmd.backendOptions, ctx)
+	r, _, _, err := loadBackend(&cmd.backendOptions, ctx)
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/cmd-query-blocks.go

@@ -29,7 +29,7 @@ type queryBlocksCmd struct {
 }
 
 func (cmd *queryBlocksCmd) Run(ctx *globalOptions) error {
-	r, c, err := loadBackend(&cmd.backendOptions, ctx)
+	r, _, c, err := loadBackend(&cmd.backendOptions, ctx)
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/cmd-view-index.go

@@ -22,7 +22,7 @@ func (cmd *viewIndexCmd) Run(ctx *globalOptions) error {
 		return err
 	}
 
-	r, _, err := loadBackend(&cmd.backendOptions, ctx)
+	r, _, _, err := loadBackend(&cmd.backendOptions, ctx)
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/main.go

@@ -44,6 +44,10 @@ var cli struct {
 		Index viewIndexCmd `cmd:"" help:"View contents of block index"`
 	} `cmd:""`
 
+	Gen struct {
+		GenIndex genIndexCmd `cmd:"" help:"Generate bloom/index for a block"`
+	} `cmd:""`
+
 	Query struct {
 		API    queryCmd       `cmd:"" help:"query tempo http api"`
 		Blocks queryBlocksCmd `cmd:"" help:"query for a traceid directly from backend blocks"`
@@ -61,7 +65,7 @@ func main() {
 	ctx.FatalIfErrorf(err)
 }
 
-func loadBackend(b *backendOptions, g *globalOptions) (backend.Reader, backend.Compactor, error) {
+func loadBackend(b *backendOptions, g *globalOptions) (backend.Reader, backend.Writer, backend.Compactor, error) {
 	// Defaults
 	cfg := app.Config{}
 	cfg.RegisterFlagsAndApplyDefaults("", &flag.FlagSet{})
@@ -70,12 +74,12 @@ func loadBackend(b *backendOptions, g *globalOptions) (backend.Reader, backend.C
 	if g.ConfigFile != "" {
 		buff, err := ioutil.ReadFile(g.ConfigFile)
 		if err != nil {
-			return nil, nil, fmt.Errorf("failed to read configFile %s: %w", g.ConfigFile, err)
+			return nil, nil, nil, fmt.Errorf("failed to read configFile %s: %w", g.ConfigFile, err)
 		}
 
 		err = yaml.UnmarshalStrict(buff, &cfg)
 		if err != nil {
-			return nil, nil, fmt.Errorf("failed to parse configFile %s: %w", g.ConfigFile, err)
+			return nil, nil, nil, fmt.Errorf("failed to parse configFile %s: %w", g.ConfigFile, err)
 		}
 	}
 
@@ -97,24 +101,25 @@ func loadBackend(b *backendOptions, g *globalOptions) (backend.Reader, backend.C
 
 	var err error
 	var r backend.RawReader
+	var w backend.RawWriter
 	var c backend.Compactor
 
 	switch cfg.StorageConfig.Trace.Backend {
 	case "local":
-		r, _, c, err = local.New(cfg.StorageConfig.Trace.Local)
+		r, w, c, err = local.New(cfg.StorageConfig.Trace.Local)
 	case "gcs":
-		r, _, c, err = gcs.New(cfg.StorageConfig.Trace.GCS)
+		r, w, c, err = gcs.New(cfg.StorageConfig.Trace.GCS)
 	case "s3":
-		r, _, c, err = s3.New(cfg.StorageConfig.Trace.S3)
+		r, w, c, err = s3.New(cfg.StorageConfig.Trace.S3)
 	case "azure":
-		r, _, c, err = azure.New(cfg.StorageConfig.Trace.Azure)
+		r, w, c, err = azure.New(cfg.StorageConfig.Trace.Azure)
 	default:
 		err = fmt.Errorf("unknown backend %s", cfg.StorageConfig.Trace.Backend)
 	}
 
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
-	return backend.NewReader(r), c, nil
+	return backend.NewReader(r), backend.NewWriter(w), c, nil
 }

cmd/tempo-cli/test-data/single-tenant/b18beca6-4d7f-4464-9f72-f343e688a4a0/meta.json

@@ -0,0 +1 @@
+{"format":"v2","blockID":"b18beca6-4d7f-4464-9f72-f343e688a4a0","minID":"AAAAAAAAAAAAsSEqOi2DZA==","maxID":"AAAAAAAAAAD/7CsmwNUYYA==","tenantID":"single-tenant","startTime":"2021-08-18T11:42:35.8352083Z","endTime":"2021-08-18T11:44:35.7427324Z","totalObjects":621,"size":462536,"compactionLevel":0,"encoding":"zstd","indexPageSize":256000,"totalRecords":611,"dataEncoding":"v1","bloomShards":1}

cmd/tempo-cli/test-data/tempo.yaml

@@ -0,0 +1,5 @@
+storage:
+  trace:
+    backend: local
+    local:
+      path: ./cmd/tempo-cli/test-data/

docs/tempo/dev-internals/Tempo Encoding.drawio

@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2021-08-23T13:13:05.036Z" agent="5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36" etag="4qcQmB-KW2EqxMwi1NjN" version="14.2.3" type="device"><diagram id="fipMrtXJD5Bw3guFqnoV" name="Page-1">7V1bd9u4Ef4tfdBp9iE6AHh/tJ249Z6kya7bpts3SoQkJhSpUpRt7a8vQIA3gKRoSyRIneTBoUACojAz31wwA8y0u+3L32J3t/kceTiYIeC9zLQPM4R0y9DIf7TlyFqQnrWsY99jbbBoePT/xLwR8NaD7+F95cEkioLE31Ubl1EY4mVSaXPjOHquPraKguq37tw1lhoel24gt37zvWTDWjUNgOLG37G/3vCvJrccdmfrZk/zR/cb14ueS03ax5l2F0dRwq62L3c4oNOXTQzrd99wN3+zGIdJlw7Rrw/W4r8vPz79ahy/W/f3O/0fy/eGxV8uOWY/GXtkBvjHKE420ToK3eBj0XobR4fQw3RYQD4Vz3yKoh1phKTxO06SIyene0gi0rRJtgG/i1/85D+0+9zgn/4o3fnwwkdOPxyzD2ESH0ud6Mc/yveKbumnrB/7ffRHNc4bb9pHh3iJWyYrY0A3XuOk5Tkrpy4RDBxtMXkf0i/GgZv4T9X3cDmDrvPnChKSC07F11DUngpFFVDGVkkZ22ADP7nBgX+Vv90FeEt+Lvn6KNzXUu6TuyCoWpltN/DXIblekp44Jg1POE58glo3/MbW9zxGWLz3/3QX6Xh0zneRHybpDzNuZ8aHnAp0APxSB6m8c4FjZfq0cKA8y3z492Bu6YDjDtcL73X2sTMh+Ohf6c8pPRKtVnvCACKl8pd4O/GgDWqpM0K5mhBSQss5UyIFRshYOKMW5y9bA9Uh2JvxXj1wiyTpN7sdJqwQz5AZkBm5XdCrNb1690CFeOUu8S8Sh1X553njJ/hx56ZT/0wsriqvNEqyRMFG2dQdozJvps7n7bkwfmAGBJuS3WOCZhKdBZoQOErlriR1hQyekrtmqTNksUs/fcWxTyaMgvmlZRF1lUWg1G5Bksg8JjF2t364vg2i5Q9ZcOSWf258okPBgj0P0g+HPeED6jWkPERHJBfR4jtRbfT2Ko5ogxvSP6mM8m9jPVw6nLv8Qdqzgefk4tEPyZzX3qaC4e6zIX3Chi8zdEdFCLO3oUNv6N/bw2qFY+wV0KBY+i1B+mEu6s8V30cWfwj6kn/rFUiatXj+UzOPZC3kbWqeKzVxZnLzrwN7aq4d8zZ6N8TPZXYy3S0lS7jY70qD7neUF9ioKVcx+u/i6Mn3Uu703MT9FhP6xvPS+5X7Nb22YpaBsMoyuaItcQwCdRyD+uIY3ZygoYZUWWp2R+2AVOoGR/acOKwvjrXEnpzPxJi21WeyEafVeJ0kWyKTrONOa/EaLF5IwwDWtK8oc/I7oxkLyjENnLLIwQ9SrKbK2ouew71LvW6vC1pz9T0lRDac8SEylGZl/IiszHU2utrr57rO59EU/aRpDzRVGqDUtZ807YGmSt1q25R08lUGnRnvthlQtmNWg85w7OYUlM3e5LijruO/ieVDaIe9j+Ey8vxwzYwfFkZUboI4glNo1YQR8ojhMFFER60J8pYoIqwAW4FzA0CbNQl1BeXY0JO8SDNADAQZgsltGhK/Dxw1d9SaZxPj965BELUmN5Td6ycZ1gbgd2CNjd+Nnx7mK9jd6cjuSmN+xhTDuKMnKdNUl19v10xn7pT+CQtJGuy2/n4Tx+6x9Bj3HBq/1zGr6/wQGEISnPSiRnsPcsFe4rKWvDmZrKwxLEp0ZmfnXAP0QlwHTnCd6bT36InrrOnlLAlWIBrSDOwMohdJWjqf65wTXIfO72Fy16ZfPrWnmIVsjR4dS2nmSvnUauc65FitHXpiugySS+5UdQmwxI7m/w40Tf92FYXJ+33KUDfkAYh2L+S/lLxpHtA6Zcr3yyiIYvZIvF68Q7pNn0qzgJDuFNcG+KUYu8g7wbNyVtEqircuna5yOhMmHtzxeYNj/JfG5clrCOtyaGiL6zpQE5KJeexjxIFdR2K9JzlgM0CgFlRFVb0jD4FSQxmWFMFU0z1BR+2gON8TQhl/ryyphzNzo/iBuWaiqqecZY6PF7uQTDaWlfPaLN30D8uE3GA3T7tNldqMp8uyREv6cLiMtmydK4kJIu5r8uh9OgFehPfhX+nVxn3KUnQrWT9ZTi4gKprmCB2TTTrwXfoiRV5vmu1ZvASO1afni7GLEeTnIrULDaMK03U0yDuA+LnI3GCZ28K6rFgVw95LispJAxmWaOKLBkHP9TWOnNVwZboDNQTfC7vX0JFTpcLoVYds9hb5C6f1xn3mCEWrGUuZn/EqjUWqMGI/STAruig7UMNb1YJON+oyMPUhrWpNrkeTpST0bmjBPJWIwN3v/eUUIPe0Odw1WAIbVkZKRDNqaJa1nbuAYmh6ywKKKervrkgNkVaFaksfFqk1JLHe42G3C/wcrbkNSPA7zmSXoObKX18HkGvtYEGAXEf2mZn92ViwSmttrqEyT9ladcQeAxxAaSrpBdz4IfNPjK6ppOZFlrtqlqf01tVbR1y9bUCfy3GPLoHG9hAkfl7YSeTcTaj/JtoIBEWoW5k+zbw3WgziyQ9mQxQ1oOITt2Jd5zWgERfMlpCEbkG9iiJjNyuhodYHnX7MsDMAQaNBmw2UEyQnRHMpva0V0SFyP8WVfduWrf1hY+im2jKON4lD1T+As+F072CLqedFy7OVmUnF2ZQlEHWm6tn5cGdSVY5j0fdF4Cu1Wu79+pLZZz8dGj+ReTm4QbolwYL7ULlX9duB6Iu0t1DlehVmjNlAt8KrghBV7ZjxL60YcnyswgeDazdLKGyA6vcDMieT6X1BOMtms//MtDMLcuQKhWqx4XWgj3kypAMcOLFaw4zF5FrDDH3AQ2OB4QBIpFlzAYsMUINFGWANgkXTSa+9JBbpHbHIbNjSZBgssiajJ0awSNKdpkpThqzJ+EAqaGPpKmnjqN2m8Url7ez6ljOTMSej4KZE1Mus4rw2LZ+WvgkGlH4iL1+H7T36Scw35cWfh8b1nnd+8+a5/VulCIgOsubUGKWDbnVhTbGWRlmgMAvAjdzygVCpOTux9fTORFVrMtUEfwugGxzLoC1kj0DDVB3sg4qjfW9iewHMBlzLMrvu26N6LUuTGD9N41xGNFiY5BtrszVdYe/sNJ8fe4cd3/Wq0TRIIjIJmzRy9ewT/mVdyXcs/LAmmj7JSCQ8EYokXGsL2ahodlYksvrix9ruPaZ6WGg8gIDetrGRNaQm7IoIahcOTHknrw91Ij68gS8eJAKtuqizM6RStCe4S2ezdW+AOSj/g5I49J38ZHbe/Qs25M6diaWSn03cyGGTtG3Z5b6ychr7VNqjJu6bPfq0R1NebP1M015npQNQFCKnhgTktA3VyGn93ITpFbhoTWRRSE6a6eQeUK5nhwRE3EtolJZJIp518hQ+4uCb54FcngQ7t8xqp/5gz5J11e94GcVUW6kMn0jb3qgPn1jXt/f3KXNdM6tE0E1r7mgQaLpuI8vICq16iGnoE9Qtyrawcrra3Irzc7MskpIE/auUY3sXbQmmXI3K4EzcpjNsYE5tsyVHtpO/spowpfrCHJ++gECOwzyEa7xPauKl+427o5fLw4Ly64kZWzCk+7TIG/Ld0r4cksAPMW/33PjHFzKMn6SwMwdGtRGlrZfaT0X0UbSu+c12bzSAcnC8GWaugQhIM5vCH+qogGoikm7i/o5dNZn4UHSnDeVlZlCfnsmj8IwB1DnMqNbkyaI0DSZPZa8vkFcXsb+yl52f4UserhysS3drwfLzvFqb7+5ylaVL+mkjS7er62vjL13K2VsATHaorQrAtLTxAeb0ctJVAua5O/cM5SPKBlsZMDNfo6GIb6IYdrICyjINwVHUx49hcij5ge4Vqc7q04X80jGA2PTOZVEIYpkSGjuI6TLrl0GsxoyT060Kqy7bdIdttYqnF2ZuY/1Wy80WAtCjX0XOOVQEPXWWm22Nz3JDP0HvFaAHpwF6NTXP1xzdZ0zcCl/IGb+nWbOZ6ZfFdzIlzExTh1uGMzrcsqZnrCk8RC/bq/Q0biHF+2xMscZrAmTV1NZF1OyVy1JbFCMbAkJpkN51gaZHi2x6pckqY2mdT/Tu6XxQXa/meUsHxFfuour4nTe7NlDbPrY2MNq+pecca2jJm1eWrE3RnW7dAK4m4a22/6wlxWGiNmwDQpcCj3o20W/l2SwZQMCuPjewlU3afPs+gWzXsOJvC2vZCNScYzPwgr89xdxwZSZVlnQ99kpTWw5r8sKW9JyBbzef0nfZBe6V1LlwPm4FSEuvxijReXA5RNWnTEamAlk2ghKjWBfPTlSftOTIK5EyV5845qUrSsjzMsxBKcIRaCjPV3m9sXhqpL4NQkeuZPjgx0Ts040y2SlHbkorWvG+IsqaHQBLLDuJrISPk/5wiioudurt5dIzxNmvy7ysqw5D/YmPbKCPz/zKyAAuQwdTJEPN/kV2izT3QAW5aIWeo7wh351WaQX4CdNRtpF3CKYvCEhQI4XfURaEujMdexQEOflVgYIVs4KRJU+MVTMv4gFXF5wXOceNsuZjEh+WREXclzbIBWUzcxl502fUPH6R06NueaMOK97AqORjHNFT0QuFS1hs85lMJH3i/w==</diagram></mxfile>

docs/tempo/website/operations/tempo_cli.md

@@ -199,3 +199,12 @@ Arguments:
 ```bash
 tempo-cli view index -c ./tempo.yaml single-tenant ca314fba-efec-4852-ba3f-8d2b0bbf69f1
 ```
+
+## Generate Index
+
+To generate the index/bloom for a block if the files were deleted/corrupted.
+
+**Example:**
+```bash
+tempo-cli gen gen-index -c ./tempo.yaml single-tenant b18beca6-4d7f-4464-9f72-f343e688a4a0
+```