Skip to content

Commit

Permalink
better inverted index validation
Browse files Browse the repository at this point in the history
  • Loading branch information
owen-d committed May 25, 2022
1 parent 4fc3d41 commit 4bc3ddc
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 12 deletions.
11 changes: 9 additions & 2 deletions pkg/ingester/index/bitprefix.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,16 @@ type BitPrefixInvertedIndex struct {
shards []*indexShard
}

func ValidateBitPrefixShardFactor(factor uint32) error {
if requiredBits := index.NewShard(0, factor).RequiredBits(); 1<<requiredBits != factor {
return fmt.Errorf("Incompatible inverted index shard factor on ingester: It must be a power of two, got %d", factor)
}
return nil
}

func NewBitPrefixWithShards(totalShards uint32) (*BitPrefixInvertedIndex, error) {
if requiredBits := index.NewShard(0, totalShards).RequiredBits(); 1<<requiredBits != totalShards {
return nil, fmt.Errorf("Incompatible inverted index shard factor on ingester: It must be a power of two, got %d", totalShards)
if err := ValidateBitPrefixShardFactor(totalShards); err != nil {
return nil, err
}

shards := make([]*indexShard, totalShards)
Expand Down
42 changes: 42 additions & 0 deletions pkg/loki/config_compat.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package loki

import (
"fmt"

"github.com/grafana/loki/pkg/ingester/index"
"github.com/grafana/loki/pkg/storage/config"
)

func ValidateConfigCompatibility(c Config) error {
for _, fn := range []func(Config) error{
ensureInvertedIndexShardingCompatibility,
} {
if err := fn(c); err != nil {
return err
}
}
return nil
}

func ensureInvertedIndexShardingCompatibility(c Config) error {

for i, sc := range c.SchemaConfig.Configs {
switch sc.IndexType {
case config.TSDBType:
if err := index.ValidateBitPrefixShardFactor(uint32(c.Ingester.IndexShards)); err != nil {
return err
}
default:
if sc.RowShards > 0 && c.Ingester.IndexShards%int(sc.RowShards) > 0 {
return fmt.Errorf(
"incompatible ingester index shards (%d) and period config row shard factor (%d) for period config at index (%d). The ingester factor must be evenly divisible by all period config factors",
c.Ingester.IndexShards,
sc.RowShards,
i,
)
}
}

}
return nil
}
15 changes: 5 additions & 10 deletions pkg/loki/loki.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,19 +195,14 @@ func (c *Config) Validate() error {
c.LimitsConfig.MaxQueryLookback = c.ChunkStoreConfig.MaxLookBackPeriod
}

for i, sc := range c.SchemaConfig.Configs {
if sc.RowShards > 0 && c.Ingester.IndexShards%int(sc.RowShards) > 0 {
return fmt.Errorf(
"incompatible ingester index shards (%d) and period config row shard factor (%d) for period config at index (%d). The ingester factor must be evenly divisible by all period config factors",
c.Ingester.IndexShards,
sc.RowShards,
i,
)
}
}
if err := c.QueryRange.Validate(); err != nil {
return errors.Wrap(err, "invalid query_range config")
}

if err := ValidateConfigCompatibility(*c); err != nil {
return err
}

return nil
}

Expand Down

0 comments on commit 4bc3ddc

Please sign in to comment.