Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "[dbnode] Properly rebuild index segments if they fail verific… #2944

Merged
merged 1 commit into from
Nov 24, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 14 additions & 20 deletions src/dbnode/persist/fs/index_read.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ package fs

import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
Expand All @@ -42,9 +41,6 @@ const (
mmapPersistFsIndexName = "mmap.persist.fs.index"
)

// ErrIndexReaderValidationFailed is returned for corrupt index segemnts.
var ErrIndexReaderValidationFailed = errors.New("validation failed")

type indexReader struct {
opts Options
filePathPrefix string
Expand Down Expand Up @@ -309,54 +305,52 @@ func (r *indexReader) Validate() error {

func (r *indexReader) validateDigestsFileDigest() error {
if r.readDigests.digestsFileDigest != r.expectedDigestOfDigest {
return fmt.Errorf("(%w) read digests file checksum bad: expected=%d, actual=%d",
ErrIndexReaderValidationFailed, r.expectedDigestOfDigest, r.readDigests.digestsFileDigest)
return fmt.Errorf("read digests file checksum bad: expected=%d, actual=%d",
r.expectedDigestOfDigest, r.readDigests.digestsFileDigest)
}
return nil
}

func (r *indexReader) validateInfoFileDigest() error {
if r.readDigests.infoFileDigest != r.expectedDigest.InfoDigest {
return fmt.Errorf("(%w) read info file checksum bad: expected=%d, actual=%d",
ErrIndexReaderValidationFailed, r.expectedDigest.InfoDigest, r.readDigests.infoFileDigest)
return fmt.Errorf("read info file checksum bad: expected=%d, actual=%d",
r.expectedDigest.InfoDigest, r.readDigests.infoFileDigest)
}
return nil
}

func (r *indexReader) validateSegmentFileDigest(segmentIdx, fileIdx int) error {
if segmentIdx >= len(r.readDigests.segments) {
return fmt.Errorf(
"(%w) have not read correct number of segments to validate segment %d checksums: "+
"have not read correct number of segments to validate segment %d checksums: "+
"need=%d, actual=%d",
ErrIndexReaderValidationFailed, segmentIdx, segmentIdx+1, len(r.readDigests.segments))
segmentIdx, segmentIdx+1, len(r.readDigests.segments))
}
if segmentIdx >= len(r.expectedDigest.SegmentDigests) {
return fmt.Errorf(
"(%w) have not read digest files correctly to validate segment %d checksums: "+
"have not read digest files correctly to validate segment %d checksums: "+
"need=%d, actual=%d",
ErrIndexReaderValidationFailed, segmentIdx, segmentIdx+1, len(r.expectedDigest.SegmentDigests))
segmentIdx, segmentIdx+1, len(r.expectedDigest.SegmentDigests))
}

if fileIdx >= len(r.readDigests.segments[segmentIdx].files) {
return fmt.Errorf(
"(%w) have not read correct number of segment files to validate segment %d checksums: "+
"have not read correct number of segment files to validate segment %d checksums: "+
"need=%d, actual=%d",
ErrIndexReaderValidationFailed, segmentIdx, fileIdx+1,
len(r.readDigests.segments[segmentIdx].files))
segmentIdx, fileIdx+1, len(r.readDigests.segments[segmentIdx].files))
}
if fileIdx >= len(r.expectedDigest.SegmentDigests[segmentIdx].Files) {
return fmt.Errorf(
"(%w) have not read correct number of segment files to validate segment %d checksums: "+
"have not read correct number of segment files to validate segment %d checksums: "+
"need=%d, actual=%d",
ErrIndexReaderValidationFailed, segmentIdx, fileIdx+1,
len(r.expectedDigest.SegmentDigests[segmentIdx].Files))
segmentIdx, fileIdx+1, len(r.expectedDigest.SegmentDigests[segmentIdx].Files))
}

expected := r.expectedDigest.SegmentDigests[segmentIdx].Files[fileIdx].Digest
actual := r.readDigests.segments[segmentIdx].files[fileIdx].digest
if actual != expected {
return fmt.Errorf("(%w) read segment file %d for segment %d checksum bad: expected=%d, actual=%d",
ErrIndexReaderValidationFailed, segmentIdx, fileIdx, expected, actual)
return fmt.Errorf("read segment file %d for segment %d checksum bad: expected=%d, actual=%d",
segmentIdx, fileIdx, expected, actual)
}
return nil
}
Expand Down
3 changes: 0 additions & 3 deletions src/dbnode/persist/fs/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -668,6 +668,3 @@ type IndexClaimsManager interface {
blockStart time.Time,
) (int, error)
}

// DeleteFilesFn deletes files passed in as arg.
type DeleteFilesFn func(files []string) error
15 changes: 0 additions & 15 deletions src/dbnode/storage/bootstrap/bootstrap_mock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion src/dbnode/storage/bootstrap/bootstrapper/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ The collection of bootstrappers comprise the task executed when bootstrapping a

- `fs`: The filesystem bootstrapper, used to bootstrap as much data as possible from the local filesystem.
- `peers`: The peers bootstrapper, used to bootstrap any remaining data from peers. This is used for a full node join too.
- *NOTE*: For the node leave case, the peers bs will persist default volume type index filesets to disk with non-overlapping shard time ranges to avoid re-building the entire index segment w/ new shards.
- `commitlog`: The commit log bootstrapper, currently only used in the case that peers bootstrapping fails. Once the current block is being snapshotted frequently to disk it might be faster and make more sense to not actively use the peers bootstrapper and just use a combination of the filesystem bootstrapper and the minimal time range required from the commit log bootstrapper.
- *NOTE*: the commitlog bootstrapper is special cased in that it runs for the *entire* bootstrappable range per shard whereas other bootstrappers fill in the unfulfilled gaps as bootstrapping progresses.

Expand Down
Loading