Skip to content

Commit

Permalink
update py aof.go
Browse files Browse the repository at this point in the history
  • Loading branch information
@bug-superman
bug-superman committed Oct 15, 2023
1 parent e510f50 commit 8dc452e
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 74 deletions.
5 changes: 2 additions & 3 deletions internal/aof/aof.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func ReadCompleteLine(reader *bufio.Reader) ([]byte, error) {
return line, err
}

func (ld *Loader) LoadSingleAppendOnlyFile(LastFile bool, AOFTimeStamp int64) int {
func (ld *Loader) LoadSingleAppendOnlyFile(AOFTimeStamp int64) int {
ret := AOFOK
AOFFilepath := ld.filPath
fp, err := os.Open(AOFFilepath)
Expand Down Expand Up @@ -105,7 +105,7 @@ func (ld *Loader) LoadSingleAppendOnlyFile(LastFile bool, AOFTimeStamp int64) in

if ts > AOFTimeStamp {
ret = AOFTruncated
log.Infof("AOFTruncated%s", line)
log.Infof("AOFTruncated %s", line)
return ret
}
}
Expand All @@ -125,7 +125,6 @@ func (ld *Loader) LoadSingleAppendOnlyFile(LastFile bool, AOFTimeStamp int64) in
var argv []string

for j := 0; j < int(argc); j++ {
//line, err := reader.ReadString('\n')
line, err := ReadCompleteLine(reader)
if err != nil || line[0] != '$' {
log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix <FileName.manifest>", AOFFilepath)
Expand Down
10 changes: 5 additions & 5 deletions internal/reader/aof_reader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,11 @@ func (r *aofReader) StartRead() chan *entry.Entry {
}
log.Infof("the file stat:%v", fi)
aofLoader := aof.NewLoader(r.path, r.ch)
ret := aofLoader.LoadSingleAppendOnlyFile(true, r.stat.AOFTimestamp)
ret := aofLoader.LoadSingleAppendOnlyFile(r.stat.AOFTimestamp)
if ret == AOFOK || ret == AOFTruncated {
log.Infof("The AOF File was successfully loaded\n")
log.Infof("The AOF File was successfully loaded")
} else {
log.Infof("There was an error opening the AOF File.\n")
log.Infof("There was an error opening the AOF File.")
}
log.Infof("Send single AOF finished. path=[%s]", r.path)
close(r.ch)
Expand All @@ -104,9 +104,9 @@ func (r *aofReader) StartRead() chan *entry.Entry {
aofLoader := NewAOFFileInfo(r.path, r.ch)
ret := aofLoader.LoadAppendOnlyFile(manifestInfo, r.ch, r.stat.AOFTimestamp, r.stat.FilterDangerousCommands)
if ret == AOFOK || ret == AOFTruncated {
log.Infof("The AOF File was successfully loaded\n")
log.Infof("The AOF File was successfully loaded")
} else {
log.Infof("There was an error opening the AOF File.\n")
log.Infof("There was an error opening the AOF File.")
}
log.Infof("Send multi-part AOF finished. path=[%s]", r.path)
close(r.ch)
Expand Down
69 changes: 29 additions & 40 deletions internal/reader/parsing_aof.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ func AOFLoadManifestFromFile(amFilepath string) *AOFManifest {
}

} else {
log.Infof("Reading the manifest file, at line %d", linenum)
log.Infof("Read AOF manifest failed")
am = nil
return am
Expand All @@ -370,18 +371,21 @@ func AOFLoadManifestFromFile(amFilepath string) *AOFManifest {
continue
}
if !strings.Contains(buf, "\n") {
log.Infof("Reading the manifest file, at line %d", linenum)
log.Infof("The AOF manifest File contains too long line")
return nil
}
line = strings.Trim(buf, " \t\r\n")
if len(line) == 0 {
log.Infof("Reading the manifest file, at line %d", linenum)
log.Infof("Invalid AOF manifest File format")
return nil
}
argc := 0
argv, argc = SplitArgs(line)

if argc < 6 || argc%2 != 0 {
log.Infof("Reading the manifest file, at line %d", linenum)
log.Infof("Invalid AOF manifest File format")
am = nil
return am
Expand All @@ -391,6 +395,7 @@ func AOFLoadManifestFromFile(amFilepath string) *AOFManifest {
if strings.EqualFold(argv[i], AOFManifestKeyFileName) {
ai.FileName = argv[i+1]
if !PathIsBaseName(ai.FileName) {
log.Infof("Reading the manifest file, at line %d", linenum)
log.Panicf("File can't be a path, just a Filename")
}
} else if strings.EqualFold(argv[i], AOFManifestKeyFileSeq) {
Expand All @@ -400,10 +405,12 @@ func AOFLoadManifestFromFile(amFilepath string) *AOFManifest {
}
}
if ai.FileName == "" || ai.FileSeq == 0 || ai.AOFFileType == "" {
log.Infof("Reading the manifest file, at line %d", linenum)
log.Panicf("Invalid AOF manifest File format")
}
if ai.AOFFileType == AOFManifestFileTypeBase {
if am.BaseAOFInfo != nil {
log.Infof("Reading the manifest file, at line %d", linenum)
log.Panicf("Found duplicate Base File information")
}
am.BaseAOFInfo = ai
Expand All @@ -412,12 +419,14 @@ func AOFLoadManifestFromFile(amFilepath string) *AOFManifest {
am.HistoryList.PushBack(ai)
} else if ai.AOFFileType == AOFManifestTypeIncr {
if ai.FileSeq <= maxseq {
log.Infof("Reading the manifest file, at line %d", linenum)
log.Panicf("Found a non-monotonic sequence number")
}
am.incrAOFList.PushBack(ai)
am.CurrIncrFileSeq = ai.FileSeq
maxseq = ai.FileSeq
} else {
log.Infof("Reading the manifest file, at line %d", linenum)
log.Panicf("Unknown AOF File type")
}
ai = nil
Expand Down Expand Up @@ -463,14 +472,14 @@ func GetAOFManifestAsString(am *AOFManifest) string {
func (manifestInfo *INFO) AOFLoadManifestFromDisk() {
manifestInfo.AOFManifest = AOFManifestCreate()
if DirExists(manifestInfo.AOFDirName) == 0 {
log.Infof("The AOF Directory %v doesn't exist\n", manifestInfo.AOFDirName)
log.Infof("The AOF Directory %v doesn't exist", manifestInfo.AOFDirName)
return
}

amName := manifestInfo.GetAOFManifestFileName()
amFilepath := path.Join(manifestInfo.AOFDirName, amName)
if FileExist(amFilepath) == 0 {
log.Infof("The AOF Directory %v doesn't exist\n", manifestInfo.AOFDirName)
log.Infof("The AOF Directory %v doesn't exist", manifestInfo.AOFDirName)
return
}

Expand Down Expand Up @@ -560,7 +569,7 @@ func (manifestInfo *INFO) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Ent
var totalSize int64 = 0
var BaseSize int64 = 0
var AOFName string
var totalNum, AOFNum, lastFile int
var totalNum, AOFNum int

if manifestInfo.AOFFileExist(manifestInfo.AOFFileName) == 1 {
if DirExists(manifestInfo.AOFDirName) == 0 ||
Expand Down Expand Up @@ -590,40 +599,28 @@ func (manifestInfo *INFO) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Ent
return AOFEmpty
}

log.Infof("The AOF File starts loading.\n")
log.Infof("The AOF File starts loading.")
if am.BaseAOFInfo != nil {
if am.BaseAOFInfo.AOFFileType == AOFManifestFileTypeBase {
AOFName = am.BaseAOFInfo.FileName
manifestInfo.UpdateLoadingFileName(AOFName)
BaseSize = manifestInfo.GetAppendOnlyFileSize(AOFName, nil)
lastFile = totalNum
start = Ustime()
if lastFile != 1 {
ret = manifestInfo.ParsingSingleAppendOnlyFile(AOFName, false, 0)
} else {
ret = manifestInfo.ParsingSingleAppendOnlyFile(AOFName, true, AOFTimeStamp)
}
if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) {
ret = manifestInfo.ParsingSingleAppendOnlyFile(AOFName, 0) //Currently, RDB files cannot be restored at a point in time.
if ret == AOFOK || (ret == AOFTruncated) {
log.Infof("DB loaded from Base File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000)
}

if ret == AOFEmpty {
ret = AOFOK
}

if ret == AOFTruncated && lastFile == 0 {
ret = AOFFailed
log.Infof("Fatal error: the truncated File is not the last File")
}

if ret == AOFOpenErr || ret == AOFFailed {
if ret == AOFOK || ret == AOFTruncated {
log.Infof("The AOF File was successfully loaded\n")
log.Infof("The AOF File was successfully loaded")
} else {
if ret == AOFOpenErr {
log.Panicf("There was an error opening the AOF File.\n")
log.Panicf("There was an error opening the AOF File.")
} else {
log.Panicf("Failed to open AOF File.\n")
log.Panicf("Failed to open AOF File.")
}
}
return ret
Expand All @@ -640,46 +637,38 @@ func (manifestInfo *INFO) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Ent
}
AOFName = ai.FileName
manifestInfo.UpdateLoadingFileName(AOFName)
lastFile = totalNum
AOFNum++
start = Ustime()
if lastFile == 1 {
ret = manifestInfo.ParsingSingleAppendOnlyFile(AOFName, true, AOFTimeStamp)
} else {
ret = manifestInfo.ParsingSingleAppendOnlyFile(AOFName, false, 0)
}
if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) {
ret = manifestInfo.ParsingSingleAppendOnlyFile(AOFName, AOFTimeStamp)
if ret == AOFOK || (ret == AOFTruncated) {
log.Infof("DB loaded from incr File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000)
}

if ret == AOFEmpty {
ret = AOFOK
}

if ret == AOFTruncated && lastFile == 0 {
ret = AOFFailed
log.Infof("Fatal error: the truncated File is not the last File\n")
}

if ret == AOFOpenErr || ret == AOFFailed {
if ret == AOFOpenErr {
log.Infof("There was an error opening the AOF File.\n")
log.Panicf("There was an error opening the AOF File.")
} else {
log.Infof("Failed to open AOF File.\n")
log.Infof("Failed to open AOF File.")
}
return ret
}
totalNum--
}
}

if totalNum == 0 {
log.Infof("All AOF files have been sent.")
} else {
log.Panicf("There are still %d AOF files that were not successfully sent.", totalNum)
}
manifestInfo.AOFCurrentSize = totalSize
manifestInfo.AOFRewriteBaseSize = BaseSize
return ret

}

func (manifestInfo *INFO) ParsingSingleAppendOnlyFile(FileName string, LastFile bool, AOFTimeStamp int64) int {
func (manifestInfo *INFO) ParsingSingleAppendOnlyFile(FileName string, AOFTimeStamp int64) int {
ret := AOFOK
AOFFilepath := path.Join(manifestInfo.AOFDirName, FileName)
println(AOFFilepath)
Expand Down Expand Up @@ -718,7 +707,7 @@ func (manifestInfo *INFO) ParsingSingleAppendOnlyFile(FileName string, LastFile
}
// load single aof file
aofSingleReader := aof.NewLoader(MakePath(manifestInfo.AOFDirName, FileName), manifestInfo.ch)
ret = aofSingleReader.LoadSingleAppendOnlyFile(LastFile, AOFTimeStamp)
ret = aofSingleReader.LoadSingleAppendOnlyFile(AOFTimeStamp)
return ret

}
3 changes: 2 additions & 1 deletion test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@ set -e

# black box test
cd tests/
pybbt cases --verbose --flags modules
pybbt cases --verbose
#--flags modules
Loading

0 comments on commit 8dc452e

Please sign in to comment.