From b21efb436a9d8ca5f11c2f53c7a2295865f94a01 Mon Sep 17 00:00:00 2001 From: sriv Date: Wed, 16 Jun 2021 23:16:25 +0530 Subject: [PATCH] optimize filewatcher logic, init cache before registering, #1896 Signed-off-by: sriv --- api/infoGatherer/specDetails.go | 25 +++++++++++-------------- api/infoGatherer/specDetails_test.go | 13 ------------- 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/api/infoGatherer/specDetails.go b/api/infoGatherer/specDetails.go index b43336cce..81361ef79 100644 --- a/api/infoGatherer/specDetails.go +++ b/api/infoGatherer/specDetails.go @@ -22,7 +22,6 @@ import ( // SpecInfoGatherer contains the caches for specs, concepts, and steps type SpecInfoGatherer struct { - waitGroup sync.WaitGroup conceptDictionary *gauge.ConceptDictionary specsCache specsCache conceptsCache conceptCache @@ -73,16 +72,15 @@ func NewSpecInfoGatherer(conceptDictionary *gauge.ConceptDictionary) *SpecInfoGa // Init initializes all the SpecInfoGatherer caches func (s *SpecInfoGatherer) Init() { - go s.watchForFileChanges() - s.waitGroup.Wait() - // Concepts parsed first because we need to create a concept dictionary that spec parsing can use s.initConceptsCache() s.initSpecsCache() s.initStepsCache() s.initParamsCache() s.initTagsCache() + go s.watchForFileChanges() } + func (s *SpecInfoGatherer) initTagsCache() { s.tagsCache.mutex.Lock() defer s.tagsCache.mutex.Unlock() @@ -91,7 +89,7 @@ func (s *SpecInfoGatherer) initTagsCache() { for file, specDetail := range s.specsCache.specDetails { s.updateTagsCacheFromSpecs(file, specDetail) } - defer s.specsCache.mutex.Unlock() + s.specsCache.mutex.Unlock() } func (s *SpecInfoGatherer) initParamsCache() { @@ -224,7 +222,11 @@ func removeDuplicateTags(tags []string) []string { } func (s *SpecInfoGatherer) addToSpecsCache(key string, value *SpecDetail) { - s.specsCache.specDetails[key] = value + if s.specsCache.specDetails != nil { + s.specsCache.specDetails[key] = value + } else { + logger.Debugf(false, "specsCache does not have specDetails initialized. Skip adding %s to specsCache", key) + } } func (s *SpecInfoGatherer) addToConceptsCache(key string, value *gauge.Concept) { @@ -243,7 +245,9 @@ func (s *SpecInfoGatherer) deleteFromConceptDictionary(file string) { } func (s *SpecInfoGatherer) addToStepsCache(fileName string, allSteps []*gauge.Step) { - s.stepsCache.steps[fileName] = allSteps + if s.stepsCache.steps != nil { + s.stepsCache.steps[fileName] = allSteps + } } func (s *SpecInfoGatherer) getParsedSpecs(specFiles []string) []*SpecDetail { @@ -407,8 +411,6 @@ func (s *SpecInfoGatherer) onFileRename(watcher *fsnotify.Watcher, file string) } func (s *SpecInfoGatherer) handleEvent(event fsnotify.Event, watcher *fsnotify.Watcher) { - s.waitGroup.Wait() - file, err := filepath.Abs(event.Name) if err != nil { logger.Errorf(false, "Failed to get abs file path for %s: %s", event.Name, err) @@ -429,15 +431,12 @@ func (s *SpecInfoGatherer) handleEvent(event fsnotify.Event, watcher *fsnotify.W } func (s *SpecInfoGatherer) watchForFileChanges() { - s.waitGroup.Add(1) - watcher, err := fsnotify.NewWatcher() if err != nil { logger.Errorf(false, "Error creating fileWatcher: %s", err) } defer watcher.Close() - done := make(chan bool) go func() { for { select { @@ -461,8 +460,6 @@ func (s *SpecInfoGatherer) watchForFileChanges() { for _, dir := range allDirsToWatch { addDirToFileWatcher(watcher, dir) } - s.waitGroup.Done() - <-done } // GetAvailableSpecs returns the list of all the specs in the gauge project diff --git a/api/infoGatherer/specDetails_test.go b/api/infoGatherer/specDetails_test.go index d36630f20..29d778cf3 100644 --- a/api/infoGatherer/specDetails_test.go +++ b/api/infoGatherer/specDetails_test.go @@ -194,7 +194,6 @@ func (s *MySuite) TestInitSpecsCache(c *C) { _, err := createFileIn(s.specsDir, "spec1.spec", spec1) c.Assert(err, Equals, nil) specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(1) specInfoGatherer.initSpecsCache() @@ -207,7 +206,6 @@ func (s *MySuite) TestInitConceptsCache(c *C) { _, err = createFileIn(s.specsDir, "concept2.cpt", concept2) c.Assert(err, Equals, nil) specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.projectDir + string(filepath.Separator) + specDir}} - specInfoGatherer.waitGroup.Add(1) specInfoGatherer.initConceptsCache() @@ -220,7 +218,6 @@ func (s *MySuite) TestInitStepsCache(c *C) { f1, _ := createFileIn(s.specsDir, "concept2.cpt", concept2) f1, _ = filepath.Abs(f1) specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(3) specInfoGatherer.initConceptsCache() specInfoGatherer.initSpecsCache() @@ -237,7 +234,6 @@ func (s *MySuite) TestInitTagsCache(c *C) { } specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(2) specInfoGatherer.initSpecsCache() specInfoGatherer.initTagsCache() @@ -256,7 +252,6 @@ func (s *MySuite) TestInitTagsCacheWithMultipleFiles(c *C) { } specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(2) specInfoGatherer.initSpecsCache() specInfoGatherer.initTagsCache() @@ -267,7 +262,6 @@ func (s *MySuite) TestGetStepsFromCachedSpecs(c *C) { f, _ := createFileIn(s.specsDir, "spec1.spec", spec1) f, _ = filepath.Abs(f) specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(3) specInfoGatherer.initSpecsCache() stepsFromSpecsMap := specInfoGatherer.getStepsFromCachedSpecs() @@ -280,7 +274,6 @@ func (s *MySuite) TestGetStepsFromCachedConcepts(c *C) { f, _ := createFileIn(s.specsDir, "concept1.cpt", concept1) f, _ = filepath.Abs(f) specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(3) specInfoGatherer.initSpecsCache() specInfoGatherer.initConceptsCache() @@ -299,7 +292,6 @@ func (s *MySuite) TestGetAvailableSteps(c *C) { } specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(2) specInfoGatherer.initSpecsCache() specInfoGatherer.initStepsCache() @@ -321,7 +313,6 @@ func (s *MySuite) TestGetAvailableStepsShouldFilterDuplicates(c *C) { } specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(2) specInfoGatherer.initSpecsCache() specInfoGatherer.initStepsCache() @@ -348,7 +339,6 @@ func (s *MySuite) TestGetAvailableStepsShouldFilterConcepts(c *C) { } specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(3) specInfoGatherer.initConceptsCache() specInfoGatherer.initSpecsCache() specInfoGatherer.initStepsCache() @@ -378,7 +368,6 @@ func (s *MySuite) TestGetAvailableAllStepsShouldFilterConcepts(c *C) { } specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(3) specInfoGatherer.initConceptsCache() specInfoGatherer.initSpecsCache() specInfoGatherer.initStepsCache() @@ -461,7 +450,6 @@ func (s *MySuite) TestParamsForStepFile(c *C) { file, _ := createFileIn(s.specsDir, "spec3.spec", spec3) file, _ = filepath.Abs(file) specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(2) specInfoGatherer.initConceptsCache() specInfoGatherer.initSpecsCache() specInfoGatherer.initStepsCache() @@ -497,7 +485,6 @@ func (s *MySuite) TestParamsForConceptFile(c *C) { file, _ := createFileIn(s.specsDir, "concept3.cpt", concept3) file, _ = filepath.Abs(file) specInfoGatherer := &SpecInfoGatherer{SpecDirs: []string{s.specsDir}} - specInfoGatherer.waitGroup.Add(2) specInfoGatherer.initConceptsCache() specInfoGatherer.initSpecsCache() specInfoGatherer.initStepsCache()