fix: check module existence & add in the same txn

This is to prevent a race condition

internal/state/module.go

@@ -230,6 +230,16 @@ func (s *ModuleStore) Add(modPath string) error {
 	txn := s.db.Txn(true)
 	defer txn.Abort()
 
+	err := s.add(txn, modPath)
+	if err != nil {
+		return err
+	}
+	txn.Commit()
+
+	return nil
+}
+
+func (s *ModuleStore) add(txn *memdb.Txn, modPath string) error {
 	// TODO: Introduce Exists method to Txn?
 	obj, err := txn.First(s.tableName, "id", modPath)
 	if err != nil {
@@ -252,7 +262,6 @@ func (s *ModuleStore) Add(modPath string) error {
 		return err
 	}
 
-	txn.Commit()
 	return nil
 }
 
@@ -318,19 +327,25 @@ func (s *ModuleStore) ModuleByPath(path string) (*Module, error) {
 	return mod, nil
 }
 
-func (s *ModuleStore) Exists(path string) (bool, error) {
-	txn := s.db.Txn(false)
+func (s *ModuleStore) AddIfNotExists(path string) error {
+	txn := s.db.Txn(true)
+	defer txn.Abort()
 
 	_, err := moduleByPath(txn, path)
 	if err != nil {
 		if IsModuleNotFound(err) {
-			return false, nil
+			err := s.add(txn, path)
+			if err != nil {
+				return err
+			}
+			txn.Commit()
+			return nil
 		}
 
-		return false, err
+		return err
 	}
 
-	return true, nil
+	return nil
 }
 
 func (s *ModuleStore) ModuleCalls(modPath string) (tfmod.ModuleCalls, error) {

internal/walker/walker.go

@@ -56,8 +56,7 @@ type PathStore interface {
 }
 
 type ModuleStore interface {
-	Exists(dir string) (bool, error)
-	Add(dir string) error
+	AddIfNotExists(dir string) error
 }
 
 func NewWalker(fs fs.ReadDirFS, pathStore PathStore, modStore ModuleStore, walkFunc WalkFunc) *Walker {
@@ -190,20 +189,14 @@ func (w *Walker) walk(ctx context.Context, dir document.DirHandle) error {
 			dirIndexed = true
 			w.logger.Printf("found module %s", dir)
 
-			exists, err := w.modStore.Exists(dir.Path())
+			err := w.modStore.AddIfNotExists(dir.Path())
 			if err != nil {
 				return err
 			}
-			if !exists {
-				err := w.modStore.Add(dir.Path())
-				if err != nil {
-					return err
-				}
-			}
 
 			ids, err := w.walkFunc(ctx, dir)
 			if err != nil {
-				w.collectError(err)
+				w.collectError(fmt.Errorf("walkFunc: %w", err))
 			}
 			w.collectJobIds(ids)
 			continue