Multiple instances of a periodic job are run simultaneously, when pro…

…hibit_overlap is true

Fixes #11052
When restoring periodic dispatcher, all periodic jobs are forced without checking for previous childre.

nomad/leader.go

@@ -774,11 +774,15 @@ func (s *Server) restorePeriodicDispatcher() error {
 			continue
 		}
 
-		if _, err := s.periodicDispatcher.ForceRun(job.Namespace, job.ID); err != nil {
+		eval, err := s.periodicDispatcher.ForceRunIfNotRunning(job)
+		if err != nil {
 			logger.Error("force run of periodic job failed", "job", job.NamespacedID(), "error", err)
 			return fmt.Errorf("force run of periodic job %q failed: %v", job.NamespacedID(), err)
 		}
-		logger.Debug("periodic job force runned during leadership establishment", "job", job.NamespacedID())
+
+		if eval != nil {
+			logger.Debug("periodic job force ran during leadership establishment", "job", job.NamespacedID())
+		}
 	}
 
 	return nil

nomad/periodic.go

@@ -274,14 +274,32 @@ func (p *PeriodicDispatch) removeLocked(jobID structs.NamespacedID) error {
 	return nil
 }
 
+func (p *PeriodicDispatch) ForceRunIfNotRunning(job *structs.Job) (*structs.Evaluation, error) {
+
+	if job.Periodic.ProhibitOverlap {
+		running, err := p.dispatcher.RunningChildren(job)
+		if err != nil {
+			p.logger.Error("failed to determine if periodic job has running children", "job", job.NamespacedID(), "error", err)
+			return nil, nil
+		}
+
+		if running {
+			p.logger.Debug("skipping launch of periodic job because job prohibits overlap", "job", job.NamespacedID())
+			return nil, nil
+		}
+	}
+
+	return p.ForceRun(job.Namespace, job.ID)
+}
+
 // ForceRun causes the periodic job to be evaluated immediately and returns the
 // subsequent eval.
 func (p *PeriodicDispatch) ForceRun(namespace, jobID string) (*structs.Evaluation, error) {
 	p.l.Lock()
+	defer p.l.Unlock()
 
 	// Do nothing if not enabled
 	if !p.enabled {
-		p.l.Unlock()
 		return nil, fmt.Errorf("periodic dispatch disabled")
 	}
 
@@ -291,11 +309,9 @@ func (p *PeriodicDispatch) ForceRun(namespace, jobID string) (*structs.Evaluatio
 	}
 	job, tracked := p.tracked[tuple]
 	if !tracked {
-		p.l.Unlock()
 		return nil, fmt.Errorf("can't force run non-tracked job %q (%s)", jobID, namespace)
 	}
 
-	p.l.Unlock()
 	return p.createEval(job, time.Now().In(job.Periodic.GetLocation()))
 }
 
@@ -335,6 +351,7 @@ func (p *PeriodicDispatch) run(ctx context.Context, updateCh <-chan struct{}) {
 // based on the passed launch time.
 func (p *PeriodicDispatch) dispatch(job *structs.Job, launchTime time.Time) {
 	p.l.Lock()
+	defer p.l.Unlock()
 
 	nextLaunch, err := job.Periodic.Next(launchTime)
 	if err != nil {
@@ -349,19 +366,16 @@ func (p *PeriodicDispatch) dispatch(job *structs.Job, launchTime time.Time) {
 		running, err := p.dispatcher.RunningChildren(job)
 		if err != nil {
 			p.logger.Error("failed to determine if periodic job has running children", "job", job.NamespacedID(), "error", err)
-			p.l.Unlock()
 			return
 		}
 
 		if running {
 			p.logger.Debug("skipping launch of periodic job because job prohibits overlap", "job", job.NamespacedID())
-			p.l.Unlock()
 			return
 		}
 	}
 
 	p.logger.Debug(" launching job", "job", job.NamespacedID(), "launch_time", launchTime)
-	p.l.Unlock()
 	p.createEval(job, launchTime)
 }