Merge pull request #2427 from hashicorp/f-gc-limit

Limit parallelism during garbage collection

client/client.go

@@ -238,6 +238,7 @@ func NewClient(cfg *config.Config, consulSyncer *consul.Syncer, logger *log.Logg
 		DiskUsageThreshold:  cfg.GCDiskUsageThreshold,
 		InodeUsageThreshold: cfg.GCInodeUsageThreshold,
 		Interval:            cfg.GCInterval,
+		ParallelDestroys:    cfg.GCParallelDestroys,
 		ReservedDiskMB:      cfg.Node.Reserved.DiskMB,
 	}
 	c.garbageCollector = NewAllocGarbageCollector(logger, statsCollector, gcConfig)
@@ -1832,10 +1833,11 @@ func (c *Client) removeAlloc(alloc *structs.Allocation) error {
 	delete(c.allocs, alloc.ID)
 	c.allocLock.Unlock()
 
-	// Remove the allocrunner from garbage collector
-	c.garbageCollector.Remove(ar)
+	// Ensure the GC has a reference and then collect. Collecting through the GC
+	// applies rate limiting
+	c.garbageCollector.MarkForCollection(ar)
+	go c.garbageCollector.Collect(alloc.ID)
 
-	ar.Destroy()
 	return nil
 }
 

client/config/config.go

@@ -155,12 +155,16 @@ type Config struct {
 	// collection
 	GCInterval time.Duration
 
-	// GCDiskUsageThreshold is the disk usage threshold beyond which the Nomad
-	// client triggers GC of terminal allocations
+	// GCParallelDestroys is the number of parallel destroys the garbage
+	// collector will allow.
+	GCParallelDestroys int
+
+	// GCDiskUsageThreshold is the disk usage threshold given as a percent
+	// beyond which the Nomad client triggers GC of terminal allocations
 	GCDiskUsageThreshold float64
 
-	// GCInodeUsageThreshold is the inode usage threshold beyond which the Nomad
-	// client triggers GC of the terminal allocations
+	// GCInodeUsageThreshold is the inode usage threshold given as a percent
+	// beyond which the Nomad client triggers GC of the terminal allocations
 	GCInodeUsageThreshold float64
 
 	// LogLevel is the level of the logs to putout
@@ -194,6 +198,7 @@ func DefaultConfig() *Config {
 		TLSConfig:               &config.TLSConfig{},
 		LogLevel:                "DEBUG",
 		GCInterval:              1 * time.Minute,
+		GCParallelDestroys:      2,
 		GCDiskUsageThreshold:    80,
 		GCInodeUsageThreshold:   70,
 	}