diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go
index 3367caeba8d3..2eb8dd86116f 100644
--- a/drivers/docker/driver.go
+++ b/drivers/docker/driver.go
@@ -275,6 +275,10 @@ CREATE:
 	container, err := d.createContainer(client, containerCfg, driverConfig.Image)
 	if err != nil {
 		d.logger.Error("failed to create container", "error", err)
+		client.RemoveContainer(docker.RemoveContainerOptions{
+			ID:    containerCfg.Name,
+			Force: true,
+		})
 		return nil, nil, nstructs.WrapRecoverable(fmt.Sprintf("failed to create container: %v", err), err)
 	}
 
@@ -307,6 +311,10 @@ CREATE:
 		if err != nil {
 			msg := "failed to inspect started container"
 			d.logger.Error(msg, "error", err)
+			client.RemoveContainer(docker.RemoveContainerOptions{
+				ID:    container.ID,
+				Force: true,
+			})
 			return nil, nil, nstructs.NewRecoverableError(fmt.Errorf("%s %s: %s", msg, container.ID, err), true)
 		}
 		container = runningContainer