Make sync operations timeout configurable

chart/flux/README.md

@@ -243,6 +243,7 @@ The following tables lists the configurable parameters of the Flux chart and the
 | `service.type`                                    | `ClusterIP`                                          | Service type to be used (exposing the Flux API outside of the cluster is not advised)
 | `service.port`                                    | `3030`                                               | Service port to be used
 | `sync.state`                                      | `git`                                                | Where to keep sync state; either a tag in the upstream repo (`git`), or as an annotation on the SSH secret (`secret`)
+| `sync.timeout`                                    | `None`                                               |  Duration after which sync operations time out (defaults to `1m`)
 | `git.url`                                         | `None`                                               | URL of git repo with Kubernetes manifests
 | `git.readonly`                                    | `false`                                              | If `true`, the git repo will be considered read-only, and Flux will not attempt to write to it
 | `git.branch`                                      | `master`                                             | Branch of git repo to use for Kubernetes manifests

chart/flux/templates/deployment.yaml

@@ -184,6 +184,9 @@ spec:
           - --k8s-secret-name={{ .Values.git.secretName | default (printf "%s-git-deploy" (include "flux.fullname" .)) }}
           - --memcached-hostname={{ .Values.memcached.hostnameOverride | default (printf "%s-memcached" (include "flux.fullname" .)) }}
           - --sync-state={{ .Values.sync.state }}
+          {{- if .Values.sync.timeout }}
+          - --sync-timeout={{ .Values.sync.timeout }}
+          {{- end }}
           {{- if .Values.memcached.createClusterIP }}
           - --memcached-service=
           {{- end }}

chart/flux/values.yaml

@@ -147,6 +147,8 @@ gpgKeys:
 sync:
   # use `.sync.state: secret` to store flux's state as an annotation on the secret (instead of a git tag)
   state: git
+  # Duration after which sync operations time out (defaults to 1m)
+  timeout:
 
 git:
   # URL of git repo with Kubernetes manifests; e.g. git.url=ssh://git@github.com/fluxcd/flux-get-started

cmd/fluxd/main.go

@@ -136,6 +136,7 @@ func main() {
 
 		// syncing
 		syncInterval = fs.Duration("sync-interval", 5*time.Minute, "apply config in git to cluster at least this often, even if there are no new commits")
+		syncTimeout  = fs.Duration("sync-timeout", 1*time.Minute, "duration after which sync operations time out")
 		syncGC       = fs.Bool("sync-garbage-collection", false, "experimental; delete resources that were created by fluxd, but are no longer in the git repo")
 		dryGC        = fs.Bool("sync-garbage-collection-dry", false, "experimental; only log what would be garbage collected, rather than deleting. Implies --sync-garbage-collection")
 		syncState    = fs.String("sync-state", fluxsync.GitTagStateMode, fmt.Sprintf("method used by flux for storing state (one of {%s})", strings.Join([]string{fluxsync.GitTagStateMode, fluxsync.NativeStateMode}, ",")))
@@ -696,6 +697,7 @@ func main() {
 		GitSecretEnabled:          *gitSecret,
 		LoopVars: &daemon.LoopVars{
 			SyncInterval:        *syncInterval,
+			SyncTimeout:         *syncTimeout,
 			SyncState:           syncProvider,
 			AutomationInterval:  *automationInterval,
 			GitTimeout:          *gitTimeout,

docs/references/daemon.md

@@ -59,6 +59,7 @@ Version controlling of cluster manifests provides reproducibility and a historic
 | --git-timeout                                    | `20s`                    | duration after which git operations time out
 | **syncing:** control over how config is applied to the cluster
 | --sync-interval                                  | `5m`                     | apply the git config to the cluster at least this often. New commits may provoke more frequent syncs
+| --sync-timeout                                   | `1m`                     |  duration after which sync operations time out
 | --sync-garbage-collection                        | `false`                  | experimental: when set, fluxd will delete resources that it created, but are no longer present in git
 | **registry cache:** (none of these need overriding, usually)
 | --memcached-hostname                             | `memcached`                        | hostname for memcached service to use for caching image metadata

docs/references/fluxyaml-config-files.md

@@ -168,6 +168,10 @@ In addition, `updaters` are provided with some environment variables:
   * `FLUX_POLICY_VALUE`: value of the policy to be added or updated in the controller. If the `FLUX_POLICY_VALUE`
   environment variable is not set, it means the policy should be removed.
 
+Please note that the default timeout for sync commands is set to one minute.
+If you run into errors like `error executing generator command: context deadline exceeded`,
+you can increase the timeout with the `--sync-timeout` fluxd command flag or the `sync.timeout` Helm chart option.
+
 ### Combining generators, updaters and raw manifest files
 
 The `.flux.yaml` files support including multiple generators and updaters. Here is an example combining multiple

pkg/daemon/daemon.go

@@ -31,15 +31,6 @@ import (
 	"github.com/fluxcd/flux/pkg/update"
 )
 
-const (
-	// This is set to be in sympathy with the request / RPC timeout (i.e., empirically)
-	defaultHandlerTimeout = 10 * time.Second
-	// A job can take an arbitrary amount of time but we want to have
-	// a (generous) threshold for considering a job stuck and
-	// abandoning it
-	defaultJobTimeout = 60 * time.Second
-)
-
 // Daemon is the fully-functional state of a daemon (compare to
 // `NotReadyDaemon`).
 type Daemon struct {
@@ -277,7 +268,7 @@ func (d *Daemon) makeJobFromUpdate(update updateFunc) jobFunc {
 // executeJob runs a job func and keeps track of its status, so the
 // daemon can report it when asked.
 func (d *Daemon) executeJob(id job.ID, do jobFunc, logger log.Logger) (job.Result, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultJobTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), d.SyncTimeout)
 	defer cancel()
 	d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusRunning})
 	result, err := do(ctx, id, logger)
@@ -372,7 +363,7 @@ func (d *Daemon) UpdateManifests(ctx context.Context, spec update.Spec) (job.ID,
 func (d *Daemon) sync() jobFunc {
 	return func(ctx context.Context, jobID job.ID, logger log.Logger) (job.Result, error) {
 		var result job.Result
-		ctx, cancel := context.WithTimeout(ctx, defaultJobTimeout)
+		ctx, cancel := context.WithTimeout(ctx, d.SyncTimeout)
 		defer cancel()
 		err := d.Repo.Refresh(ctx)
 		if err != nil {

pkg/daemon/daemon_test.go

@@ -741,7 +741,7 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *mock.Mock, *mockEventWr
 		JobStatusCache: &job.StatusCache{Size: 100},
 		EventWriter:    events,
 		Logger:         logger,
-		LoopVars:       &LoopVars{GitTimeout: timeout, SyncState: gitSync},
+		LoopVars:       &LoopVars{SyncTimeout: timeout, GitTimeout: timeout, SyncState: gitSync},
 	}
 
 	start := func() {

pkg/daemon/loop.go

@@ -15,6 +15,7 @@ import (
 
 type LoopVars struct {
 	SyncInterval        time.Duration
+	SyncTimeout         time.Duration
 	AutomationInterval  time.Duration
 	GitTimeout          time.Duration
 	GitVerifySignatures bool

pkg/daemon/sync_test.go

@@ -75,7 +75,7 @@ func daemon(t *testing.T) (*Daemon, func()) {
 		JobStatusCache: &job.StatusCache{Size: 100},
 		EventWriter:    events,
 		Logger:         log.NewLogfmtLogger(os.Stdout),
-		LoopVars:       &LoopVars{GitTimeout: timeout},
+		LoopVars:       &LoopVars{SyncTimeout: timeout, GitTimeout: timeout},
 	}
 	return d, func() {
 		close(shutdown)