Skip to content

Commit

Permalink
fix: support managing cluster with multiple argocd instances and anno…
Browse files Browse the repository at this point in the history
…tation based tracking

Signed-off-by: Alexander Matyushentsev <AMatyushentsev@gmail.com>
  • Loading branch information
alexmt committed Oct 3, 2024
1 parent f506127 commit a580a0f
Show file tree
Hide file tree
Showing 27 changed files with 645 additions and 310 deletions.
3 changes: 3 additions & 0 deletions assets/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion cmd/argocd/commands/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -1375,7 +1375,7 @@ func groupObjsForDiff(resources *application.ManagedResourcesResponse, objs map[
}
if local, ok := objs[key]; ok || live != nil {
if local != nil && !kube.IsCRD(local) {
err = resourceTracking.SetAppInstance(local, argoSettings.AppLabelKey, appName, namespace, argoappv1.TrackingMethod(argoSettings.GetTrackingMethod()))
err = resourceTracking.SetAppInstance(local, argoSettings.AppLabelKey, appName, namespace, argoappv1.TrackingMethod(argoSettings.GetTrackingMethod()), argoSettings.GetInstallationID())
errors.CheckError(err)
}

Expand Down
1 change: 1 addition & 0 deletions common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ const (

// AnnotationKeyAppInstance is the Argo CD application name is used as the instance name
AnnotationKeyAppInstance = "argocd.argoproj.io/tracking-id"
AnnotationInstallationID = "argocd.argoproj.io/installation-id"

// AnnotationCompareOptions is a comma-separated list of options for comparison
AnnotationCompareOptions = "argocd.argoproj.io/compare-options"
Expand Down
9 changes: 7 additions & 2 deletions controller/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ type cacheSettings struct {
clusterSettings clustercache.Settings
appInstanceLabelKey string
trackingMethod appv1.TrackingMethod
installationID string
// resourceOverrides provides a list of ignored differences to ignore watched resource updates
resourceOverrides map[string]appv1.ResourceOverride

Expand Down Expand Up @@ -225,6 +226,10 @@ func (c *liveStateCache) loadCacheSettings() (*cacheSettings, error) {
if err != nil {
return nil, err
}
installationID, err := c.settingsMgr.GetInstallationID()
if err != nil {
return nil, err
}
resourceUpdatesOverrides, err := c.settingsMgr.GetIgnoreResourceUpdatesOverrides()
if err != nil {
return nil, err
Expand All @@ -246,7 +251,7 @@ func (c *liveStateCache) loadCacheSettings() (*cacheSettings, error) {
ResourcesFilter: resourcesFilter,
}

return &cacheSettings{clusterSettings, appInstanceLabelKey, argo.GetTrackingMethod(c.settingsMgr), resourceUpdatesOverrides, ignoreResourceUpdatesEnabled}, nil
return &cacheSettings{clusterSettings, appInstanceLabelKey, argo.GetTrackingMethod(c.settingsMgr), installationID, resourceUpdatesOverrides, ignoreResourceUpdatesEnabled}, nil
}

func asResourceNode(r *clustercache.Resource) appv1.ResourceNode {
Expand Down Expand Up @@ -523,7 +528,7 @@ func (c *liveStateCache) getCluster(server string) (clustercache.ClusterCache, e

res.Health, _ = health.GetResourceHealth(un, cacheSettings.clusterSettings.ResourceHealthOverride)

appName := c.resourceTracking.GetAppName(un, cacheSettings.appInstanceLabelKey, cacheSettings.trackingMethod)
appName := c.resourceTracking.GetAppName(un, cacheSettings.appInstanceLabelKey, cacheSettings.trackingMethod, cacheSettings.installationID)
if isRoot && appName != "" {
res.AppName = appName
}
Expand Down
31 changes: 21 additions & 10 deletions controller/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,11 @@ func (m *appStateManager) GetRepoObjs(app *v1alpha1.Application, sources []v1alp
return nil, nil, false, fmt.Errorf("failed to get Helm settings: %w", err)
}

installationID, err := m.settingsMgr.GetInstallationID()
if err != nil {
return nil, nil, false, fmt.Errorf("failed to get installation ID: %w", err)
}

ts.AddCheckpoint("build_options_ms")
serverVersion, apiResources, err := m.liveStateCache.GetVersionsInfo(app.Spec.Destination.Server)
if err != nil {
Expand Down Expand Up @@ -230,6 +235,7 @@ func (m *appStateManager) GetRepoObjs(app *v1alpha1.Application, sources []v1alp
TrackingMethod: string(argo.GetTrackingMethod(m.settingsMgr)),
RefSources: refSources,
HasMultipleSources: app.Spec.HasMultipleSources(),
InstallationID: installationID,
})
if err != nil {
return nil, nil, false, fmt.Errorf("failed to compare revisions for source %d of %d: %w", i+1, len(sources), err)
Expand Down Expand Up @@ -271,6 +277,7 @@ func (m *appStateManager) GetRepoObjs(app *v1alpha1.Application, sources []v1alp
ProjectName: proj.Name,
ProjectSourceRepos: proj.Spec.SourceRepos,
AnnotationManifestGeneratePaths: app.GetAnnotation(v1alpha1.AnnotationKeyManifestGeneratePaths),
InstallationID: installationID,
})
if err != nil {
return nil, nil, false, fmt.Errorf("failed to generate manifest for source %d of %d: %w", i+1, len(sources), err)
Expand Down Expand Up @@ -354,20 +361,24 @@ func DeduplicateTargetObjects(

// getComparisonSettings will return the system level settings related to the
// diff/normalization process.
func (m *appStateManager) getComparisonSettings() (string, map[string]v1alpha1.ResourceOverride, *settings.ResourcesFilter, error) {
func (m *appStateManager) getComparisonSettings() (string, map[string]v1alpha1.ResourceOverride, *settings.ResourcesFilter, string, error) {
resourceOverrides, err := m.settingsMgr.GetResourceOverrides()
if err != nil {
return "", nil, nil, err
return "", nil, nil, "", err
}
appLabelKey, err := m.settingsMgr.GetAppInstanceLabelKey()
if err != nil {
return "", nil, nil, err
return "", nil, nil, "", err
}
resFilter, err := m.settingsMgr.GetResourcesFilter()
if err != nil {
return "", nil, nil, err
return "", nil, nil, "", err
}
installationID, err := m.settingsMgr.GetInstallationID()
if err != nil {
return "", nil, nil, "", err
}
return appLabelKey, resourceOverrides, resFilter, nil
return appLabelKey, resourceOverrides, resFilter, installationID, nil
}

// verifyGnuPGSignature verifies the result of a GnuPG operation for a given git
Expand Down Expand Up @@ -418,7 +429,7 @@ func isManagedNamespace(ns *unstructured.Unstructured, app *v1alpha1.Application
// revision and overrides in the app spec.
func (m *appStateManager) CompareAppState(app *v1alpha1.Application, project *v1alpha1.AppProject, revisions []string, sources []v1alpha1.ApplicationSource, noCache bool, noRevisionCache bool, localManifests []string, hasMultipleSources bool, rollback bool) (*comparisonResult, error) {
ts := stats.NewTimingStats()
appLabelKey, resourceOverrides, resFilter, err := m.getComparisonSettings()
appLabelKey, resourceOverrides, resFilter, installationID, err := m.getComparisonSettings()

ts.AddCheckpoint("settings_ms")

Expand Down Expand Up @@ -586,7 +597,7 @@ func (m *appStateManager) CompareAppState(app *v1alpha1.Application, project *v1

for _, liveObj := range liveObjByKey {
if liveObj != nil {
appInstanceName := m.resourceTracking.GetAppName(liveObj, appLabelKey, trackingMethod)
appInstanceName := m.resourceTracking.GetAppName(liveObj, appLabelKey, trackingMethod, installationID)
if appInstanceName != "" && appInstanceName != app.InstanceName(m.namespace) {
fqInstanceName := strings.ReplaceAll(appInstanceName, "_", "/")
conditions = append(conditions, v1alpha1.ApplicationCondition{
Expand Down Expand Up @@ -725,7 +736,7 @@ func (m *appStateManager) CompareAppState(app *v1alpha1.Application, project *v1
}
gvk := obj.GroupVersionKind()

isSelfReferencedObj := m.isSelfReferencedObj(liveObj, targetObj, app.GetName(), appLabelKey, trackingMethod)
isSelfReferencedObj := m.isSelfReferencedObj(liveObj, targetObj, app.GetName(), appLabelKey, trackingMethod, installationID)

resState := v1alpha1.ResourceStatus{
Namespace: obj.GetNamespace(),
Expand Down Expand Up @@ -1030,7 +1041,7 @@ func NewAppStateManager(
// group and kind) match the properties of the live object, or if the tracking method
// used does not provide the required properties for matching.
// Reference: https://github.com/argoproj/argo-cd/issues/8683
func (m *appStateManager) isSelfReferencedObj(live, config *unstructured.Unstructured, appName, appLabelKey string, trackingMethod v1alpha1.TrackingMethod) bool {
func (m *appStateManager) isSelfReferencedObj(live, config *unstructured.Unstructured, appName, appLabelKey string, trackingMethod v1alpha1.TrackingMethod, installationID string) bool {
if live == nil {
return true
}
Expand Down Expand Up @@ -1063,7 +1074,7 @@ func (m *appStateManager) isSelfReferencedObj(live, config *unstructured.Unstruc
// to match the properties from the live object. Cluster scoped objects
// carry the app's destination namespace in the tracking annotation,
// but are unique in GVK + name combination.
appInstance := m.resourceTracking.GetAppInstance(live, appLabelKey, trackingMethod)
appInstance := m.resourceTracking.GetAppInstance(live, appLabelKey, trackingMethod, installationID)
if appInstance != nil {
return isSelfReferencedObj(live, *appInstance)
}
Expand Down
7 changes: 6 additions & 1 deletion controller/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,11 @@ func (m *appStateManager) SyncAppState(app *v1alpha1.Application, state *v1alpha
log.Errorf("Could not get appInstanceLabelKey: %v", err)
return
}
installationID, err := m.settingsMgr.GetInstallationID()
if err != nil {
log.Errorf("Could not get installation ID: %v", err)
return
}
trackingMethod := argo.GetTrackingMethod(m.settingsMgr)

impersonationEnabled, err := m.settingsMgr.IsImpersonationEnabled()
Expand Down Expand Up @@ -340,7 +345,7 @@ func (m *appStateManager) SyncAppState(app *v1alpha1.Application, state *v1alpha
return (len(syncOp.Resources) == 0 ||
isPostDeleteHook(target) ||
argo.ContainsSyncResource(key.Name, key.Namespace, schema.GroupVersionKind{Kind: key.Kind, Group: key.Group}, syncOp.Resources)) &&
m.isSelfReferencedObj(live, target, app.GetName(), appLabelKey, trackingMethod)
m.isSelfReferencedObj(live, target, app.GetName(), appLabelKey, trackingMethod, installationID)
}),
sync.WithManifestValidation(!syncOp.SyncOptions.HasOption(common.SyncOptionsDisableValidation)),
sync.WithSyncWaveHook(delayBetweenSyncWaves),
Expand Down
3 changes: 3 additions & 0 deletions docs/operator-manual/argocd-cm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,9 @@ data:
# - annotation+label : Also uses an annotation for tracking, but additionally labels the resource with the application name
application.resourceTrackingMethod: annotation

# Optional installation id. Allows to have multiple installations of Argo CD in the same cluster.
installationID: "my-unique-id"

# disables admin user. Admin is enabled by default
admin.enabled: "false"
# add an additional local user with apiKey and login capabilities
Expand Down
8 changes: 8 additions & 0 deletions docs/user-guide/resource_tracking.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,14 @@ metadata:
The advantages of using the tracking id annotation is that there are no clashes any
more with other Kubernetes tools and Argo CD is never confused about the owner of a resource. The `annotation+label` can also be used if you want other tools to understand resources managed by Argo CD.

### Installation ID

If you are managing one cluster using multiple Argo CD instances, you will need to set `installationID` in the Argo CD ConfigMap. This will prevent conflicts between
the different Argo CD instances:

* Each managed resource will have the annotation `argocd.argoproj.io/tracking-id: <installation-id>`
* It is possible to have applications with the same name in Argo CD instances without causing conflicts.

### Non self-referencing annotations
When using the tracking method `annotation` or `annotation+label`, Argo CD will consider the resource properties in the annotation (name, namespace, group and kind) to determine whether the resource should be compared against the desired state. If the tracking annotation does not reference the resource it is applied to, the resource will neither affect the application's sync status nor be marked for pruning.

Expand Down
Loading

0 comments on commit a580a0f

Please sign in to comment.