diff --git a/rollouts/Makefile b/rollouts/Makefile index 2cc041386e..39e0af927f 100644 --- a/rollouts/Makefile +++ b/rollouts/Makefile @@ -174,3 +174,13 @@ $(CONTROLLER_GEN): $(LOCALBIN) envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + +.PHONY: github_test +github_test: + go test -v -tags=github ./pkg/packagediscovery/... + +# GitLab test requires an access token, so run these tests with: +# GITLAB_TOKEN= make gitlab_test +.PHONY: gitlab_test +gitlab_test: + go test -v -tags=gitlab ./pkg/packagediscovery/... \ No newline at end of file diff --git a/rollouts/api/v1alpha1/rollout_types.go b/rollouts/api/v1alpha1/rollout_types.go index b6a408cd72..8ba75aa43d 100644 --- a/rollouts/api/v1alpha1/rollout_types.go +++ b/rollouts/api/v1alpha1/rollout_types.go @@ -78,32 +78,57 @@ type ClusterSourceGCPFleet struct { const ( GitHub PackageSourceType = "GitHub" + GitLab PackageSourceType = "GitLab" ) -// +kubebuilder:validation:Enum=GitHub +// +kubebuilder:validation:Enum=GitHub;GitLab type PackageSourceType string // PackagesConfig defines the packages the Rollout should deploy. type PackagesConfig struct { SourceType PackageSourceType `json:"sourceType"` - GitHub GitHubSource `json:"github"` + GitHub GitHubSource `json:"github,omitempty"` + GitLab GitLabSource `json:"gitlab,omitempty"` } -// GitHubSource defines the packages source in Git. +// GitHubSource defines the packages source in GitHub. type GitHubSource struct { Selector GitHubSelector `json:"selector"` } -// GitHubSelector defines the selector to apply to Git. +// GitHubSelector defines the selector to apply to packages in GitHub. type GitHubSelector struct { Org string `json:"org"` Repo string `json:"repo"` Directory string `json:"directory,omitempty"` - Revision string `json:"revision"` + Revision string `json:"revision,omitempty"` + Branch string `json:"branch,omitempty"` SecretRef SecretReference `json:"secretRef,omitempty"` } +// GitLabSource defines the packages source in GitLab. +type GitLabSource struct { + // SecretReference is the reference to a kubernetes secret + // that contains GitLab access token + SecretRef SecretReference `json:"secretRef,omitempty"` + // Selector defines the package selector in GitLab. + Selector GitLabSelector `json:"selector"` +} + +// GitLabSelector defines how to select packages in GitLab. +type GitLabSelector struct { + // ProjectID is the numerical identifier of the GitLab project + // It will not be specified if selection involves multiple projects + ProjectID string `json:"projectID,omitempty"` + // Directory refers to the subdirectory path in the project + Directory string `json:"directory,omitempty"` + // Revision refers to the branch, tag of the GitLab repo + Revision string `json:"revision,omitempty"` + // Branch refers to the branch + Branch string `json:"branch,omitempty"` +} + // SecretReference contains the reference to the secret type SecretReference struct { // Name represents the secret name @@ -235,6 +260,13 @@ type Rollout struct { Status RolloutStatus `json:"status,omitempty"` } +func (rollout *Rollout) GetSyncTemplateType() SyncTemplateType { + if rollout.Spec.SyncTemplate == nil { + return TemplateTypeRootSync + } + return rollout.Spec.SyncTemplate.Type +} + //+kubebuilder:object:root=true // RolloutList contains a list of Rollout diff --git a/rollouts/api/v1alpha1/zz_generated.deepcopy.go b/rollouts/api/v1alpha1/zz_generated.deepcopy.go index c0b14cd3c9..09f2b9a255 100644 --- a/rollouts/api/v1alpha1/zz_generated.deepcopy.go +++ b/rollouts/api/v1alpha1/zz_generated.deepcopy.go @@ -166,6 +166,38 @@ func (in *GitInfo) DeepCopy() *GitInfo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitLabSelector) DeepCopyInto(out *GitLabSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitLabSelector. +func (in *GitLabSelector) DeepCopy() *GitLabSelector { + if in == nil { + return nil + } + out := new(GitLabSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitLabSource) DeepCopyInto(out *GitLabSource) { + *out = *in + out.SecretRef = in.SecretRef + out.Selector = in.Selector +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitLabSource. +func (in *GitLabSource) DeepCopy() *GitLabSource { + if in == nil { + return nil + } + out := new(GitLabSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Metadata) DeepCopyInto(out *Metadata) { *out = *in @@ -229,6 +261,7 @@ func (in *PackageToClusterMatcher) DeepCopy() *PackageToClusterMatcher { func (in *PackagesConfig) DeepCopyInto(out *PackagesConfig) { *out = *in out.GitHub = in.GitHub + out.GitLab = in.GitLab } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PackagesConfig. diff --git a/rollouts/config/crd/bases/gitops.kpt.dev_rollouts.yaml b/rollouts/config/crd/bases/gitops.kpt.dev_rollouts.yaml index 022ef2d93e..200a14a221 100644 --- a/rollouts/config/crd/bases/gitops.kpt.dev_rollouts.yaml +++ b/rollouts/config/crd/bases/gitops.kpt.dev_rollouts.yaml @@ -92,12 +92,14 @@ spec: description: Packages source for this Rollout. properties: github: - description: GitHubSource defines the packages source in Git. + description: GitHubSource defines the packages source in GitHub. properties: selector: description: GitHubSelector defines the selector to apply - to Git. + to packages in GitHub. properties: + branch: + type: string directory: type: string org: @@ -117,7 +119,40 @@ spec: required: - org - repo - - revision + type: object + required: + - selector + type: object + gitlab: + description: GitLabSource defines the packages source in GitLab. + properties: + secretRef: + description: SecretReference is the reference to a kubernetes + secret that contains GitLab access token + properties: + name: + description: Name represents the secret name + type: string + type: object + selector: + description: Selector defines the package selector in GitLab. + properties: + branch: + description: Branch refers to the branch + type: string + directory: + description: Directory refers to the subdirectory path + in the project + type: string + projectID: + description: ProjectID is the numerical identifier of + the GitLab project It will not be specified if selection + involves multiple projects + type: string + revision: + description: Revision refers to the branch, tag of the + GitLab repo + type: string type: object required: - selector @@ -125,9 +160,9 @@ spec: sourceType: enum: - GitHub + - GitLab type: string required: - - github - sourceType type: object strategy: diff --git a/rollouts/controllers/rollout_controller.go b/rollouts/controllers/rollout_controller.go index ef63a09bd8..755afad3d3 100644 --- a/rollouts/controllers/rollout_controller.go +++ b/rollouts/controllers/rollout_controller.go @@ -291,12 +291,17 @@ func (r *RolloutReconciler) reconcileRollout(ctx context.Context, rollout *gitop logger := klog.FromContext(ctx) targetClusters, err := r.store.ListClusters(ctx, &rollout.Spec.Clusters, rollout.Spec.Targets.Selector) + if err != nil { + logger.Error(err, "Failed to list clusters") + return client.IgnoreNotFound(err) + } + discoveredPackages, err := packageDiscoveryClient.GetPackages(ctx, rollout.Spec.Packages) if err != nil { logger.Error(err, "Failed to discover packages") return client.IgnoreNotFound(err) } - logger.Info("Discovered packages", "packagesCount", len(discoveredPackages), "packages", discoveredPackages) + logger.Info("Discovered packages", "packagesCount", len(discoveredPackages), "packages", packagediscovery.ToStr(discoveredPackages)) packageClusterMatcherClient := packageclustermatcher.NewPackageClusterMatcher(targetClusters, discoveredPackages) clusterPackages, err := packageClusterMatcherClient.GetClusterPackages(rollout.Spec.PackageToTargetMatcher) @@ -427,7 +432,7 @@ func (r *RolloutReconciler) computeTargets(ctx context.Context, } } else { // remoterootsync already exists - updated, needsUpdate := pkgNeedsUpdate(rollout, rrs, pkg) + updated, needsUpdate := pkgNeedsUpdate(ctx, rollout, rrs, pkg) if needsUpdate { targets.ToBeUpdated = append(targets.ToBeUpdated, updated) } else { @@ -443,13 +448,16 @@ func (r *RolloutReconciler) computeTargets(ctx context.Context, return targets, nil } -func pkgNeedsUpdate(rollout *gitopsv1alpha1.Rollout, rrs gitopsv1alpha1.RemoteRootSync, pkg *packagediscovery.DiscoveredPackage) (*gitopsv1alpha1.RemoteRootSync, bool) { +func pkgNeedsUpdate(ctx context.Context, rollout *gitopsv1alpha1.Rollout, rrs gitopsv1alpha1.RemoteRootSync, pkg *packagediscovery.DiscoveredPackage) (*gitopsv1alpha1.RemoteRootSync, bool) { // TODO: We need to check other things here besides git.Revision and metadata metadata := getSpecMetadata(rollout) - if pkg.Revision != rrs.Spec.Template.Spec.Git.Revision || !reflect.DeepEqual(metadata, rrs.Spec.Template.Metadata) || rrs.Spec.Type != rollout.Spec.SyncTemplate.Type { + if pkg.Revision != rrs.Spec.Template.Spec.Git.Revision || + !reflect.DeepEqual(metadata, rrs.Spec.Template.Metadata) || + rrs.Spec.Type != rollout.GetSyncTemplateType() { + rrs.Spec.Template.Spec.Git.Revision = pkg.Revision rrs.Spec.Template.Metadata = metadata - rrs.Spec.Type = rollout.Spec.SyncTemplate.Type + rrs.Spec.Type = rollout.GetSyncTemplateType() return &rrs, true } return nil, false @@ -756,21 +764,19 @@ func toRootSyncSpec(dpkg *packagediscovery.DiscoveredPackage) *gitopsv1alpha1.Ro return &gitopsv1alpha1.RootSyncSpec{ SourceFormat: "unstructured", Git: &gitopsv1alpha1.GitInfo{ - Repo: fmt.Sprintf("https://github.com/%s/%s.git", dpkg.Org, dpkg.Repo), + // TODO(droot): Repo URL can be an HTTP, GIT or SSH based URL + // Need to make it configurable + Repo: dpkg.HTTPURL(), Revision: dpkg.Revision, Dir: dpkg.Directory, - Branch: "main", + Branch: dpkg.Branch, Auth: "none", }, } } func pkgID(dpkg *packagediscovery.DiscoveredPackage) string { - if dpkg.Directory == "" || dpkg.Directory == "." || dpkg.Directory == "/" { - return fmt.Sprintf("%s-%s", dpkg.Org, dpkg.Repo) - } - - return fmt.Sprintf("%s-%s-%s", dpkg.Org, dpkg.Repo, dpkg.Directory) + return dpkg.ID() } // SetupWithManager sets up the controller with the Manager. diff --git a/rollouts/go.mod b/rollouts/go.mod index 386134704e..95b9f056d7 100644 --- a/rollouts/go.mod +++ b/rollouts/go.mod @@ -11,7 +11,9 @@ require ( github.com/google/go-github/v48 v48.2.0 github.com/onsi/ginkgo/v2 v2.2.0 github.com/onsi/gomega v1.20.2 - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 + github.com/stretchr/testify v1.8.1 + github.com/xanzy/go-gitlab v0.80.2 + golang.org/x/oauth2 v0.3.0 google.golang.org/api v0.103.0 google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c k8s.io/api v0.25.3 @@ -20,6 +22,7 @@ require ( k8s.io/klog/v2 v2.80.1 sigs.k8s.io/cli-utils v0.34.0 sigs.k8s.io/controller-runtime v0.13.1 + sigs.k8s.io/kustomize/kyaml v0.13.9 ) require ( @@ -52,6 +55,8 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -61,6 +66,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.2 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect @@ -76,7 +82,7 @@ require ( golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect - golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect + golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc v1.50.1 // indirect diff --git a/rollouts/go.sum b/rollouts/go.sum index d897953a30..14ccd9839a 100644 --- a/rollouts/go.sum +++ b/rollouts/go.sum @@ -104,6 +104,7 @@ github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMi github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= @@ -216,6 +217,13 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.2.1 h1:YQsLlGDJgwhXFpucSPyVbCBviQtjlHv3jLTlp8YmtEw= +github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= +github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -250,6 +258,8 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -325,6 +335,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/xanzy/go-gitlab v0.80.2 h1:CH1Q7NDklqZllox4ICVF4PwlhQGfPtE+w08Jsb74ZX0= +github.com/xanzy/go-gitlab v0.80.2/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -437,8 +449,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -508,8 +520,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -713,6 +725,7 @@ sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= +sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/rollouts/pkg/packageclustermatcher/packageclustermatcher.go b/rollouts/pkg/packageclustermatcher/packageclustermatcher.go index 71d7302861..65c46932db 100644 --- a/rollouts/pkg/packageclustermatcher/packageclustermatcher.go +++ b/rollouts/pkg/packageclustermatcher/packageclustermatcher.go @@ -62,8 +62,7 @@ func (m *PackageClusterMatcher) GetClusterPackages(matcher gitopsv1alpha1.Packag for _, discoveredPackage := range packages { celPackage := map[string]interface{}{ - "org": discoveredPackage.Org, - "repo": discoveredPackage.Repo, + "repo": discoveredPackage.String(), "directory": discoveredPackage.Directory, } diff --git a/rollouts/pkg/packagediscovery/github.go b/rollouts/pkg/packagediscovery/github.go new file mode 100644 index 0000000000..db63f85445 --- /dev/null +++ b/rollouts/pkg/packagediscovery/github.go @@ -0,0 +1,166 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packagediscovery + +import ( + "context" + "fmt" + "net/http" + + gitopsv1alpha1 "github.com/GoogleContainerTools/kpt/rollouts/api/v1alpha1" + "github.com/google/go-github/v48/github" + "golang.org/x/oauth2" + coreapi "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kustomize/kyaml/sets" +) + +func (d *PackageDiscovery) NewGitHubClient(ctx context.Context, ghConfig gitopsv1alpha1.GitHubSelector) (*github.Client, error) { + httpClient := &http.Client{} + + if secretName := ghConfig.SecretRef.Name; secretName != "" { + var repositorySecret coreapi.Secret + key := client.ObjectKey{Namespace: d.namespace, Name: secretName} + if err := d.client.Get(ctx, key, &repositorySecret); err != nil { + return nil, fmt.Errorf("cannot retrieve github credentials %s: %v", key, err) + } + + accessToken := string(repositorySecret.Data["password"]) + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: accessToken}, + ) + + httpClient = oauth2.NewClient(ctx, ts) + } + + gitHubClient := github.NewClient(httpClient) + + return gitHubClient, nil +} + +func (d *PackageDiscovery) getGitHubPackages(ctx context.Context, config gitopsv1alpha1.PackagesConfig) ([]DiscoveredPackage, error) { + discoveredPackages := []DiscoveredPackage{} + var ghc *github.Client + var err error + if config.SourceType != gitopsv1alpha1.GitHub { + return nil, fmt.Errorf("%v source type not supported yet", config.SourceType) + } + + gitHubSelector := config.GitHub.Selector + + if d.gitHubClientMaker != nil { + ghc, err = d.gitHubClientMaker.NewGitHubClient(ctx, gitHubSelector) + } else { + ghc, err = d.NewGitHubClient(ctx, gitHubSelector) + } + if err != nil { + return nil, err + } + + repos, err := d.getRepos(ghc, gitHubSelector, ctx) + if err != nil { + return nil, fmt.Errorf("unable to get repositories: %w", err) + } + + for _, repo := range repos { + repoPackages, err := d.getPackagesForRepo(ghc, ctx, gitHubSelector, repo) + if err != nil { + return nil, fmt.Errorf("unable to get packages: %w", err) + } + discoveredPackages = append(discoveredPackages, repoPackages...) + } + return discoveredPackages, nil +} + +func (d *PackageDiscovery) getRepos(gitHubClient *github.Client, selector gitopsv1alpha1.GitHubSelector, ctx context.Context) ([]*github.Repository, error) { + var matchingRepos []*github.Repository + + if isSelectorField(selector.Repo) { + // TOOD: add pagination + listOptions := github.RepositoryListOptions{} + listOptions.PerPage = 150 + + repos, _, err := gitHubClient.Repositories.List(ctx, selector.Org, &listOptions) + if err != nil { + return nil, err + } + + allRepoNames := []string{} + for _, repo := range repos { + allRepoNames = append(allRepoNames, *repo.Name) + } + + matchingRepoNames := filterByPattern(selector.Repo, allRepoNames) + matchingRepoNameSet := sets.String{} + matchingRepoNameSet.Insert(matchingRepoNames...) + + for _, repo := range repos { + if matchingRepoNameSet.Has(*repo.Name) { + matchingRepos = append(matchingRepos, repo) + } + } + } else { + repo, _, err := gitHubClient.Repositories.Get(ctx, selector.Org, selector.Repo) + if err != nil { + return nil, err + } + matchingRepos = append(matchingRepos, repo) + } + + return matchingRepos, nil +} + +func (d *PackageDiscovery) getPackagesForRepo(gitHubClient *github.Client, ctx context.Context, selector gitopsv1alpha1.GitHubSelector, repo *github.Repository) ([]DiscoveredPackage, error) { + discoveredPackages := []DiscoveredPackage{} + branch := selector.Branch + if branch == "" { + branch = repo.GetDefaultBranch() + } + if isSelectorField(selector.Directory) { + tree, _, err := gitHubClient.Git.GetTree(ctx, selector.Org, *repo.Name, branch, true) + if err != nil { + return nil, err + } + + allDirectories := []string{} + for _, entry := range tree.Entries { + if *entry.Type == "tree" { + allDirectories = append(allDirectories, *entry.Path) + } + } + + directories := filterByPattern(selector.Directory, allDirectories) + + for _, directory := range directories { + thisDiscoveredPackage := DiscoveredPackage{ + Revision: selector.Revision, + Directory: directory, + GitHubRepo: repo, + Branch: branch, + } + discoveredPackages = append(discoveredPackages, thisDiscoveredPackage) + } + } else { + thisDiscoveredPackage := DiscoveredPackage{ + Revision: selector.Revision, + Directory: selector.Directory, + GitHubRepo: repo, + Branch: branch, + } + discoveredPackages = append(discoveredPackages, thisDiscoveredPackage) + } + + return discoveredPackages, nil +} diff --git a/rollouts/pkg/packagediscovery/github_test.go b/rollouts/pkg/packagediscovery/github_test.go new file mode 100644 index 0000000000..0cf480d53d --- /dev/null +++ b/rollouts/pkg/packagediscovery/github_test.go @@ -0,0 +1,95 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build github + +package packagediscovery + +import ( + "context" + "testing" + + gitopsv1alpha1 "github.com/GoogleContainerTools/kpt/rollouts/api/v1alpha1" + "github.com/stretchr/testify/assert" +) + +func TestGitHubGetPackages_SingleRepo(t *testing.T) { + config := gitopsv1alpha1.PackagesConfig{ + SourceType: gitopsv1alpha1.GitHub, + GitHub: gitopsv1alpha1.GitHubSource{ + Selector: gitopsv1alpha1.GitHubSelector{ + Org: "droot", + Repo: "store", + Directory: "namespaces", + }, + }, + } + + pd := &PackageDiscovery{} + packages, err := pd.GetPackages(context.Background(), config) + assert.NoError(t, err) + if assert.NotEmpty(t, packages) { + pkg := packages[0] + assert.Equal(t, pkg.Directory, "namespaces") + assert.Equal(t, pkg.String(), "store") + t.Logf("package URLs: HTTP:%s SSH:%s", pkg.HTTPURL(), pkg.SSHURL()) + } +} + +func TestGitHubGetPackages_MultipleDirectory(t *testing.T) { + config := gitopsv1alpha1.PackagesConfig{ + SourceType: gitopsv1alpha1.GitHub, + GitHub: gitopsv1alpha1.GitHubSource{ + Selector: gitopsv1alpha1.GitHubSelector{ + Org: "droot", + Repo: "echo-deployments", + Directory: "store-*", + }, + }, + } + pd := &PackageDiscovery{} + packages, err := pd.GetPackages(context.Background(), config) + assert.NoError(t, err) + want := []string{"store-1", "store-2", "store-3", "store-4", "store-5"} + got := []string{} + for _, pkg := range packages { + got = append(got, pkg.Directory) + } + if assert.NotEmpty(t, packages) { + assert.ElementsMatch(t, got, want) + } +} + +func TestGitHubGetPackages_MultipleRepos(t *testing.T) { + config := gitopsv1alpha1.PackagesConfig{ + SourceType: gitopsv1alpha1.GitHub, + GitHub: gitopsv1alpha1.GitHubSource{ + Selector: gitopsv1alpha1.GitHubSelector{ + Org: "droot", + Repo: "store-*", + }, + }, + } + pd := &PackageDiscovery{} + packages, err := pd.GetPackages(context.Background(), config) + assert.NoError(t, err) + want := []string{"store-1", "store-2", "store-3", "store-4", "store-5"} + got := []string{} + for _, pkg := range packages { + got = append(got, pkg.String()) + } + if assert.NotEmpty(t, packages) { + assert.ElementsMatch(t, got, want) + } +} diff --git a/rollouts/pkg/packagediscovery/gitlab.go b/rollouts/pkg/packagediscovery/gitlab.go new file mode 100644 index 0000000000..be2a2fdf6a --- /dev/null +++ b/rollouts/pkg/packagediscovery/gitlab.go @@ -0,0 +1,174 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packagediscovery + +import ( + "context" + "fmt" + + gitopsv1alpha1 "github.com/GoogleContainerTools/kpt/rollouts/api/v1alpha1" + "github.com/xanzy/go-gitlab" + coreapi "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kustomize/kyaml/sets" +) + +func (d *PackageDiscovery) NewGitLabClient(ctx context.Context, gitlabSource gitopsv1alpha1.GitLabSource) (*gitlab.Client, error) { + + // initialize a gitlab client + secretName := gitlabSource.SecretRef.Name + if secretName == "" { + return nil, fmt.Errorf("gitlab secret reference is missing from the config") + } + var repositorySecret coreapi.Secret + key := client.ObjectKey{ + Namespace: d.namespace, + Name: secretName, + } + if err := d.client.Get(ctx, key, &repositorySecret); err != nil { + return nil, fmt.Errorf("cannot retrieve gitlab credentials %s: %v", key, err) + } + + accessToken := string(repositorySecret.Data["token"]) + // TODO(droot): BaseURL should also be configurable through the API + baseURL := "https://gitlab.com/" + + glc, err := gitlab.NewClient(accessToken, gitlab.WithBaseURL(baseURL)) + if err != nil { + return nil, fmt.Errorf("failed to create gitlab client: %w", err) + } + return glc, nil +} + +// getGitLabPackages looks up GitLab for packages specified by a given package selector. +func (d *PackageDiscovery) getGitLabPackages(ctx context.Context, config gitopsv1alpha1.PackagesConfig) ([]DiscoveredPackage, error) { + var discoveredPackages []DiscoveredPackage + var glc *gitlab.Client + var err error + + if d.gitLabClientMaker != nil { + glc, err = d.gitLabClientMaker.NewGitLabClient(ctx, config.GitLab) + } else { + glc, err = d.NewGitLabClient(ctx, config.GitLab) + } + if err != nil { + return nil, fmt.Errorf("failed to create gitlab client: %w", err) + } + gitlabSelector := config.GitLab.Selector + + projects, err := d.getGitLabProjects(ctx, glc, gitlabSelector) + if err != nil { + return nil, err + } + + for _, project := range projects { + repoPackages, err := d.getGitLabPackagesForProject(ctx, glc, project, gitlabSelector) + if err != nil { + return nil, err + } + discoveredPackages = append(discoveredPackages, repoPackages...) + } + return discoveredPackages, nil +} + +// getGitLabPackages looks up GitLab for packages specified by a given package selector. +func (d *PackageDiscovery) getGitLabProjects(ctx context.Context, glc *gitlab.Client, selector gitopsv1alpha1.GitLabSelector) ([]*gitlab.Project, error) { + var matchingProjects []*gitlab.Project + + if isSelectorField(selector.ProjectID) { + membershipAccess := true + options := &gitlab.ListProjectsOptions{ + // TODO: support pagination + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 150, + }, + Membership: gitlab.Bool(membershipAccess), + } + projects, _, err := glc.Projects.ListProjects(options) + if err != nil { + return nil, fmt.Errorf("failed to fetch gitlab projects: %w", err) + } + allRepoNames := []string{} + for _, project := range projects { + allRepoNames = append(allRepoNames, project.Name) + } + + matchingRepoNames := filterByPattern(selector.ProjectID, allRepoNames) + matchingRepoNameSet := sets.String{} + matchingRepoNameSet.Insert(matchingRepoNames...) + + for _, project := range projects { + if matchingRepoNameSet.Has(project.Name) { + matchingProjects = append(matchingProjects, project) + } + } + } else { + project, _, err := glc.Projects.GetProject(selector.ProjectID, &gitlab.GetProjectOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to fetch gitlab projects: %w", err) + } + matchingProjects = append(matchingProjects, project) + } + return matchingProjects, nil +} + +func (d *PackageDiscovery) getGitLabPackagesForProject(ctx context.Context, glc *gitlab.Client, project *gitlab.Project, selector gitopsv1alpha1.GitLabSelector) ([]DiscoveredPackage, error) { + discoveredPackages := []DiscoveredPackage{} + + branch := selector.Branch + if branch == "" { + branch = project.DefaultBranch + } + if isSelectorField(selector.Directory) { + options := &gitlab.ListTreeOptions{ + Recursive: gitlab.Bool(true), + // Ref: gitlab.String(ref), + // Path: gitlab.String(path), + } + tree, _, err := glc.Repositories.ListTree(project.ID, options) + if err != nil { + return nil, err + } + + allDirectories := []string{} + for _, item := range tree { + if item.Type == "tree" { + // Directory + allDirectories = append(allDirectories, item.Path) + } + } + + directories := filterByPattern(selector.Directory, allDirectories) + for _, directory := range directories { + thisDiscoveredPackage := DiscoveredPackage{ + Revision: selector.Revision, + Directory: directory, + GitLabProject: project, + Branch: branch, + } + discoveredPackages = append(discoveredPackages, thisDiscoveredPackage) + } + } else { + thisDiscoveredPackage := DiscoveredPackage{ + Revision: selector.Revision, + Directory: selector.Directory, + GitLabProject: project, + Branch: branch, + } + discoveredPackages = append(discoveredPackages, thisDiscoveredPackage) + } + return discoveredPackages, nil +} diff --git a/rollouts/pkg/packagediscovery/gitlab_test.go b/rollouts/pkg/packagediscovery/gitlab_test.go new file mode 100644 index 0000000000..8e673b4003 --- /dev/null +++ b/rollouts/pkg/packagediscovery/gitlab_test.go @@ -0,0 +1,120 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build gitlab + +package packagediscovery + +import ( + "context" + "fmt" + "os" + "testing" + + gitopsv1alpha1 "github.com/GoogleContainerTools/kpt/rollouts/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/xanzy/go-gitlab" +) + +type testGitLabClientMaker struct{} + +func (*testGitLabClientMaker) NewGitLabClient(ctx context.Context, config gitopsv1alpha1.GitLabSource) (*gitlab.Client, error) { + // initialize a gitlab client + baseURL := "https://gitlab.com/" + token, found := os.LookupEnv("GITLAB_TOKEN") + if !found { + return nil, fmt.Errorf("GITLAB_TOKEN environment variable must be defined") + } + glc, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL)) + if err != nil { + return nil, fmt.Errorf("failed to create gitlab client: %w", err) + } + return glc, nil +} + +func TestGitLabGetPackages_SingleProject(t *testing.T) { + config := gitopsv1alpha1.PackagesConfig{ + SourceType: gitopsv1alpha1.GitLab, + GitLab: gitopsv1alpha1.GitLabSource{ + Selector: gitopsv1alpha1.GitLabSelector{ + ProjectID: "19466078", + Directory: "b", + }, + }, + } + + pd := &PackageDiscovery{ + gitLabClientMaker: &testGitLabClientMaker{}, + } + packages, err := pd.GetPackages(context.Background(), config) + assert.NoError(t, err) + if assert.NotEmpty(t, packages) { + pkg := packages[0] + assert.Equal(t, pkg.Directory, "b") + assert.Equal(t, pkg.String(), "echo-deployments") + t.Logf("package URLs: HTTP:%s SSH:%s", pkg.HTTPURL(), pkg.SSHURL()) + } +} + +func TestGitLabGetPackages_MultipleDirectory(t *testing.T) { + config := gitopsv1alpha1.PackagesConfig{ + SourceType: gitopsv1alpha1.GitLab, + GitLab: gitopsv1alpha1.GitLabSource{ + Selector: gitopsv1alpha1.GitLabSelector{ + ProjectID: "19466078", + Directory: "*", + }, + }, + } + + pd := &PackageDiscovery{ + gitLabClientMaker: &testGitLabClientMaker{}, + } + packages, err := pd.GetPackages(context.Background(), config) + assert.NoError(t, err) + got := []string{} + wants := []string{"a", "b", "c", "namespaces"} + for _, pkg := range packages { + got = append(got, pkg.Directory) + } + if assert.NotEmpty(t, packages) { + assert.ElementsMatch(t, got, wants) + } +} + +func TestGitLabGetPackages_MultipleProjects(t *testing.T) { + config := gitopsv1alpha1.PackagesConfig{ + SourceType: gitopsv1alpha1.GitLab, + GitLab: gitopsv1alpha1.GitLabSource{ + Selector: gitopsv1alpha1.GitLabSelector{ + ProjectID: "*", + Directory: "b", + }, + }, + } + + pd := &PackageDiscovery{ + gitLabClientMaker: &testGitLabClientMaker{}, + } + packages, err := pd.GetPackages(context.Background(), config) + assert.NoError(t, err) + got := []string{} + for _, pkg := range packages { + got = append(got, pkg.String()) + } + if assert.NotEmpty(t, packages) { + wants := []string{"echo-deployments"} + assert.ElementsMatch(t, got, wants) + } +} diff --git a/rollouts/pkg/packagediscovery/packagediscovery.go b/rollouts/pkg/packagediscovery/packagediscovery.go index 114b555271..4d909641f5 100644 --- a/rollouts/pkg/packagediscovery/packagediscovery.go +++ b/rollouts/pkg/packagediscovery/packagediscovery.go @@ -17,7 +17,6 @@ package packagediscovery import ( "context" "fmt" - "net/http" "regexp" "strings" "sync" @@ -26,23 +25,96 @@ import ( gitopsv1alpha1 "github.com/GoogleContainerTools/kpt/rollouts/api/v1alpha1" "github.com/google/go-cmp/cmp" "github.com/google/go-github/v48/github" - "golang.org/x/oauth2" - coreapi "k8s.io/api/core/v1" + "github.com/xanzy/go-gitlab" "sigs.k8s.io/controller-runtime/pkg/client" ) type PackageDiscovery struct { - client client.Client - namespace string - mutex sync.Mutex - cache *Cache + client client.Client + namespace string + mutex sync.Mutex + cache *Cache + gitLabClientMaker GitLabClientMaker + gitHubClientMaker GitHubClientMaker } +// Maker interfaces to help with injecting our own clients during testing + +// GitLabClientMaker knows how to make a GitLab Client. +type GitLabClientMaker interface { + NewGitLabClient(ctx context.Context, config gitopsv1alpha1.GitLabSource) (*gitlab.Client, error) +} + +// GitHubClientMaker knows how to make a GitHub Client. +type GitHubClientMaker interface { + NewGitHubClient(ctx context.Context, config gitopsv1alpha1.GitHubSelector) (*github.Client, error) +} + +// DiscoveredPackage represents a config package that will +// be rolled out. type DiscoveredPackage struct { - Org string - Repo string + // User specified properties Directory string Revision string + Branch string + + // Discovered properties of the project/repo + + // GitLabProject contains the info retrieved from GitLab + GitLabProject *gitlab.Project + // GithubRepo contains the info retrieved from GitHub + GitHubRepo *github.Repository +} + +// HTTPURL refers to the HTTP URL for the repository. +func (dp *DiscoveredPackage) HTTPURL() string { + switch { + case dp.GitLabProject != nil: + return dp.GitLabProject.HTTPURLToRepo + case dp.GitHubRepo != nil: + return dp.GitHubRepo.GetCloneURL() + } + return "" +} + +// SSHURL refers to the SSH(Git) URL for the repository. +func (dp *DiscoveredPackage) SSHURL() string { + switch { + case dp.GitLabProject != nil: + return dp.GitLabProject.SSHURLToRepo + case dp.GitHubRepo != nil: + return dp.GitHubRepo.GetSSHURL() + } + return "" +} + +// ID returns an identifier for the package. +// This is currently being used to generate the unique name +// of the RemoteSync object. +// TODO (droot): figure out a naming scheme for the package identity. +func (dp *DiscoveredPackage) ID() (id string) { + switch { + case dp.GitLabProject != nil: + id = "gitlab-" + fmt.Sprintf("%d", dp.GitLabProject.ID) + case dp.GitHubRepo != nil: + id = "github-" + fmt.Sprintf("%d", dp.GitHubRepo.GetID()) + default: + return "" + } + if dp.Directory == "" || dp.Directory == "." || dp.Directory == "/" { + return id + } + return id + fmt.Sprintf("-%s", dp.Directory) +} + +func (dp *DiscoveredPackage) String() string { + switch { + case dp.GitLabProject != nil: + return dp.GitLabProject.Name + case dp.GitHubRepo != nil: + return *dp.GitHubRepo.Name + } + return "" } type Cache struct { @@ -58,7 +130,7 @@ func NewPackageDiscovery(client client.Client, namespace string) *PackageDiscove } } -func (d *PackageDiscovery) GetPackages(ctx context.Context, config gitopsv1alpha1.PackagesConfig) ([]DiscoveredPackage, error) { +func (d *PackageDiscovery) GetPackages(ctx context.Context, config gitopsv1alpha1.PackagesConfig) (discoveredPackages []DiscoveredPackage, err error) { d.mutex.Lock() defer d.mutex.Unlock() @@ -66,31 +138,19 @@ func (d *PackageDiscovery) GetPackages(ctx context.Context, config gitopsv1alpha return d.cache.packages, nil } - if config.SourceType != gitopsv1alpha1.GitHub { - return nil, fmt.Errorf("%v source type not supported yet", config.SourceType) - } - - gitHubSelector := config.GitHub.Selector - - gitHubClient, err := d.getGitHubClient(ctx, gitHubSelector) - if err != nil { - return nil, fmt.Errorf("unable to create github client: %w", err) - } - - discoveredPackages := []DiscoveredPackage{} - - repositoryNames, err := d.getRepositoryNames(gitHubClient, gitHubSelector, ctx) - if err != nil { - return nil, fmt.Errorf("unable to get repositories: %w", err) - } - - for _, repositoryName := range repositoryNames { - repoPackages, err := d.getPackagesForRepository(gitHubClient, ctx, gitHubSelector, repositoryName) + switch config.SourceType { + case gitopsv1alpha1.GitHub: + discoveredPackages, err = d.getGitHubPackages(ctx, config) if err != nil { - return nil, fmt.Errorf("unable to get packages: %w", err) + return nil, fmt.Errorf("unable to fetch github packages: %w", err) } - - discoveredPackages = append(discoveredPackages, repoPackages...) + case gitopsv1alpha1.GitLab: + discoveredPackages, err = d.getGitLabPackages(ctx, config) + if err != nil { + return nil, fmt.Errorf("unable to fetch gitlab packages: %w", err) + } + default: + return nil, fmt.Errorf("%v source type not supported yet", config.SourceType) } d.cache = &Cache{ @@ -102,88 +162,6 @@ func (d *PackageDiscovery) GetPackages(ctx context.Context, config gitopsv1alpha return discoveredPackages, nil } -func (d *PackageDiscovery) getRepositoryNames(gitHubClient *github.Client, selector gitopsv1alpha1.GitHubSelector, ctx context.Context) ([]string, error) { - repositoryNames := []string{} - - if isSelectorField(selector.Repo) { - // TOOD: add pagination - listOptions := github.RepositoryListOptions{} - listOptions.PerPage = 150 - - repositories, _, err := gitHubClient.Repositories.List(ctx, selector.Org, &listOptions) - if err != nil { - return nil, err - } - - allRepositoryNames := []string{} - for _, repository := range repositories { - allRepositoryNames = append(allRepositoryNames, *repository.Name) - } - - matchRepositoryNames := filterByPattern(selector.Repo, allRepositoryNames) - - repositoryNames = append(repositoryNames, matchRepositoryNames...) - } else { - repositoryNames = append(repositoryNames, selector.Repo) - } - - return repositoryNames, nil -} - -func (d *PackageDiscovery) getPackagesForRepository(gitHubClient *github.Client, ctx context.Context, selector gitopsv1alpha1.GitHubSelector, repoName string) ([]DiscoveredPackage, error) { - discoveredPackages := []DiscoveredPackage{} - - if isSelectorField(selector.Directory) { - tree, _, err := gitHubClient.Git.GetTree(ctx, selector.Org, repoName, selector.Revision, true) - if err != nil { - return nil, err - } - - allDirectories := []string{} - for _, entry := range tree.Entries { - if *entry.Type == "tree" { - allDirectories = append(allDirectories, *entry.Path) - } - } - - directories := filterByPattern(selector.Directory, allDirectories) - - for _, directory := range directories { - thisDiscoveredPackage := DiscoveredPackage{Org: selector.Org, Repo: repoName, Revision: selector.Revision, Directory: directory} - discoveredPackages = append(discoveredPackages, thisDiscoveredPackage) - } - } else { - thisDiscoveredPackage := DiscoveredPackage{Org: selector.Org, Repo: repoName, Revision: selector.Revision, Directory: selector.Directory} - discoveredPackages = append(discoveredPackages, thisDiscoveredPackage) - } - - return discoveredPackages, nil -} - -func (d *PackageDiscovery) getGitHubClient(ctx context.Context, selector gitopsv1alpha1.GitHubSelector) (*github.Client, error) { - httpClient := &http.Client{} - - if secretName := selector.SecretRef.Name; secretName != "" { - var repositorySecret coreapi.Secret - key := client.ObjectKey{Namespace: d.namespace, Name: secretName} - if err := d.client.Get(ctx, key, &repositorySecret); err != nil { - return nil, fmt.Errorf("cannot retrieve github credentials %s: %v", key, err) - } - - accessToken := string(repositorySecret.Data["password"]) - - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: accessToken}, - ) - - httpClient = oauth2.NewClient(ctx, ts) - } - - gitHubClient := github.NewClient(httpClient) - - return gitHubClient, nil -} - func (d *PackageDiscovery) useCache(config gitopsv1alpha1.PackagesConfig) bool { return d.cache != nil && cmp.Equal(config, d.cache.config) && time.Now().Before(d.cache.expiration) } @@ -225,3 +203,12 @@ func match(pattern string, value string) bool { func isSelectorField(value string) bool { return strings.Contains(value, "*") } + +// ToStr is convenient method to pretty print set of packages. +func ToStr(packages []DiscoveredPackage) string { + pkgNames := []string{} + for _, pkg := range packages { + pkgNames = append(pkgNames, pkg.String()) + } + return strings.Join(pkgNames, ",") +}