diff --git a/chart/monocular/templates/_helpers.tpl b/chart/monocular/templates/_helpers.tpl index 248a20676..0ed1cec5c 100644 --- a/chart/monocular/templates/_helpers.tpl +++ b/chart/monocular/templates/_helpers.tpl @@ -55,6 +55,9 @@ spec: - --mongo-url={{ template "mongodb.fullname" $global }} - --mongo-user=root {{- end }} + {{- range $repo.args }} + - {{ . }} + {{- end }} - {{ $repo.name }} - {{ $repo.url }} command: diff --git a/chart/monocular/values.yaml b/chart/monocular/values.yaml index 9be78745d..4b8f1655d 100644 --- a/chart/monocular/values.yaml +++ b/chart/monocular/values.yaml @@ -16,6 +16,10 @@ sync: # source: my-repository-source # schedule: "*/5 * * * *" # successfulJobsHistoryLimit: 1 + # args: + # - --filter-name=name # sync if chart is named + # - --filter-annotation=annotation # sync if annotation exists + # - --filter-annotation=annotation=value # sync if annotation has value # Uncomment these properties to set HTTP proxy for chart synchronization jobs # httpProxy: # httpsProxy: @@ -154,9 +158,8 @@ mongodb: # ref: https://docs.mongodb.com/manual/reference/connection-string/ global: mongoUrl: - # This parameter will allow to set securityContext: { "runAsUser": 1001} by example -# this is usefull to set a user different to 0 (root) +# this is usefull to set a user different to 0 (root) # It allows to be launch in a Kubernetes with PodSecurityPolicy with a policy set runAsNonRoot securityContext: {} diff --git a/cmd/chart-repo/chart_repo.go b/cmd/chart-repo/chart_repo.go index a61b31784..9ebbd8f12 100644 --- a/cmd/chart-repo/chart_repo.go +++ b/cmd/chart-repo/chart_repo.go @@ -39,12 +39,17 @@ func main() { func init() { cmds := []*cobra.Command{syncCmd, deleteCmd} + filterAnnotations := []string{} + filterNames := []string{} for _, cmd := range cmds { rootCmd.AddCommand(cmd) cmd.Flags().String("mongo-url", "localhost", "MongoDB URL (see https://godoc.org/github.com/globalsign/mgo#Dial for format)") cmd.Flags().String("mongo-database", "charts", "MongoDB database") cmd.Flags().String("mongo-user", "", "MongoDB user") + cmd.Flags().StringSliceVar(&filterAnnotations, "filter-annotation", []string{}, "Filter by charts that match any of these annotations") + cmd.Flags().StringSliceVar(&filterNames, "filter-name", []string{}, "Filter by charts that match these names") + // see version.go cmd.Flags().StringVarP(&userAgentComment, "user-agent-comment", "", "", "UserAgent comment used during outbound requests") cmd.Flags().Bool("debug", false, "verbose logging") diff --git a/cmd/chart-repo/sync.go b/cmd/chart-repo/sync.go index 8527876db..cc014e2b7 100644 --- a/cmd/chart-repo/sync.go +++ b/cmd/chart-repo/sync.go @@ -18,6 +18,7 @@ package main import ( "os" + "strings" "github.com/kubeapps/common/datastore" "github.com/sirupsen/logrus" @@ -46,6 +47,27 @@ var syncCmd = &cobra.Command{ if err != nil { logrus.Fatal(err) } + + filter := new(filters) + filter.Annotations = make(map[string]string) + filterAnnotationsStrings, err := cmd.Flags().GetStringSlice("filter-annotation") + if err != nil { + logrus.Fatal(err) + } + for _, a := range filterAnnotationsStrings { + kv := strings.Split(a, "=") + if len(kv) == 2 { + filter.Annotations[kv[0]] = kv[1] + } else { + filter.Annotations[a] = "" + } + } + filterNammesStrings, err := cmd.Flags().GetStringSlice("filter-name") + if err != nil { + logrus.Fatal(err) + } + filter.Names = filterNammesStrings + mongoPW := os.Getenv("MONGO_PASSWORD") debug, err := cmd.Flags().GetBool("debug") if err != nil { @@ -61,7 +83,7 @@ var syncCmd = &cobra.Command{ } authorizationHeader := os.Getenv("AUTHORIZATION_HEADER") - if err = syncRepo(dbSession, args[0], args[1], authorizationHeader); err != nil { + if err = syncRepo(dbSession, args[0], args[1], authorizationHeader, filter); err != nil { logrus.Fatalf("Can't add chart repository to database: %v", err) } diff --git a/cmd/chart-repo/testdata/valid-index.yaml b/cmd/chart-repo/testdata/valid-index.yaml index d5a93537c..549ff28f7 100644 --- a/cmd/chart-repo/testdata/valid-index.yaml +++ b/cmd/chart-repo/testdata/valid-index.yaml @@ -17,6 +17,53 @@ entries: urls: - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.1.1.tgz version: 2.1.1 + nginx-ingress: + - apiVersion: v1 + appVersion: 0.28.0 + created: 2020-02-14T00:58:48.780420335Z + description: An nginx Ingress controller that uses ConfigMap to store the nginx + configuration. + digest: c170639916a16e33a570923cebefe06066558a266f28a9b2c04d98357f984427 + engine: gotpl + home: https://github.com/kubernetes/ingress-nginx + icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png + keywords: + - ingress + - nginx + kubeVersion: '>=1.10.0-0' + maintainers: + - name: ChiefAlexander + - email: Trevor.G.Wood@gmail.com + name: taharah + name: nginx-ingress + sources: + - https://github.com/kubernetes/ingress-nginx + urls: + - https://kubernetes-charts.storage.googleapis.com/nginx-ingress-1.30.2.tgz + version: 1.30.2 + - apiVersion: v1 + appVersion: 0.28.0 + created: 2020-02-13T21:29:23.810801158Z + description: An nginx Ingress controller that uses ConfigMap to store the nginx + configuration. + digest: be955d4e77599468d63d61d1dc471fc488e36a3fca2263efd639f17260db1968 + engine: gotpl + home: https://github.com/kubernetes/ingress-nginx + icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png + keywords: + - ingress + - nginx + kubeVersion: '>=1.10.0-0' + maintainers: + - name: ChiefAlexander + - email: Trevor.G.Wood@gmail.com + name: taharah + name: nginx-ingress + sources: + - https://github.com/kubernetes/ingress-nginx + urls: + - https://kubernetes-charts.storage.googleapis.com/nginx-ingress-1.30.1.tgz + version: 1.30.1 wordpress: - appVersion: 4.9.1 created: 2017-12-06T18:48:59.644981487Z @@ -42,6 +89,9 @@ entries: urls: - https://kubernetes-charts.storage.googleapis.com/wordpress-0.7.5.tgz version: 0.7.5 + annotations: + sync: "true" + sync-by-name-only: "true" - appVersion: 4.9.0 created: 2017-12-01T11:49:00.136950565Z description: Web publishing platform for building blogs and websites. diff --git a/cmd/chart-repo/types.go b/cmd/chart-repo/types.go index 46b9e2730..4274902dd 100644 --- a/cmd/chart-repo/types.go +++ b/cmd/chart-repo/types.go @@ -66,3 +66,8 @@ type repoCheck struct { LastUpdate time.Time `bson:"last_update"` Checksum string `bson:"checksum"` } + +type filters struct { + Annotations map[string]string + Names []string +} diff --git a/cmd/chart-repo/utils.go b/cmd/chart-repo/utils.go index 79b17c6db..5059be7ca 100644 --- a/cmd/chart-repo/utils.go +++ b/cmd/chart-repo/utils.go @@ -31,6 +31,7 @@ import ( "net/url" "os" "path" + "path/filepath" "strings" "sync" "time" @@ -86,7 +87,7 @@ func init() { // These steps are processed in this way to ensure relevant chart data is // imported into the database as fast as possible. E.g. we want all icons for // charts before fetching readmes for each chart and version pair. -func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizationHeader string) error { +func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizationHeader string, filter *filters) error { url, err := parseRepoURL(repoURL) if err != nil { log.WithFields(log.Fields{"url": repoURL}).WithError(err).Error("failed to parse URL") @@ -115,7 +116,7 @@ func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizati return err } - charts := chartsFromIndex(index, r) + charts := chartsFromIndex(index, r, filter) if len(charts) == 0 { return errors.New("no charts in repository index") } @@ -271,13 +272,22 @@ func parseRepoIndex(body []byte) (*helmrepo.IndexFile, error) { return &index, nil } -func chartsFromIndex(index *helmrepo.IndexFile, r repo) []chart { +func chartsFromIndex(index *helmrepo.IndexFile, r repo, filter *filters) []chart { var charts []chart for _, entry := range index.Entries { if entry[0].GetDeprecated() { log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping deprecated chart") continue } + + if len(filter.Annotations) > 0 || + len(filter.Names) > 0 { + if !filterEntry(entry[0], filter) { + log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping chart as filters did not match") + continue + } + } + charts = append(charts, newChart(entry, r)) } return charts @@ -536,3 +546,28 @@ func initNetClient(additionalCA string) (*http.Client, error) { }, }, nil } + +// return true if entry matches any filter +func filterEntry(entry *helmrepo.ChartVersion, filter *filters) bool { + if len(filter.Annotations) > 0 { + for a, av := range filter.Annotations { + if v, ok := entry.Annotations[a]; ok { + if len(av) == 0 { + return true + } else if v == av { + return true + } + } + } + } + + if len(filter.Names) > 0 { + for _, n := range filter.Names { + matched, _ := filepath.Match(n, entry.Name) + if matched { + return true + } + } + } + return false +} diff --git a/cmd/chart-repo/utils_test.go b/cmd/chart-repo/utils_test.go index 0adc27918..51f58f0f7 100644 --- a/cmd/chart-repo/utils_test.go +++ b/cmd/chart-repo/utils_test.go @@ -183,7 +183,7 @@ func Test_syncURLInvalidity(t *testing.T) { dbSession := mockstore.NewMockSession(&m) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := syncRepo(dbSession, "test", tt.repoURL, "") + err := syncRepo(dbSession, "test", tt.repoURL, "", new(filters)) assert.ExistsErr(t, err, tt.name) }) } @@ -277,7 +277,7 @@ func Test_parseRepoIndex(t *testing.T) { t.Run("valid", func(t *testing.T) { index, err := parseRepoIndex([]byte(validRepoIndexYAML)) assert.NoErr(t, err) - assert.Equal(t, len(index.Entries), 2, "number of charts") + assert.Equal(t, len(index.Entries), 3, "number of charts") assert.Equal(t, index.Entries["acs-engine-autoscaler"][0].GetName(), "acs-engine-autoscaler", "chart version populated") }) } @@ -285,17 +285,90 @@ func Test_parseRepoIndex(t *testing.T) { func Test_chartsFromIndex(t *testing.T) { r := repo{Name: "test", URL: "http://testrepo.com"} index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, r) - assert.Equal(t, len(charts), 2, "number of charts") - + charts := chartsFromIndex(index, r, new(filters)) + assert.Equal(t, len(charts), 3, "number of charts") indexWithDeprecated := validRepoIndexYAML + ` deprecated-chart: - name: deprecated-chart deprecated: true` index2, err := parseRepoIndex([]byte(indexWithDeprecated)) assert.NoErr(t, err) - charts = chartsFromIndex(index2, r) - assert.Equal(t, len(charts), 2, "number of charts") + charts = chartsFromIndex(index2, r, new(filters)) + assert.Equal(t, len(charts), 3, "number of charts") +} + +func Test_chartsFromIndexFilterByName(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Names = append(filter.Names, "wordpress") + filter.Names = append(filter.Names, "not-found") + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByNameGlobbedSingleChar(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Names = append(filter.Names, "word?ress") + filter.Names = append(filter.Names, "not-found") + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByNameGlobbedWildcard(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Names = append(filter.Names, "word*") + filter.Names = append(filter.Names, "not-found") + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByAnnotationWithValue(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Annotations = make(map[string]string) + filter.Annotations["sync"] = "true" + filter.Annotations["not-found"] = "missing" + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByAnnotation(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + // filter on annotation + filter := new(filters) + filter.Annotations = make(map[string]string) + filter.Annotations["sync-by-name-only"] = "" + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByAnnotationDuplicateMatches(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Annotations = make(map[string]string) + filter.Annotations["sync"] = "true" + filter.Annotations["sync-by-name-only"] = "" + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByAnnotationAndName(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Annotations = make(map[string]string) + filter.Annotations["sync"] = "true" + filter.Names = append(filter.Names, "wordpress") + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") } func Test_newChart(t *testing.T) { @@ -316,7 +389,7 @@ func Test_importCharts(t *testing.T) { m.On("RemoveAll", mock.Anything) dbSession := mockstore.NewMockSession(m) index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}) + charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}, new(filters)) importCharts(dbSession, charts) m.AssertExpectations(t) @@ -357,7 +430,7 @@ func Test_fetchAndImportIcon(t *testing.T) { }) index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}) + charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}, new(filters)) t.Run("failed download", func(t *testing.T) { netClient = &badHTTPClient{} @@ -402,7 +475,7 @@ func Test_fetchAndImportIcon(t *testing.T) { func Test_fetchAndImportFiles(t *testing.T) { index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com", AuthorizationHeader: "Bearer ThisSecretAccessTokenAuthenticatesTheClient1s"}) + charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com", AuthorizationHeader: "Bearer ThisSecretAccessTokenAuthenticatesTheClient1s"}, new(filters)) cv := charts[0].ChartVersions[0] t.Run("http error", func(t *testing.T) { @@ -622,7 +695,7 @@ func Test_emptyChartRepo(t *testing.T) { m := mock.Mock{} m.On("One", &repoCheck{}).Return(nil) dbSession := mockstore.NewMockSession(&m) - err := syncRepo(dbSession, "testRepo", "https://my.examplerepo.com", "") + err := syncRepo(dbSession, "testRepo", "https://my.examplerepo.com", "", new(filters)) assert.ExistsErr(t, err, "Failed Request") }