From 05a243483070a6e797b27f9012713bffa20e9604 Mon Sep 17 00:00:00 2001 From: Greg Sidelinger Date: Mon, 10 Feb 2020 22:09:29 -0500 Subject: [PATCH 1/7] Adding filter options to chart-repo Signed-off-by: Greg Sidelinger --- chart/monocular/templates/_helpers.tpl | 3 +++ chart/monocular/values.yaml | 3 +++ cmd/chart-repo/chart_repo.go | 5 ++++ cmd/chart-repo/sync.go | 24 ++++++++++++++++++- cmd/chart-repo/types.go | 5 ++++ cmd/chart-repo/utils.go | 32 +++++++++++++++++++++++--- cmd/chart-repo/utils_test.go | 27 ++++++++++++++++------ 7 files changed, 88 insertions(+), 11 deletions(-) diff --git a/chart/monocular/templates/_helpers.tpl b/chart/monocular/templates/_helpers.tpl index 248a20676..0ed1cec5c 100644 --- a/chart/monocular/templates/_helpers.tpl +++ b/chart/monocular/templates/_helpers.tpl @@ -55,6 +55,9 @@ spec: - --mongo-url={{ template "mongodb.fullname" $global }} - --mongo-user=root {{- end }} + {{- range $repo.args }} + - {{ . }} + {{- end }} - {{ $repo.name }} - {{ $repo.url }} command: diff --git a/chart/monocular/values.yaml b/chart/monocular/values.yaml index 9be78745d..46f3680b4 100644 --- a/chart/monocular/values.yaml +++ b/chart/monocular/values.yaml @@ -16,6 +16,9 @@ sync: # source: my-repository-source # schedule: "*/5 * * * *" # successfulJobsHistoryLimit: 1 + # args: + # - --filter-name=name + # - --filter-annotation=annotation # Uncomment these properties to set HTTP proxy for chart synchronization jobs # httpProxy: # httpsProxy: diff --git a/cmd/chart-repo/chart_repo.go b/cmd/chart-repo/chart_repo.go index a61b31784..0ddd4a3fc 100644 --- a/cmd/chart-repo/chart_repo.go +++ b/cmd/chart-repo/chart_repo.go @@ -39,12 +39,17 @@ func main() { func init() { cmds := []*cobra.Command{syncCmd, deleteCmd} + filterAnnotations := []string{} + filterNames := []string{} for _, cmd := range cmds { rootCmd.AddCommand(cmd) cmd.Flags().String("mongo-url", "localhost", "MongoDB URL (see https://godoc.org/github.com/globalsign/mgo#Dial for format)") cmd.Flags().String("mongo-database", "charts", "MongoDB database") cmd.Flags().String("mongo-user", "", "MongoDB user") + cmd.Flags().StringSliceVar(&filterAnnotations, "filter-annotation", []string{}, "Filter charts with annotation") + cmd.Flags().StringSliceVar(&filterNames, "filter-name", []string{}, "Filter charts name") + // see version.go cmd.Flags().StringVarP(&userAgentComment, "user-agent-comment", "", "", "UserAgent comment used during outbound requests") cmd.Flags().Bool("debug", false, "verbose logging") diff --git a/cmd/chart-repo/sync.go b/cmd/chart-repo/sync.go index 8527876db..cc014e2b7 100644 --- a/cmd/chart-repo/sync.go +++ b/cmd/chart-repo/sync.go @@ -18,6 +18,7 @@ package main import ( "os" + "strings" "github.com/kubeapps/common/datastore" "github.com/sirupsen/logrus" @@ -46,6 +47,27 @@ var syncCmd = &cobra.Command{ if err != nil { logrus.Fatal(err) } + + filter := new(filters) + filter.Annotations = make(map[string]string) + filterAnnotationsStrings, err := cmd.Flags().GetStringSlice("filter-annotation") + if err != nil { + logrus.Fatal(err) + } + for _, a := range filterAnnotationsStrings { + kv := strings.Split(a, "=") + if len(kv) == 2 { + filter.Annotations[kv[0]] = kv[1] + } else { + filter.Annotations[a] = "" + } + } + filterNammesStrings, err := cmd.Flags().GetStringSlice("filter-name") + if err != nil { + logrus.Fatal(err) + } + filter.Names = filterNammesStrings + mongoPW := os.Getenv("MONGO_PASSWORD") debug, err := cmd.Flags().GetBool("debug") if err != nil { @@ -61,7 +83,7 @@ var syncCmd = &cobra.Command{ } authorizationHeader := os.Getenv("AUTHORIZATION_HEADER") - if err = syncRepo(dbSession, args[0], args[1], authorizationHeader); err != nil { + if err = syncRepo(dbSession, args[0], args[1], authorizationHeader, filter); err != nil { logrus.Fatalf("Can't add chart repository to database: %v", err) } diff --git a/cmd/chart-repo/types.go b/cmd/chart-repo/types.go index 46b9e2730..4274902dd 100644 --- a/cmd/chart-repo/types.go +++ b/cmd/chart-repo/types.go @@ -66,3 +66,8 @@ type repoCheck struct { LastUpdate time.Time `bson:"last_update"` Checksum string `bson:"checksum"` } + +type filters struct { + Annotations map[string]string + Names []string +} diff --git a/cmd/chart-repo/utils.go b/cmd/chart-repo/utils.go index 79b17c6db..bd047a0a6 100644 --- a/cmd/chart-repo/utils.go +++ b/cmd/chart-repo/utils.go @@ -86,7 +86,7 @@ func init() { // These steps are processed in this way to ensure relevant chart data is // imported into the database as fast as possible. E.g. we want all icons for // charts before fetching readmes for each chart and version pair. -func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizationHeader string) error { +func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizationHeader string, filter *filters) error { url, err := parseRepoURL(repoURL) if err != nil { log.WithFields(log.Fields{"url": repoURL}).WithError(err).Error("failed to parse URL") @@ -115,7 +115,7 @@ func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizati return err } - charts := chartsFromIndex(index, r) + charts := chartsFromIndex(index, r, filter) if len(charts) == 0 { return errors.New("no charts in repository index") } @@ -271,13 +271,39 @@ func parseRepoIndex(body []byte) (*helmrepo.IndexFile, error) { return &index, nil } -func chartsFromIndex(index *helmrepo.IndexFile, r repo) []chart { +func chartsFromIndex(index *helmrepo.IndexFile, r repo, filter *filters) []chart { var charts []chart for _, entry := range index.Entries { if entry[0].GetDeprecated() { log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping deprecated chart") continue } + if len(filter.Annotations) > 0 { + for a, av := range filter.Annotations { + if v, ok := entry[0].Annotations[a]; ok { + if v == av { + charts = append(charts, newChart(entry, r)) + continue + } + } + } + } + + if len(filter.Names) > 0 { + for _, n := range filter.Names { + if entry[0].Name == n { + charts = append(charts, newChart(entry, r)) + continue + } + } + } + + if len(filter.Annotations) > 0 || + len(filter.Names) > 0 { + log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping chart as filters did not match") + continue + } + charts = append(charts, newChart(entry, r)) } return charts diff --git a/cmd/chart-repo/utils_test.go b/cmd/chart-repo/utils_test.go index 0adc27918..40a76ab31 100644 --- a/cmd/chart-repo/utils_test.go +++ b/cmd/chart-repo/utils_test.go @@ -183,7 +183,7 @@ func Test_syncURLInvalidity(t *testing.T) { dbSession := mockstore.NewMockSession(&m) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := syncRepo(dbSession, "test", tt.repoURL, "") + err := syncRepo(dbSession, "test", tt.repoURL, "", new(filters)) assert.ExistsErr(t, err, tt.name) }) } @@ -285,8 +285,21 @@ func Test_parseRepoIndex(t *testing.T) { func Test_chartsFromIndex(t *testing.T) { r := repo{Name: "test", URL: "http://testrepo.com"} index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, r) + charts := chartsFromIndex(index, r, new(filters)) assert.Equal(t, len(charts), 2, "number of charts") + // filter on name + filter := new(filters) + filter.Names = append(filter.Names, "wordpress") + filter.Names = append(filter.Names, "not-found") + charts = chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") + // filter on annotation + filter = new(filters) + filter.Annotations = make(map[string]string) + filter.Annotations["sync"] = "true" + filter.Annotations["not-found"] = "missing" + filter.Annotations["no-value"] = "" + assert.Equal(t, len(charts), 1, "number of charts") indexWithDeprecated := validRepoIndexYAML + ` deprecated-chart: @@ -294,7 +307,7 @@ func Test_chartsFromIndex(t *testing.T) { deprecated: true` index2, err := parseRepoIndex([]byte(indexWithDeprecated)) assert.NoErr(t, err) - charts = chartsFromIndex(index2, r) + charts = chartsFromIndex(index2, r, new(filters)) assert.Equal(t, len(charts), 2, "number of charts") } @@ -316,7 +329,7 @@ func Test_importCharts(t *testing.T) { m.On("RemoveAll", mock.Anything) dbSession := mockstore.NewMockSession(m) index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}) + charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}, new(filters)) importCharts(dbSession, charts) m.AssertExpectations(t) @@ -357,7 +370,7 @@ func Test_fetchAndImportIcon(t *testing.T) { }) index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}) + charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}, new(filters)) t.Run("failed download", func(t *testing.T) { netClient = &badHTTPClient{} @@ -402,7 +415,7 @@ func Test_fetchAndImportIcon(t *testing.T) { func Test_fetchAndImportFiles(t *testing.T) { index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com", AuthorizationHeader: "Bearer ThisSecretAccessTokenAuthenticatesTheClient1s"}) + charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com", AuthorizationHeader: "Bearer ThisSecretAccessTokenAuthenticatesTheClient1s"}, new(filters)) cv := charts[0].ChartVersions[0] t.Run("http error", func(t *testing.T) { @@ -622,7 +635,7 @@ func Test_emptyChartRepo(t *testing.T) { m := mock.Mock{} m.On("One", &repoCheck{}).Return(nil) dbSession := mockstore.NewMockSession(&m) - err := syncRepo(dbSession, "testRepo", "https://my.examplerepo.com", "") + err := syncRepo(dbSession, "testRepo", "https://my.examplerepo.com", "", new(filters)) assert.ExistsErr(t, err, "Failed Request") } From 7ae103182da6c65cc2cb4d46428fdc3f38f74b66 Mon Sep 17 00:00:00 2001 From: Greg Sidelinger Date: Wed, 12 Feb 2020 00:33:11 -0500 Subject: [PATCH 2/7] Updating filter tests Add filter on annoation name only Signed-off-by: Greg Sidelinger --- cmd/chart-repo/testdata/valid-index.yaml | 3 ++ cmd/chart-repo/utils.go | 10 ++-- cmd/chart-repo/utils_test.go | 58 ++++++++++++++++++------ 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/cmd/chart-repo/testdata/valid-index.yaml b/cmd/chart-repo/testdata/valid-index.yaml index d5a93537c..cab568ca7 100644 --- a/cmd/chart-repo/testdata/valid-index.yaml +++ b/cmd/chart-repo/testdata/valid-index.yaml @@ -42,6 +42,9 @@ entries: urls: - https://kubernetes-charts.storage.googleapis.com/wordpress-0.7.5.tgz version: 0.7.5 + annotations: + sync: "true" + sync-by-name-only: "true" - appVersion: 4.9.0 created: 2017-12-01T11:49:00.136950565Z description: Web publishing platform for building blogs and websites. diff --git a/cmd/chart-repo/utils.go b/cmd/chart-repo/utils.go index bd047a0a6..9e0435644 100644 --- a/cmd/chart-repo/utils.go +++ b/cmd/chart-repo/utils.go @@ -273,6 +273,7 @@ func parseRepoIndex(body []byte) (*helmrepo.IndexFile, error) { func chartsFromIndex(index *helmrepo.IndexFile, r repo, filter *filters) []chart { var charts []chart +ENTRYLOOP: for _, entry := range index.Entries { if entry[0].GetDeprecated() { log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping deprecated chart") @@ -281,9 +282,12 @@ func chartsFromIndex(index *helmrepo.IndexFile, r repo, filter *filters) []chart if len(filter.Annotations) > 0 { for a, av := range filter.Annotations { if v, ok := entry[0].Annotations[a]; ok { - if v == av { + if len(av) == 0 { charts = append(charts, newChart(entry, r)) - continue + continue ENTRYLOOP + } else if v == av { + charts = append(charts, newChart(entry, r)) + continue ENTRYLOOP } } } @@ -293,7 +297,7 @@ func chartsFromIndex(index *helmrepo.IndexFile, r repo, filter *filters) []chart for _, n := range filter.Names { if entry[0].Name == n { charts = append(charts, newChart(entry, r)) - continue + continue ENTRYLOOP } } } diff --git a/cmd/chart-repo/utils_test.go b/cmd/chart-repo/utils_test.go index 40a76ab31..b3cd35506 100644 --- a/cmd/chart-repo/utils_test.go +++ b/cmd/chart-repo/utils_test.go @@ -287,20 +287,6 @@ func Test_chartsFromIndex(t *testing.T) { index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) charts := chartsFromIndex(index, r, new(filters)) assert.Equal(t, len(charts), 2, "number of charts") - // filter on name - filter := new(filters) - filter.Names = append(filter.Names, "wordpress") - filter.Names = append(filter.Names, "not-found") - charts = chartsFromIndex(index, r, filter) - assert.Equal(t, len(charts), 1, "number of charts") - // filter on annotation - filter = new(filters) - filter.Annotations = make(map[string]string) - filter.Annotations["sync"] = "true" - filter.Annotations["not-found"] = "missing" - filter.Annotations["no-value"] = "" - assert.Equal(t, len(charts), 1, "number of charts") - indexWithDeprecated := validRepoIndexYAML + ` deprecated-chart: - name: deprecated-chart @@ -311,6 +297,50 @@ func Test_chartsFromIndex(t *testing.T) { assert.Equal(t, len(charts), 2, "number of charts") } +func Test_chartsFromIndexFilterByName(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Names = append(filter.Names, "wordpress") + filter.Names = append(filter.Names, "not-found") + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByAnnotationWithValue(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Annotations = make(map[string]string) + filter.Annotations["sync"] = "true" + filter.Annotations["not-found"] = "missing" + filter.Annotations["sync-by-name-only"] = "" + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByAnnotation(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + // filter on annotation + filter := new(filters) + filter.Annotations = make(map[string]string) + filter.Annotations["sync-by-name-only"] = "" + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByAnnotationDuplicateMatches(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Annotations = make(map[string]string) + filter.Annotations["sync"] = "true" + filter.Annotations["sync-by-name-only"] = "" + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + func Test_newChart(t *testing.T) { r := repo{Name: "test", URL: "http://testrepo.com"} index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) From 57ce3c5f6d3184e620e28a445f0b9079636da9d6 Mon Sep 17 00:00:00 2001 From: Greg Sidelinger Date: Wed, 12 Feb 2020 22:57:34 -0500 Subject: [PATCH 3/7] Moving filter logic to function Updating filter tests Signed-off-by: Greg Sidelinger Signed-off-by: Greg Sidelinger --- chart/monocular/values.yaml | 8 +++--- cmd/chart-repo/utils.go | 52 +++++++++++++++++++----------------- cmd/chart-repo/utils_test.go | 12 ++++++++- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/chart/monocular/values.yaml b/chart/monocular/values.yaml index 46f3680b4..4b8f1655d 100644 --- a/chart/monocular/values.yaml +++ b/chart/monocular/values.yaml @@ -17,8 +17,9 @@ sync: # schedule: "*/5 * * * *" # successfulJobsHistoryLimit: 1 # args: - # - --filter-name=name - # - --filter-annotation=annotation + # - --filter-name=name # sync if chart is named + # - --filter-annotation=annotation # sync if annotation exists + # - --filter-annotation=annotation=value # sync if annotation has value # Uncomment these properties to set HTTP proxy for chart synchronization jobs # httpProxy: # httpsProxy: @@ -157,9 +158,8 @@ mongodb: # ref: https://docs.mongodb.com/manual/reference/connection-string/ global: mongoUrl: - # This parameter will allow to set securityContext: { "runAsUser": 1001} by example -# this is usefull to set a user different to 0 (root) +# this is usefull to set a user different to 0 (root) # It allows to be launch in a Kubernetes with PodSecurityPolicy with a policy set runAsNonRoot securityContext: {} diff --git a/cmd/chart-repo/utils.go b/cmd/chart-repo/utils.go index 9e0435644..985ed3c17 100644 --- a/cmd/chart-repo/utils.go +++ b/cmd/chart-repo/utils.go @@ -273,38 +273,18 @@ func parseRepoIndex(body []byte) (*helmrepo.IndexFile, error) { func chartsFromIndex(index *helmrepo.IndexFile, r repo, filter *filters) []chart { var charts []chart -ENTRYLOOP: for _, entry := range index.Entries { if entry[0].GetDeprecated() { log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping deprecated chart") continue } - if len(filter.Annotations) > 0 { - for a, av := range filter.Annotations { - if v, ok := entry[0].Annotations[a]; ok { - if len(av) == 0 { - charts = append(charts, newChart(entry, r)) - continue ENTRYLOOP - } else if v == av { - charts = append(charts, newChart(entry, r)) - continue ENTRYLOOP - } - } - } - } - - if len(filter.Names) > 0 { - for _, n := range filter.Names { - if entry[0].Name == n { - charts = append(charts, newChart(entry, r)) - continue ENTRYLOOP - } - } - } if len(filter.Annotations) > 0 || len(filter.Names) > 0 { - log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping chart as filters did not match") + if filterEntry(entry[0], filter){ + charts = append(charts, newChart(entry, r)) + } + log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping chart as filters did not match") continue } @@ -566,3 +546,27 @@ func initNetClient(additionalCA string) (*http.Client, error) { }, }, nil } + +// return true if entry matches any filter +func filterEntry( entry *helmrepo.ChartVersion, filter *filters) bool { + if len(filter.Annotations) > 0 { + for a, av := range filter.Annotations { + if v, ok := entry.Annotations[a]; ok { + if len(av) == 0 { + return true + } else if v == av { + return true + } + } + } + } + + if len(filter.Names) > 0 { + for _, n := range filter.Names { + if entry.Name == n { + return true + } + } + } + return false +} diff --git a/cmd/chart-repo/utils_test.go b/cmd/chart-repo/utils_test.go index b3cd35506..ac77dc085 100644 --- a/cmd/chart-repo/utils_test.go +++ b/cmd/chart-repo/utils_test.go @@ -314,7 +314,6 @@ func Test_chartsFromIndexFilterByAnnotationWithValue(t *testing.T) { filter.Annotations = make(map[string]string) filter.Annotations["sync"] = "true" filter.Annotations["not-found"] = "missing" - filter.Annotations["sync-by-name-only"] = "" charts := chartsFromIndex(index, r, filter) assert.Equal(t, len(charts), 1, "number of charts") } @@ -341,6 +340,17 @@ func Test_chartsFromIndexFilterByAnnotationDuplicateMatches(t *testing.T) { assert.Equal(t, len(charts), 1, "number of charts") } +func Test_chartsFromIndexFilterByAnnotationAndName(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Annotations = make(map[string]string) + filter.Annotations["sync"] = "true" + filter.Names = append(filter.Names, "wordpress") + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + func Test_newChart(t *testing.T) { r := repo{Name: "test", URL: "http://testrepo.com"} index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) From b7bcef0c1f780241a6b30c7546c047feb0cd6bec Mon Sep 17 00:00:00 2001 From: Greg Date: Thu, 13 Feb 2020 20:25:48 -0500 Subject: [PATCH 4/7] Update cmd/chart-repo/chart_repo.go Co-Authored-By: Adnan Abdulhussein Signed-off-by: Greg Sidelinger --- cmd/chart-repo/chart_repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/chart-repo/chart_repo.go b/cmd/chart-repo/chart_repo.go index 0ddd4a3fc..eb69ae6c5 100644 --- a/cmd/chart-repo/chart_repo.go +++ b/cmd/chart-repo/chart_repo.go @@ -47,7 +47,7 @@ func init() { cmd.Flags().String("mongo-url", "localhost", "MongoDB URL (see https://godoc.org/github.com/globalsign/mgo#Dial for format)") cmd.Flags().String("mongo-database", "charts", "MongoDB database") cmd.Flags().String("mongo-user", "", "MongoDB user") - cmd.Flags().StringSliceVar(&filterAnnotations, "filter-annotation", []string{}, "Filter charts with annotation") + cmd.Flags().StringSliceVar(&filterAnnotations, "filter-annotation", []string{}, "Filter by charts that match any of these annotations") cmd.Flags().StringSliceVar(&filterNames, "filter-name", []string{}, "Filter charts name") // see version.go From 9a445e2125509d83c8c6706a8783e9e09c575d29 Mon Sep 17 00:00:00 2001 From: Greg Date: Thu, 13 Feb 2020 20:26:07 -0500 Subject: [PATCH 5/7] Update cmd/chart-repo/chart_repo.go Co-Authored-By: Adnan Abdulhussein Signed-off-by: Greg Sidelinger --- cmd/chart-repo/chart_repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/chart-repo/chart_repo.go b/cmd/chart-repo/chart_repo.go index eb69ae6c5..9ebbd8f12 100644 --- a/cmd/chart-repo/chart_repo.go +++ b/cmd/chart-repo/chart_repo.go @@ -48,7 +48,7 @@ func init() { cmd.Flags().String("mongo-database", "charts", "MongoDB database") cmd.Flags().String("mongo-user", "", "MongoDB user") cmd.Flags().StringSliceVar(&filterAnnotations, "filter-annotation", []string{}, "Filter by charts that match any of these annotations") - cmd.Flags().StringSliceVar(&filterNames, "filter-name", []string{}, "Filter charts name") + cmd.Flags().StringSliceVar(&filterNames, "filter-name", []string{}, "Filter by charts that match these names") // see version.go cmd.Flags().StringVarP(&userAgentComment, "user-agent-comment", "", "", "UserAgent comment used during outbound requests") From c95b4f0fdc930351639096fa59894c874e8b29fc Mon Sep 17 00:00:00 2001 From: Greg Sidelinger Date: Thu, 13 Feb 2020 20:51:30 -0500 Subject: [PATCH 6/7] Minor filter updates Adding more test data Signed-off-by: Greg Sidelinger --- cmd/chart-repo/testdata/valid-index.yaml | 47 ++++++++++++++++++++++++ cmd/chart-repo/utils.go | 9 ++--- cmd/chart-repo/utils_test.go | 6 +-- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/cmd/chart-repo/testdata/valid-index.yaml b/cmd/chart-repo/testdata/valid-index.yaml index cab568ca7..549ff28f7 100644 --- a/cmd/chart-repo/testdata/valid-index.yaml +++ b/cmd/chart-repo/testdata/valid-index.yaml @@ -17,6 +17,53 @@ entries: urls: - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.1.1.tgz version: 2.1.1 + nginx-ingress: + - apiVersion: v1 + appVersion: 0.28.0 + created: 2020-02-14T00:58:48.780420335Z + description: An nginx Ingress controller that uses ConfigMap to store the nginx + configuration. + digest: c170639916a16e33a570923cebefe06066558a266f28a9b2c04d98357f984427 + engine: gotpl + home: https://github.com/kubernetes/ingress-nginx + icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png + keywords: + - ingress + - nginx + kubeVersion: '>=1.10.0-0' + maintainers: + - name: ChiefAlexander + - email: Trevor.G.Wood@gmail.com + name: taharah + name: nginx-ingress + sources: + - https://github.com/kubernetes/ingress-nginx + urls: + - https://kubernetes-charts.storage.googleapis.com/nginx-ingress-1.30.2.tgz + version: 1.30.2 + - apiVersion: v1 + appVersion: 0.28.0 + created: 2020-02-13T21:29:23.810801158Z + description: An nginx Ingress controller that uses ConfigMap to store the nginx + configuration. + digest: be955d4e77599468d63d61d1dc471fc488e36a3fca2263efd639f17260db1968 + engine: gotpl + home: https://github.com/kubernetes/ingress-nginx + icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png + keywords: + - ingress + - nginx + kubeVersion: '>=1.10.0-0' + maintainers: + - name: ChiefAlexander + - email: Trevor.G.Wood@gmail.com + name: taharah + name: nginx-ingress + sources: + - https://github.com/kubernetes/ingress-nginx + urls: + - https://kubernetes-charts.storage.googleapis.com/nginx-ingress-1.30.1.tgz + version: 1.30.1 wordpress: - appVersion: 4.9.1 created: 2017-12-06T18:48:59.644981487Z diff --git a/cmd/chart-repo/utils.go b/cmd/chart-repo/utils.go index 985ed3c17..f54f804fe 100644 --- a/cmd/chart-repo/utils.go +++ b/cmd/chart-repo/utils.go @@ -281,11 +281,10 @@ func chartsFromIndex(index *helmrepo.IndexFile, r repo, filter *filters) []chart if len(filter.Annotations) > 0 || len(filter.Names) > 0 { - if filterEntry(entry[0], filter){ - charts = append(charts, newChart(entry, r)) + if !filterEntry(entry[0], filter) { + log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping chart as filters did not match") + continue } - log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping chart as filters did not match") - continue } charts = append(charts, newChart(entry, r)) @@ -548,7 +547,7 @@ func initNetClient(additionalCA string) (*http.Client, error) { } // return true if entry matches any filter -func filterEntry( entry *helmrepo.ChartVersion, filter *filters) bool { +func filterEntry(entry *helmrepo.ChartVersion, filter *filters) bool { if len(filter.Annotations) > 0 { for a, av := range filter.Annotations { if v, ok := entry.Annotations[a]; ok { diff --git a/cmd/chart-repo/utils_test.go b/cmd/chart-repo/utils_test.go index ac77dc085..a9fc69052 100644 --- a/cmd/chart-repo/utils_test.go +++ b/cmd/chart-repo/utils_test.go @@ -277,7 +277,7 @@ func Test_parseRepoIndex(t *testing.T) { t.Run("valid", func(t *testing.T) { index, err := parseRepoIndex([]byte(validRepoIndexYAML)) assert.NoErr(t, err) - assert.Equal(t, len(index.Entries), 2, "number of charts") + assert.Equal(t, len(index.Entries), 3, "number of charts") assert.Equal(t, index.Entries["acs-engine-autoscaler"][0].GetName(), "acs-engine-autoscaler", "chart version populated") }) } @@ -286,7 +286,7 @@ func Test_chartsFromIndex(t *testing.T) { r := repo{Name: "test", URL: "http://testrepo.com"} index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) charts := chartsFromIndex(index, r, new(filters)) - assert.Equal(t, len(charts), 2, "number of charts") + assert.Equal(t, len(charts), 3, "number of charts") indexWithDeprecated := validRepoIndexYAML + ` deprecated-chart: - name: deprecated-chart @@ -294,7 +294,7 @@ func Test_chartsFromIndex(t *testing.T) { index2, err := parseRepoIndex([]byte(indexWithDeprecated)) assert.NoErr(t, err) charts = chartsFromIndex(index2, r, new(filters)) - assert.Equal(t, len(charts), 2, "number of charts") + assert.Equal(t, len(charts), 3, "number of charts") } func Test_chartsFromIndexFilterByName(t *testing.T) { From 8f1f78204ccd0eee2663799b7b78f9ecb2fe997a Mon Sep 17 00:00:00 2001 From: Greg Sidelinger Date: Thu, 27 Feb 2020 12:52:40 -0500 Subject: [PATCH 7/7] Adding glob matching to name filters Signed-off-by: Greg Sidelinger --- cmd/chart-repo/utils.go | 4 +++- cmd/chart-repo/utils_test.go | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/cmd/chart-repo/utils.go b/cmd/chart-repo/utils.go index f54f804fe..5059be7ca 100644 --- a/cmd/chart-repo/utils.go +++ b/cmd/chart-repo/utils.go @@ -31,6 +31,7 @@ import ( "net/url" "os" "path" + "path/filepath" "strings" "sync" "time" @@ -562,7 +563,8 @@ func filterEntry(entry *helmrepo.ChartVersion, filter *filters) bool { if len(filter.Names) > 0 { for _, n := range filter.Names { - if entry.Name == n { + matched, _ := filepath.Match(n, entry.Name) + if matched { return true } } diff --git a/cmd/chart-repo/utils_test.go b/cmd/chart-repo/utils_test.go index a9fc69052..51f58f0f7 100644 --- a/cmd/chart-repo/utils_test.go +++ b/cmd/chart-repo/utils_test.go @@ -307,6 +307,26 @@ func Test_chartsFromIndexFilterByName(t *testing.T) { assert.Equal(t, len(charts), 1, "number of charts") } +func Test_chartsFromIndexFilterByNameGlobbedSingleChar(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Names = append(filter.Names, "word?ress") + filter.Names = append(filter.Names, "not-found") + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + +func Test_chartsFromIndexFilterByNameGlobbedWildcard(t *testing.T) { + r := repo{Name: "test", URL: "http://testrepo.com"} + index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) + filter := new(filters) + filter.Names = append(filter.Names, "word*") + filter.Names = append(filter.Names, "not-found") + charts := chartsFromIndex(index, r, filter) + assert.Equal(t, len(charts), 1, "number of charts") +} + func Test_chartsFromIndexFilterByAnnotationWithValue(t *testing.T) { r := repo{Name: "test", URL: "http://testrepo.com"} index, _ := parseRepoIndex([]byte(validRepoIndexYAML))