From 7be3b74de7deee938fb7ac049f294e431a5cbf95 Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Tue, 23 Apr 2019 13:23:56 -0500 Subject: [PATCH 1/2] Add context to the MixinFeed This is how we organize all the other porter structs so this is just to make things more consistent and easier as we add functionality to the mixin feeds. --- pkg/mixin/feed/feed.go | 9 +++++++++ pkg/mixin/feed/generate.go | 12 ++++++------ pkg/mixin/feed/generate_test.go | 6 +++--- pkg/mixin/feed/load.go | 29 ++++++++++++++--------------- pkg/mixin/provider/install.go | 4 ++-- 5 files changed, 34 insertions(+), 26 deletions(-) diff --git a/pkg/mixin/feed/feed.go b/pkg/mixin/feed/feed.go index 7e8bbe598..0c5923589 100644 --- a/pkg/mixin/feed/feed.go +++ b/pkg/mixin/feed/feed.go @@ -2,12 +2,15 @@ package feed import ( "fmt" + "github.com/deislabs/porter/pkg/context" "net/url" "strings" "time" ) type MixinFeed struct { + *context.Context + // Index of mixin files Index map[string]map[string]*MixinFileset @@ -18,6 +21,12 @@ type MixinFeed struct { Updated *time.Time } +func NewMixinFeed(cxt *context.Context) *MixinFeed { + return &MixinFeed{ + Context:cxt, + } +} + func (feed *MixinFeed) Search(mixin string, version string) *MixinFileset { versions, ok := feed.Index[mixin] if !ok { diff --git a/pkg/mixin/feed/generate.go b/pkg/mixin/feed/generate.go index 34f9256a6..282a4740d 100644 --- a/pkg/mixin/feed/generate.go +++ b/pkg/mixin/feed/generate.go @@ -52,11 +52,11 @@ func (o *GenerateOptions) ValidateTemplateFile(cxt *context.Context) error { return nil } -func (feed *MixinFeed) Generate(opts GenerateOptions, cxt *context.Context) error { +func (feed *MixinFeed) Generate(opts GenerateOptions) error { mixinRegex := regexp.MustCompile(`(.*/)?(.+)/([a-z]+)-(linux|windows|darwin)-(amd64)(\.exe)?`) feed.Index = make(map[string]map[string]*MixinFileset) - return cxt.FileSystem.Walk(opts.SearchDirectory, func(path string, info os.FileInfo, err error) error { + return feed.FileSystem.Walk(opts.SearchDirectory, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -88,8 +88,8 @@ func (feed *MixinFeed) Generate(opts GenerateOptions, cxt *context.Context) erro }) } -func (feed *MixinFeed) Save(opts GenerateOptions, cxt *context.Context) error { - feedTmpl, err := cxt.FileSystem.ReadFile(opts.TemplateFile) +func (feed *MixinFeed) Save(opts GenerateOptions) error { + feedTmpl, err := feed.FileSystem.ReadFile(opts.TemplateFile) if err != nil { return errors.Wrapf(err, "error reading template file at %s", opts.TemplateFile) } @@ -110,12 +110,12 @@ func (feed *MixinFeed) Save(opts GenerateOptions, cxt *context.Context) error { tmplData["Updated"] = entries[0].Updated() atomXml, err := mustache.Render(string(feedTmpl), tmplData) - err = cxt.FileSystem.WriteFile(opts.AtomFile, []byte(atomXml), 0644) + err = feed.FileSystem.WriteFile(opts.AtomFile, []byte(atomXml), 0644) if err != nil { return errors.Wrapf(err, "could not write feed to %s", opts.AtomFile) } - fmt.Fprintf(cxt.Out, "wrote feed to %s\n", opts.AtomFile) + fmt.Fprintf(feed.Out, "wrote feed to %s\n", opts.AtomFile) return nil } diff --git a/pkg/mixin/feed/generate_test.go b/pkg/mixin/feed/generate_test.go index c7299076b..d0d426d3e 100644 --- a/pkg/mixin/feed/generate_test.go +++ b/pkg/mixin/feed/generate_test.go @@ -49,10 +49,10 @@ func TestGenerate(t *testing.T) { SearchDirectory: "bin", TemplateFile: "template.xml", } - f := MixinFeed{} - err := f.Generate(opts, tc.Context) + f := NewMixinFeed(tc.Context) + err := f.Generate(opts) require.NoError(t, err) - err = f.Save(opts, tc.Context) + err = f.Save(opts) require.NoError(t, err) b, err := tc.FileSystem.ReadFile("atom.xml") diff --git a/pkg/mixin/feed/load.go b/pkg/mixin/feed/load.go index 877370817..9e8636897 100644 --- a/pkg/mixin/feed/load.go +++ b/pkg/mixin/feed/load.go @@ -5,13 +5,12 @@ import ( "fmt" "net/url" - "github.com/deislabs/porter/pkg/context" "github.com/mmcdole/gofeed/atom" "github.com/pkg/errors" ) -func (feed *MixinFeed) Load(file string, cxt *context.Context) error { - contents, err := cxt.FileSystem.ReadFile(file) +func (feed *MixinFeed) Load(file string) error { + contents, err := feed.FileSystem.ReadFile(file) if err != nil { return errors.Wrapf(err, "error reading mixin feed at %s", file) } @@ -19,8 +18,8 @@ func (feed *MixinFeed) Load(file string, cxt *context.Context) error { p := atom.Parser{} atomFeed, err := p.Parse(bytes.NewReader(contents)) if err != nil { - if cxt.Debug { - fmt.Fprintln(cxt.Err, string(contents)) + if feed.Debug { + fmt.Fprintln(feed.Err, string(contents)) } return errors.Wrap(err, "error parsing the mixin feed as an atom xml file") } @@ -36,23 +35,23 @@ func (feed *MixinFeed) Load(file string, cxt *context.Context) error { fileset := &MixinFileset{} if len(entry.Categories) == 0 { - if cxt.Debug { - fmt.Fprintf(cxt.Err, "skipping invalid entry %s, missing category (mixin name)", entry.ID) + if feed.Debug { + fmt.Fprintf(feed.Err, "skipping invalid entry %s, missing category (mixin name)", entry.ID) } continue } fileset.Mixin = entry.Categories[0].Term if fileset.Mixin == "" { - if cxt.Debug { - fmt.Fprintf(cxt.Err, "skipping invalid entry %s, empty category (mixin name)", entry.ID) + if feed.Debug { + fmt.Fprintf(feed.Err, "skipping invalid entry %s, empty category (mixin name)", entry.ID) } continue } fileset.Version = entry.Content.Value if fileset.Version == "" { - if cxt.Debug { - fmt.Fprintf(cxt.Err, "skipping invalid entry %s, empty content (version)", entry.ID) + if feed.Debug { + fmt.Fprintf(feed.Err, "skipping invalid entry %s, empty content (version)", entry.ID) } continue } @@ -61,16 +60,16 @@ func (feed *MixinFeed) Load(file string, cxt *context.Context) error { for _, link := range entry.Links { if link.Rel == "download" { if entry.UpdatedParsed == nil { - if cxt.Debug { - fmt.Fprintf(cxt.Err, "skipping invalid entry %s, invalid updated %q could not be parsed as RFC3339", entry.ID, entry.Updated) + if feed.Debug { + fmt.Fprintf(feed.Err, "skipping invalid entry %s, invalid updated %q could not be parsed as RFC3339", entry.ID, entry.Updated) } continue } parsedUrl, err := url.Parse(link.Href) if err != nil || link.Href == "" { - if cxt.Debug { - fmt.Fprintf(cxt.Err, "skipping invalid entry %s, invalid link.href %q", entry.ID, link.Href) + if feed.Debug { + fmt.Fprintf(feed.Err, "skipping invalid entry %s, invalid link.href %q", entry.ID, link.Href) } continue } diff --git a/pkg/mixin/provider/install.go b/pkg/mixin/provider/install.go index 12c9465ed..5273874f6 100644 --- a/pkg/mixin/provider/install.go +++ b/pkg/mixin/provider/install.go @@ -46,8 +46,8 @@ func (p *FileSystem) InstallFromFeedURL(opts mixin.InstallOptions) (mixin.Metada return mixin.Metadata{}, err } - searchFeed := feed.MixinFeed{} - err = searchFeed.Load(feedPath, p.Context) + searchFeed := feed.NewMixinFeed(p.Context) + err = searchFeed.Load(feedPath) if err != nil { return mixin.Metadata{}, err } From 5aeaa6734dee4a7aeec136947ab74947300e65c4 Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Tue, 23 Apr 2019 13:24:25 -0500 Subject: [PATCH 2/2] Load previous mixin feed data when generating --- pkg/mixin/feed/feed.go | 3 +- pkg/mixin/feed/generate.go | 28 +++++++++++++-- pkg/mixin/feed/generate_test.go | 42 +++++++++++++++++++++-- pkg/mixin/feed/load.go | 7 ++-- pkg/mixin/feed/testdata/atom-existing.xml | 32 +++++++++++++++++ pkg/porter/mixins.go | 8 +++-- 6 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 pkg/mixin/feed/testdata/atom-existing.xml diff --git a/pkg/mixin/feed/feed.go b/pkg/mixin/feed/feed.go index 0c5923589..760dc33c5 100644 --- a/pkg/mixin/feed/feed.go +++ b/pkg/mixin/feed/feed.go @@ -23,6 +23,7 @@ type MixinFeed struct { func NewMixinFeed(cxt *context.Context) *MixinFeed { return &MixinFeed{ + Index: make(map[string]map[string]*MixinFileset), Context:cxt, } } @@ -39,7 +40,7 @@ func (feed *MixinFeed) Search(mixin string, version string) *MixinFileset { type MixinFileset struct { Mixin string Version string - Files []MixinFile + Files []*MixinFile } func (f *MixinFileset) FindDownloadURL(os string, arch string) *url.URL { diff --git a/pkg/mixin/feed/generate.go b/pkg/mixin/feed/generate.go index 282a4740d..888474c28 100644 --- a/pkg/mixin/feed/generate.go +++ b/pkg/mixin/feed/generate.go @@ -53,9 +53,20 @@ func (o *GenerateOptions) ValidateTemplateFile(cxt *context.Context) error { } func (feed *MixinFeed) Generate(opts GenerateOptions) error { + // Check if the atom file already exists, and load in the existing data first + existingFeed, err := feed.FileSystem.Exists(opts.AtomFile) + if err != nil { + return err + } + if existingFeed { + err := feed.Load(opts.AtomFile) + if err != nil { + return err + } + } + mixinRegex := regexp.MustCompile(`(.*/)?(.+)/([a-z]+)-(linux|windows|darwin)-(amd64)(\.exe)?`) - feed.Index = make(map[string]map[string]*MixinFileset) return feed.FileSystem.Walk(opts.SearchDirectory, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -66,6 +77,7 @@ func (feed *MixinFeed) Generate(opts GenerateOptions) error { version := matches[2] mixin := matches[3] filename := info.Name() + updated := info.ModTime() versions, ok := feed.Index[mixin] if !ok { @@ -81,7 +93,17 @@ func (feed *MixinFeed) Generate(opts GenerateOptions) error { } versions[version] = fileset } - fileset.Files = append(fileset.Files, MixinFile{File: filename, Updated: info.ModTime()}) + + // Check if the file is already in the feed + for _, file := range fileset.Files { + // The file is already in the feed, bump the timestamp and move on + if file.File == filename && file.Updated.After(updated) { + file.Updated = updated + return nil + } + } + // Add the file to the feed's index + fileset.Files = append(fileset.Files, &MixinFile{File: filename, Updated: updated}) } return nil @@ -105,6 +127,8 @@ func (feed *MixinFeed) Save(opts GenerateOptions) error { } sort.Sort(sort.Reverse(entries)) + sort.Strings(mixins) + tmplData["Mixins"] = mixins tmplData["Entries"] = entries tmplData["Updated"] = entries[0].Updated() diff --git a/pkg/mixin/feed/generate_test.go b/pkg/mixin/feed/generate_test.go index d0d426d3e..49c123d97 100644 --- a/pkg/mixin/feed/generate_test.go +++ b/pkg/mixin/feed/generate_test.go @@ -66,6 +66,42 @@ func TestGenerate(t *testing.T) { assert.Equal(t, wantXml, gotXml) } +func TestGenerate_ExistingFeed(t *testing.T) { + tc := context.NewTestContext(t) + tc.AddTestFile("testdata/atom-template.xml", "template.xml") + tc.AddTestFile("testdata/atom-existing.xml", "atom.xml") + + tc.FileSystem.Create("bin/v1.2.4/helm-darwin-amd64") + tc.FileSystem.Create("bin/v1.2.4/helm-linux-amd64") + tc.FileSystem.Create("bin/v1.2.4/helm-windows-amd64.exe") + + up4, _ := time.Parse("2006-Jan-02", "2013-Feb-04") + tc.FileSystem.Chtimes("bin/v1.2.4/helm-darwin-amd64", up4, up4) + tc.FileSystem.Chtimes("bin/v1.2.4/helm-linux-amd64", up4, up4) + tc.FileSystem.Chtimes("bin/v1.2.4/helm-windows-amd64.exe", up4, up4) + + opts := GenerateOptions{ + AtomFile: "atom.xml", + SearchDirectory: "bin", + TemplateFile: "template.xml", + } + f := NewMixinFeed(tc.Context) + err := f.Generate(opts) + require.NoError(t, err) + err = f.Save(opts) + require.NoError(t, err) + + b, err := tc.FileSystem.ReadFile("atom.xml") + require.NoError(t, err) + gotXml := string(b) + + b, err = ioutil.ReadFile("testdata/atom.xml") + require.NoError(t, err) + wantXml := string(b) + + assert.Equal(t, wantXml, gotXml) +} + func TestMixinEntries_Sort(t *testing.T) { up2, _ := time.Parse("2006-Jan-02", "2013-Feb-02") up3, _ := time.Parse("2006-Jan-02", "2013-Feb-03") @@ -73,17 +109,17 @@ func TestMixinEntries_Sort(t *testing.T) { entries := MixinEntries{ { - Files: []MixinFile{ + Files: []*MixinFile{ {Updated: up3}, }, }, { - Files: []MixinFile{ + Files: []*MixinFile{ {Updated: up2}, }, }, { - Files: []MixinFile{ + Files: []*MixinFile{ {Updated: up4}, }, }, diff --git a/pkg/mixin/feed/load.go b/pkg/mixin/feed/load.go index 9e8636897..dabbc0de4 100644 --- a/pkg/mixin/feed/load.go +++ b/pkg/mixin/feed/load.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "net/url" + "path" "github.com/mmcdole/gofeed/atom" "github.com/pkg/errors" @@ -30,7 +31,6 @@ func (feed *MixinFeed) Load(file string) error { feed.Mixins = append(feed.Mixins, category.Term) } - feed.Index = make(map[string]map[string]*MixinFileset) for _, entry := range atomFeed.Entries { fileset := &MixinFileset{} @@ -56,7 +56,7 @@ func (feed *MixinFeed) Load(file string) error { continue } - fileset.Files = make([]MixinFile, 0, len(entry.Links)) + fileset.Files = make([]*MixinFile, 0, len(entry.Links)) for _, link := range entry.Links { if link.Rel == "download" { if entry.UpdatedParsed == nil { @@ -74,9 +74,10 @@ func (feed *MixinFeed) Load(file string) error { continue } - file := MixinFile{ + file := &MixinFile{ URL: parsedUrl, Updated: *entry.UpdatedParsed, + File: path.Base(parsedUrl.Path), } fileset.Files = append(fileset.Files, file) } diff --git a/pkg/mixin/feed/testdata/atom-existing.xml b/pkg/mixin/feed/testdata/atom-existing.xml new file mode 100644 index 000000000..0566636db --- /dev/null +++ b/pkg/mixin/feed/testdata/atom-existing.xml @@ -0,0 +1,32 @@ + + https://porter.sh/mixins + DeisLabs Mixins + 2013-02-03T00:00:00Z + + + DeisLabs + https://deislabs.io + + + + + https://porter.sh/mixins/v1.2.3/helm + helm @ v1.2.3 + 2013-02-03T00:00:00Z + + v1.2.3 + + + + + + https://porter.sh/mixins/v1.2.3/exec + exec @ v1.2.3 + 2013-02-02T00:00:00Z + + v1.2.3 + + + + + \ No newline at end of file diff --git a/pkg/porter/mixins.go b/pkg/porter/mixins.go index 159ff2571..66b7341f5 100644 --- a/pkg/porter/mixins.go +++ b/pkg/porter/mixins.go @@ -61,12 +61,14 @@ func (p *Porter) InstallMixin(opts mixin.InstallOptions) error { } func (p *Porter) GenerateMixinFeed(opts feed.GenerateOptions) error { - f := feed.MixinFeed{} - err := f.Generate(opts, p.Context) + f := feed.NewMixinFeed(p.Context) + + err := f.Generate(opts) if err != nil { return err } - return f.Save(opts, p.Context) + + return f.Save(opts) } func (p *Porter) CreateMixinFeedTemplate() error {