From 5aeaa6734dee4a7aeec136947ab74947300e65c4 Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Tue, 23 Apr 2019 13:24:25 -0500 Subject: [PATCH] Load previous mixin feed data when generating --- pkg/mixin/feed/feed.go | 3 +- pkg/mixin/feed/generate.go | 28 +++++++++++++-- pkg/mixin/feed/generate_test.go | 42 +++++++++++++++++++++-- pkg/mixin/feed/load.go | 7 ++-- pkg/mixin/feed/testdata/atom-existing.xml | 32 +++++++++++++++++ pkg/porter/mixins.go | 8 +++-- 6 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 pkg/mixin/feed/testdata/atom-existing.xml diff --git a/pkg/mixin/feed/feed.go b/pkg/mixin/feed/feed.go index 0c5923589..760dc33c5 100644 --- a/pkg/mixin/feed/feed.go +++ b/pkg/mixin/feed/feed.go @@ -23,6 +23,7 @@ type MixinFeed struct { func NewMixinFeed(cxt *context.Context) *MixinFeed { return &MixinFeed{ + Index: make(map[string]map[string]*MixinFileset), Context:cxt, } } @@ -39,7 +40,7 @@ func (feed *MixinFeed) Search(mixin string, version string) *MixinFileset { type MixinFileset struct { Mixin string Version string - Files []MixinFile + Files []*MixinFile } func (f *MixinFileset) FindDownloadURL(os string, arch string) *url.URL { diff --git a/pkg/mixin/feed/generate.go b/pkg/mixin/feed/generate.go index 282a4740d..888474c28 100644 --- a/pkg/mixin/feed/generate.go +++ b/pkg/mixin/feed/generate.go @@ -53,9 +53,20 @@ func (o *GenerateOptions) ValidateTemplateFile(cxt *context.Context) error { } func (feed *MixinFeed) Generate(opts GenerateOptions) error { + // Check if the atom file already exists, and load in the existing data first + existingFeed, err := feed.FileSystem.Exists(opts.AtomFile) + if err != nil { + return err + } + if existingFeed { + err := feed.Load(opts.AtomFile) + if err != nil { + return err + } + } + mixinRegex := regexp.MustCompile(`(.*/)?(.+)/([a-z]+)-(linux|windows|darwin)-(amd64)(\.exe)?`) - feed.Index = make(map[string]map[string]*MixinFileset) return feed.FileSystem.Walk(opts.SearchDirectory, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -66,6 +77,7 @@ func (feed *MixinFeed) Generate(opts GenerateOptions) error { version := matches[2] mixin := matches[3] filename := info.Name() + updated := info.ModTime() versions, ok := feed.Index[mixin] if !ok { @@ -81,7 +93,17 @@ func (feed *MixinFeed) Generate(opts GenerateOptions) error { } versions[version] = fileset } - fileset.Files = append(fileset.Files, MixinFile{File: filename, Updated: info.ModTime()}) + + // Check if the file is already in the feed + for _, file := range fileset.Files { + // The file is already in the feed, bump the timestamp and move on + if file.File == filename && file.Updated.After(updated) { + file.Updated = updated + return nil + } + } + // Add the file to the feed's index + fileset.Files = append(fileset.Files, &MixinFile{File: filename, Updated: updated}) } return nil @@ -105,6 +127,8 @@ func (feed *MixinFeed) Save(opts GenerateOptions) error { } sort.Sort(sort.Reverse(entries)) + sort.Strings(mixins) + tmplData["Mixins"] = mixins tmplData["Entries"] = entries tmplData["Updated"] = entries[0].Updated() diff --git a/pkg/mixin/feed/generate_test.go b/pkg/mixin/feed/generate_test.go index d0d426d3e..49c123d97 100644 --- a/pkg/mixin/feed/generate_test.go +++ b/pkg/mixin/feed/generate_test.go @@ -66,6 +66,42 @@ func TestGenerate(t *testing.T) { assert.Equal(t, wantXml, gotXml) } +func TestGenerate_ExistingFeed(t *testing.T) { + tc := context.NewTestContext(t) + tc.AddTestFile("testdata/atom-template.xml", "template.xml") + tc.AddTestFile("testdata/atom-existing.xml", "atom.xml") + + tc.FileSystem.Create("bin/v1.2.4/helm-darwin-amd64") + tc.FileSystem.Create("bin/v1.2.4/helm-linux-amd64") + tc.FileSystem.Create("bin/v1.2.4/helm-windows-amd64.exe") + + up4, _ := time.Parse("2006-Jan-02", "2013-Feb-04") + tc.FileSystem.Chtimes("bin/v1.2.4/helm-darwin-amd64", up4, up4) + tc.FileSystem.Chtimes("bin/v1.2.4/helm-linux-amd64", up4, up4) + tc.FileSystem.Chtimes("bin/v1.2.4/helm-windows-amd64.exe", up4, up4) + + opts := GenerateOptions{ + AtomFile: "atom.xml", + SearchDirectory: "bin", + TemplateFile: "template.xml", + } + f := NewMixinFeed(tc.Context) + err := f.Generate(opts) + require.NoError(t, err) + err = f.Save(opts) + require.NoError(t, err) + + b, err := tc.FileSystem.ReadFile("atom.xml") + require.NoError(t, err) + gotXml := string(b) + + b, err = ioutil.ReadFile("testdata/atom.xml") + require.NoError(t, err) + wantXml := string(b) + + assert.Equal(t, wantXml, gotXml) +} + func TestMixinEntries_Sort(t *testing.T) { up2, _ := time.Parse("2006-Jan-02", "2013-Feb-02") up3, _ := time.Parse("2006-Jan-02", "2013-Feb-03") @@ -73,17 +109,17 @@ func TestMixinEntries_Sort(t *testing.T) { entries := MixinEntries{ { - Files: []MixinFile{ + Files: []*MixinFile{ {Updated: up3}, }, }, { - Files: []MixinFile{ + Files: []*MixinFile{ {Updated: up2}, }, }, { - Files: []MixinFile{ + Files: []*MixinFile{ {Updated: up4}, }, }, diff --git a/pkg/mixin/feed/load.go b/pkg/mixin/feed/load.go index 9e8636897..dabbc0de4 100644 --- a/pkg/mixin/feed/load.go +++ b/pkg/mixin/feed/load.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "net/url" + "path" "github.com/mmcdole/gofeed/atom" "github.com/pkg/errors" @@ -30,7 +31,6 @@ func (feed *MixinFeed) Load(file string) error { feed.Mixins = append(feed.Mixins, category.Term) } - feed.Index = make(map[string]map[string]*MixinFileset) for _, entry := range atomFeed.Entries { fileset := &MixinFileset{} @@ -56,7 +56,7 @@ func (feed *MixinFeed) Load(file string) error { continue } - fileset.Files = make([]MixinFile, 0, len(entry.Links)) + fileset.Files = make([]*MixinFile, 0, len(entry.Links)) for _, link := range entry.Links { if link.Rel == "download" { if entry.UpdatedParsed == nil { @@ -74,9 +74,10 @@ func (feed *MixinFeed) Load(file string) error { continue } - file := MixinFile{ + file := &MixinFile{ URL: parsedUrl, Updated: *entry.UpdatedParsed, + File: path.Base(parsedUrl.Path), } fileset.Files = append(fileset.Files, file) } diff --git a/pkg/mixin/feed/testdata/atom-existing.xml b/pkg/mixin/feed/testdata/atom-existing.xml new file mode 100644 index 000000000..0566636db --- /dev/null +++ b/pkg/mixin/feed/testdata/atom-existing.xml @@ -0,0 +1,32 @@ + + https://porter.sh/mixins + DeisLabs Mixins + 2013-02-03T00:00:00Z + + + DeisLabs + https://deislabs.io + + + + + https://porter.sh/mixins/v1.2.3/helm + helm @ v1.2.3 + 2013-02-03T00:00:00Z + + v1.2.3 + + + + + + https://porter.sh/mixins/v1.2.3/exec + exec @ v1.2.3 + 2013-02-02T00:00:00Z + + v1.2.3 + + + + + \ No newline at end of file diff --git a/pkg/porter/mixins.go b/pkg/porter/mixins.go index 159ff2571..66b7341f5 100644 --- a/pkg/porter/mixins.go +++ b/pkg/porter/mixins.go @@ -61,12 +61,14 @@ func (p *Porter) InstallMixin(opts mixin.InstallOptions) error { } func (p *Porter) GenerateMixinFeed(opts feed.GenerateOptions) error { - f := feed.MixinFeed{} - err := f.Generate(opts, p.Context) + f := feed.NewMixinFeed(p.Context) + + err := f.Generate(opts) if err != nil { return err } - return f.Save(opts, p.Context) + + return f.Save(opts) } func (p *Porter) CreateMixinFeedTemplate() error {