Update an existing feed when generating a mixin feed from a directory

pkg/mixin/feed/feed.go

@@ -2,12 +2,15 @@ package feed
 
 import (
 	"fmt"
+	"github.com/deislabs/porter/pkg/context"
 	"net/url"
 	"strings"
 	"time"
 )
 
 type MixinFeed struct {
+	*context.Context
+
 	// Index of mixin files
 	Index map[string]map[string]*MixinFileset
 
@@ -18,6 +21,13 @@ type MixinFeed struct {
 	Updated *time.Time
 }
 
+func NewMixinFeed(cxt *context.Context) *MixinFeed {
+	return &MixinFeed{
+		Index: make(map[string]map[string]*MixinFileset),
+		Context:cxt,
+	}
+}
+
 func (feed *MixinFeed) Search(mixin string, version string) *MixinFileset {
 	versions, ok := feed.Index[mixin]
 	if !ok {
@@ -30,7 +40,7 @@ func (feed *MixinFeed) Search(mixin string, version string) *MixinFileset {
 type MixinFileset struct {
 	Mixin   string
 	Version string
-	Files   []MixinFile
+	Files   []*MixinFile
 }
 
 func (f *MixinFileset) FindDownloadURL(os string, arch string) *url.URL {

pkg/mixin/feed/generate.go

@@ -52,11 +52,22 @@ func (o *GenerateOptions) ValidateTemplateFile(cxt *context.Context) error {
 	return nil
 }
 
-func (feed *MixinFeed) Generate(opts GenerateOptions, cxt *context.Context) error {
+func (feed *MixinFeed) Generate(opts GenerateOptions) error {
+	// Check if the atom file already exists, and load in the existing data first
+	existingFeed, err := feed.FileSystem.Exists(opts.AtomFile)
+	if err != nil {
+		return err
+	}
+	if existingFeed {
+		err := feed.Load(opts.AtomFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	mixinRegex := regexp.MustCompile(`(.*/)?(.+)/([a-z]+)-(linux|windows|darwin)-(amd64)(\.exe)?`)
 
-	feed.Index = make(map[string]map[string]*MixinFileset)
-	return cxt.FileSystem.Walk(opts.SearchDirectory, func(path string, info os.FileInfo, err error) error {
+	return feed.FileSystem.Walk(opts.SearchDirectory, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -66,6 +77,7 @@ func (feed *MixinFeed) Generate(opts GenerateOptions, cxt *context.Context) erro
 			version := matches[2]
 			mixin := matches[3]
 			filename := info.Name()
+			updated := info.ModTime()
 
 			versions, ok := feed.Index[mixin]
 			if !ok {
@@ -81,15 +93,25 @@ func (feed *MixinFeed) Generate(opts GenerateOptions, cxt *context.Context) erro
 				}
 				versions[version] = fileset
 			}
-			fileset.Files = append(fileset.Files, MixinFile{File: filename, Updated: info.ModTime()})
+
+			// Check if the file is already in the feed
+			for _, file := range fileset.Files {
+				// The file is already in the feed, bump the timestamp and move on
+				if file.File == filename && file.Updated.After(updated) {
+					file.Updated = updated
+					return nil
+				}
+			}
+			// Add the file to the feed's index
+			fileset.Files = append(fileset.Files, &MixinFile{File: filename, Updated: updated})
 		}
 
 		return nil
 	})
 }
 
-func (feed *MixinFeed) Save(opts GenerateOptions, cxt *context.Context) error {
-	feedTmpl, err := cxt.FileSystem.ReadFile(opts.TemplateFile)
+func (feed *MixinFeed) Save(opts GenerateOptions) error {
+	feedTmpl, err := feed.FileSystem.ReadFile(opts.TemplateFile)
 	if err != nil {
 		return errors.Wrapf(err, "error reading template file at %s", opts.TemplateFile)
 	}
@@ -105,17 +127,19 @@ func (feed *MixinFeed) Save(opts GenerateOptions, cxt *context.Context) error {
 	}
 	sort.Sort(sort.Reverse(entries))
 
+	sort.Strings(mixins)
+
 	tmplData["Mixins"] = mixins
 	tmplData["Entries"] = entries
 	tmplData["Updated"] = entries[0].Updated()
 
 	atomXml, err := mustache.Render(string(feedTmpl), tmplData)
-	err = cxt.FileSystem.WriteFile(opts.AtomFile, []byte(atomXml), 0644)
+	err = feed.FileSystem.WriteFile(opts.AtomFile, []byte(atomXml), 0644)
 	if err != nil {
 		return errors.Wrapf(err, "could not write feed to %s", opts.AtomFile)
 	}
 
-	fmt.Fprintf(cxt.Out, "wrote feed to %s\n", opts.AtomFile)
+	fmt.Fprintf(feed.Out, "wrote feed to %s\n", opts.AtomFile)
 	return nil
 }
 

pkg/mixin/feed/generate_test.go

@@ -49,10 +49,46 @@ func TestGenerate(t *testing.T) {
 		SearchDirectory: "bin",
 		TemplateFile:    "template.xml",
 	}
-	f := MixinFeed{}
-	err := f.Generate(opts, tc.Context)
+	f := NewMixinFeed(tc.Context)
+	err := f.Generate(opts)
 	require.NoError(t, err)
-	err = f.Save(opts, tc.Context)
+	err = f.Save(opts)
+	require.NoError(t, err)
+
+	b, err := tc.FileSystem.ReadFile("atom.xml")
+	require.NoError(t, err)
+	gotXml := string(b)
+
+	b, err = ioutil.ReadFile("testdata/atom.xml")
+	require.NoError(t, err)
+	wantXml := string(b)
+
+	assert.Equal(t, wantXml, gotXml)
+}
+
+func TestGenerate_ExistingFeed(t *testing.T) {
+	tc := context.NewTestContext(t)
+	tc.AddTestFile("testdata/atom-template.xml", "template.xml")
+	tc.AddTestFile("testdata/atom-existing.xml", "atom.xml")
+
+	tc.FileSystem.Create("bin/v1.2.4/helm-darwin-amd64")
+	tc.FileSystem.Create("bin/v1.2.4/helm-linux-amd64")
+	tc.FileSystem.Create("bin/v1.2.4/helm-windows-amd64.exe")
+
+	up4, _ := time.Parse("2006-Jan-02", "2013-Feb-04")
+	tc.FileSystem.Chtimes("bin/v1.2.4/helm-darwin-amd64", up4, up4)
+	tc.FileSystem.Chtimes("bin/v1.2.4/helm-linux-amd64", up4, up4)
+	tc.FileSystem.Chtimes("bin/v1.2.4/helm-windows-amd64.exe", up4, up4)
+
+	opts := GenerateOptions{
+		AtomFile:        "atom.xml",
+		SearchDirectory: "bin",
+		TemplateFile:    "template.xml",
+	}
+	f := NewMixinFeed(tc.Context)
+	err := f.Generate(opts)
+	require.NoError(t, err)
+	err = f.Save(opts)
 	require.NoError(t, err)
 
 	b, err := tc.FileSystem.ReadFile("atom.xml")
@@ -73,17 +109,17 @@ func TestMixinEntries_Sort(t *testing.T) {
 
 	entries := MixinEntries{
 		{
-			Files: []MixinFile{
+			Files: []*MixinFile{
 				{Updated: up3},
 			},
 		},
 		{
-			Files: []MixinFile{
+			Files: []*MixinFile{
 				{Updated: up2},
 			},
 		},
 		{
-			Files: []MixinFile{
+			Files: []*MixinFile{
 				{Updated: up4},
 			},
 		},

pkg/mixin/feed/load.go

@@ -4,23 +4,23 @@ import (
 	"bytes"
 	"fmt"
 	"net/url"
+	"path"
 
-	"github.com/deislabs/porter/pkg/context"
 	"github.com/mmcdole/gofeed/atom"
 	"github.com/pkg/errors"
 )
 
-func (feed *MixinFeed) Load(file string, cxt *context.Context) error {
-	contents, err := cxt.FileSystem.ReadFile(file)
+func (feed *MixinFeed) Load(file string) error {
+	contents, err := feed.FileSystem.ReadFile(file)
 	if err != nil {
 		return errors.Wrapf(err, "error reading mixin feed at %s", file)
 	}
 
 	p := atom.Parser{}
 	atomFeed, err := p.Parse(bytes.NewReader(contents))
 	if err != nil {
-		if cxt.Debug {
-			fmt.Fprintln(cxt.Err, string(contents))
+		if feed.Debug {
+			fmt.Fprintln(feed.Err, string(contents))
 		}
 		return errors.Wrap(err, "error parsing the mixin feed as an atom xml file")
 	}
@@ -31,53 +31,53 @@ func (feed *MixinFeed) Load(file string, cxt *context.Context) error {
 		feed.Mixins = append(feed.Mixins, category.Term)
 	}
 
-	feed.Index = make(map[string]map[string]*MixinFileset)
 	for _, entry := range atomFeed.Entries {
 		fileset := &MixinFileset{}
 
 		if len(entry.Categories) == 0 {
-			if cxt.Debug {
-				fmt.Fprintf(cxt.Err, "skipping invalid entry %s, missing category (mixin name)", entry.ID)
+			if feed.Debug {
+				fmt.Fprintf(feed.Err, "skipping invalid entry %s, missing category (mixin name)", entry.ID)
 			}
 			continue
 		}
 		fileset.Mixin = entry.Categories[0].Term
 		if fileset.Mixin == "" {
-			if cxt.Debug {
-				fmt.Fprintf(cxt.Err, "skipping invalid entry %s, empty category (mixin name)", entry.ID)
+			if feed.Debug {
+				fmt.Fprintf(feed.Err, "skipping invalid entry %s, empty category (mixin name)", entry.ID)
 			}
 			continue
 		}
 
 		fileset.Version = entry.Content.Value
 		if fileset.Version == "" {
-			if cxt.Debug {
-				fmt.Fprintf(cxt.Err, "skipping invalid entry %s, empty content (version)", entry.ID)
+			if feed.Debug {
+				fmt.Fprintf(feed.Err, "skipping invalid entry %s, empty content (version)", entry.ID)
 			}
 			continue
 		}
 
-		fileset.Files = make([]MixinFile, 0, len(entry.Links))
+		fileset.Files = make([]*MixinFile, 0, len(entry.Links))
 		for _, link := range entry.Links {
 			if link.Rel == "download" {
 				if entry.UpdatedParsed == nil {
-					if cxt.Debug {
-						fmt.Fprintf(cxt.Err, "skipping invalid entry %s, invalid updated %q could not be parsed as RFC3339", entry.ID, entry.Updated)
+					if feed.Debug {
+						fmt.Fprintf(feed.Err, "skipping invalid entry %s, invalid updated %q could not be parsed as RFC3339", entry.ID, entry.Updated)
 					}
 					continue
 				}
 
 				parsedUrl, err := url.Parse(link.Href)
 				if err != nil || link.Href == "" {
-					if cxt.Debug {
-						fmt.Fprintf(cxt.Err, "skipping invalid entry %s, invalid link.href %q", entry.ID, link.Href)
+					if feed.Debug {
+						fmt.Fprintf(feed.Err, "skipping invalid entry %s, invalid link.href %q", entry.ID, link.Href)
 					}
 					continue
 				}
 
-				file := MixinFile{
+				file := &MixinFile{
 					URL:     parsedUrl,
 					Updated: *entry.UpdatedParsed,
+					File: path.Base(parsedUrl.Path),
 				}
 				fileset.Files = append(fileset.Files, file)
 			}

pkg/mixin/feed/testdata/atom-existing.xml

@@ -0,0 +1,32 @@
+<feed xmlns="http://www.w3.org/2005/Atom">
+    <id>https://porter.sh/mixins</id>
+    <title>DeisLabs Mixins</title>
+    <updated>2013-02-03T00:00:00Z</updated>
+    <link rel="self" href="https://porter.sh/mixins/atom.xml"/>
+    <author>
+        <name>DeisLabs</name>
+        <uri>https://deislabs.io</uri>
+    </author>
+    <category term="exec"/>
+    <category term="helm"/>
+    <entry>
+        <id>https://porter.sh/mixins/v1.2.3/helm</id>
+        <title>helm @ v1.2.3</title>
+        <updated>2013-02-03T00:00:00Z</updated>
+        <category term="helm"/>
+        <content>v1.2.3</content>
+        <link rel="download" href="https://porter.sh/mixins/v1.2.3/helm-darwin-amd64" />
+        <link rel="download" href="https://porter.sh/mixins/v1.2.3/helm-linux-amd64" />
+        <link rel="download" href="https://porter.sh/mixins/v1.2.3/helm-windows-amd64.exe" />
+    </entry>
+    <entry>
+        <id>https://porter.sh/mixins/v1.2.3/exec</id>
+        <title>exec @ v1.2.3</title>
+        <updated>2013-02-02T00:00:00Z</updated>
+        <category term="exec"/>
+        <content>v1.2.3</content>
+        <link rel="download" href="https://porter.sh/mixins/v1.2.3/exec-darwin-amd64" />
+        <link rel="download" href="https://porter.sh/mixins/v1.2.3/exec-linux-amd64" />
+        <link rel="download" href="https://porter.sh/mixins/v1.2.3/exec-windows-amd64.exe" />
+    </entry>
+</feed>

pkg/mixin/provider/install.go

@@ -46,8 +46,8 @@ func (p *FileSystem) InstallFromFeedURL(opts mixin.InstallOptions) (mixin.Metada
 		return mixin.Metadata{}, err
 	}
 
-	searchFeed := feed.MixinFeed{}
-	err = searchFeed.Load(feedPath, p.Context)
+	searchFeed := feed.NewMixinFeed(p.Context)
+	err = searchFeed.Load(feedPath)
 	if err != nil {
 		return mixin.Metadata{}, err
 	}

pkg/porter/mixins.go

@@ -61,12 +61,14 @@ func (p *Porter) InstallMixin(opts mixin.InstallOptions) error {
 }
 
 func (p *Porter) GenerateMixinFeed(opts feed.GenerateOptions) error {
-	f := feed.MixinFeed{}
-	err := f.Generate(opts, p.Context)
+	f := feed.NewMixinFeed(p.Context)
+
+	err := f.Generate(opts)
 	if err != nil {
 		return err
 	}
-	return f.Save(opts, p.Context)
+
+	return f.Save(opts)
 }
 
 func (p *Porter) CreateMixinFeedTemplate() error {