diff --git a/go.mod b/go.mod index e8f7b6d..c12889a 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,17 @@ module github.com/FileFormatInfo/fflint -go 1.18 +go 1.22 require ( github.com/JoshVarga/svgparser v0.0.0-20200804023048-5eaba627a7d1 github.com/adrg/frontmatter v0.2.0 github.com/antchfx/xmlquery v1.3.3 github.com/bmatcuk/doublestar/v4 v4.0.2 + github.com/cevaris/ordered_map v0.0.0-20220813181356-34664b69742b github.com/cheggaaa/pb/v3 v3.0.8 github.com/mattn/go-isatty v0.0.18 github.com/mitchellh/go-homedir v1.1.0 + github.com/mmcdole/gofeed v1.3.0 github.com/muesli/mango-cobra v1.2.0 github.com/muesli/roff v0.1.0 github.com/olekukonko/tablewriter v0.0.5 @@ -22,22 +24,28 @@ require ( github.com/zyxar/image2ascii v0.0.0-20180912034614-460a04e371ae golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/net v0.14.0 + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/BurntSushi/toml v1.2.1 // indirect + github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/VividCortex/ewma v1.1.1 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/antchfx/xpath v1.2.0 // indirect - github.com/cevaris/ordered_map v0.0.0-20220813181356-34664b69742b // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fatih/color v1.13.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/muesli/mango v0.1.0 // indirect github.com/muesli/mango-pflag v0.1.0 // indirect github.com/rivo/uniseg v0.4.2 // indirect @@ -46,5 +54,4 @@ require ( golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 3d9e412..c5cd403 100644 --- a/go.sum +++ b/go.sum @@ -3,10 +3,14 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/JoshVarga/svgparser v0.0.0-20200804023048-5eaba627a7d1 h1:RAQocNl+YQYGPt5yh4SR5zFUIHKrXnLhjIGhHO4Vwnc= github.com/JoshVarga/svgparser v0.0.0-20200804023048-5eaba627a7d1/go.mod h1:tMmgUTWcco9d1ZmK7zjxuTv7XWZhyutXIsgu0uJ3gDw= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/adrg/frontmatter v0.2.0 h1:/DgnNe82o03riBd1S+ZDjd43wAmC6W35q67NHeLkPd4= github.com/adrg/frontmatter v0.2.0/go.mod h1:93rQCj3z3ZlwyxxpQioRKC1wDLto4aXHrbqIsnH9wmE= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/antchfx/xmlquery v1.3.3 h1:HYmadPG0uz8CySdL68rB4DCLKXz2PurCjS3mnkVF4CQ= github.com/antchfx/xmlquery v1.3.3/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc= github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= @@ -30,8 +34,11 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk= github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -56,6 +63,15 @@ github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWV github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= +github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/muesli/mango v0.1.0 h1:DZQK45d2gGbql1arsYA4vfg4d7I9Hfx5rX/GCmzsAvI= github.com/muesli/mango v0.1.0/go.mod h1:5XFpbC8jY5UUv89YQciiXNlbi+iJgt29VDC5xbzrLL4= github.com/muesli/mango-cobra v1.2.0 h1:DQvjzAM0PMZr85Iv9LIMaYISpTOliMEg+uMFtNbYvWg= @@ -91,6 +107,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -103,6 +120,7 @@ golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZ golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -110,15 +128,20 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/command/all.go b/internal/command/all.go index 792081b..db23744 100644 --- a/internal/command/all.go +++ b/internal/command/all.go @@ -5,6 +5,7 @@ import "github.com/spf13/cobra" func AddAllCommands(rootCmd *cobra.Command) { AddExtCommand(rootCmd) + AddFeedCommand(rootCmd) AddFrontmatterCommand(rootCmd) AddHtmlCommand(rootCmd) AddIcoCommand(rootCmd) diff --git a/internal/command/feed.go b/internal/command/feed.go new file mode 100644 index 0000000..13dea6a --- /dev/null +++ b/internal/command/feed.go @@ -0,0 +1,223 @@ +package command + +import ( + "bytes" + "fmt" + "net/url" + + "github.com/FileFormatInfo/fflint/internal/argtype" + "github.com/FileFormatInfo/fflint/internal/shared" + "github.com/mmcdole/gofeed" + "github.com/spf13/cobra" +) + +var ( + feedFormat = argtype.NewStringSet("Feed file format", "auto", []string{"atom", "auto", "jsonfeed", "rss"}) + feedStrict bool +) + +// xmlCmd represents the xml command +var feedCmd = &cobra.Command{ + Args: cobra.MinimumNArgs(1), + Use: "feed [options] files...", + Short: "Validate feeds (RSS/Atom/Jsonfeed)", + Long: `Checks that your feeds are valid. RSS, Atom and JSONFeed are supported.`, + PreRunE: feedInit, + RunE: shared.MakeFileCommand(feedCheck), + PostRunE: feedCleanup, +} + +func AddFeedCommand(rootCmd *cobra.Command) { + rootCmd.AddCommand(feedCmd) + feedCmd.Flags().Var(&feedFormat, "format", feedFormat.HelpText()) + feedCmd.Flags().BoolVar(&feedStrict, "strict", true, "Check contents in addition to parsability") +} + +func feedCheck(f *shared.FileContext) { + + data, readErr := f.ReadFile() + if readErr != nil { + f.RecordResult("fileRead", false, map[string]interface{}{ + "error": readErr, + }) + return + } + var parseErr error + var feed *gofeed.Feed + + detectedFeedType := gofeed.DetectFeedType(bytes.NewReader(data)) + if feedFormat.String() == "auto" { + if detectedFeedType == gofeed.FeedTypeUnknown { + f.RecordResult("feedDetectType", false, map[string]interface{}{ + "error": "Unknown feed type", + "startOfInput": Substr(string(data), 0, 100), + }) + return + } + } else if (feedFormat.String() == "rss" && detectedFeedType != gofeed.FeedTypeRSS) || + (feedFormat.String() == "atom" && detectedFeedType != gofeed.FeedTypeAtom) || + (feedFormat.String() == "jsonfeed" && detectedFeedType != gofeed.FeedTypeJSON) { + f.RecordResult("feedTypeMismatch", false, map[string]interface{}{ + "error": "Feed type mismatch", + "detected": detectedFeedType, + "expected": feedFormat.String(), + }) + return + } + + p := gofeed.NewParser() + feed, parseErr = p.Parse(bytes.NewReader(data)) + + if parseErr != nil { + f.RecordResult("feedParse", false, map[string]interface{}{ + "error": parseErr, + }) + return + } + + if feed == nil { + f.RecordResult("feedEmpty", false, map[string]interface{}{ + "error": "Empty feed", + }) + return + } + + if !feedStrict { + return + } + + if feed.Title == "" { + f.RecordResult("feedTitle", false, map[string]interface{}{ + "error": "Missing title", + }) + } + + if feed.Description == "" { + f.RecordResult("feedTitle", false, map[string]interface{}{ + "error": "Missing title", + }) + } + + if feedParentLinkErr := IsValidUrl(feed.Link); feedParentLinkErr != nil { + f.RecordResult("feedParentLink", false, map[string]interface{}{ + "error": feedParentLinkErr, + "url": feed.Link, + }) + } + + if feedSelfLinkErr := IsValidUrl(feed.FeedLink); feedSelfLinkErr != nil { + f.RecordResult("feedSelfLink", false, map[string]interface{}{ + "error": feedSelfLinkErr, + "url": feed.FeedLink, + }) + } + + if feed.Updated != "" && feed.UpdatedParsed == nil { + f.RecordResult("feedUpdated", false, map[string]interface{}{ + "error": "Invalid updated date", + "rawdate": feed.Updated, + }) + } + + if feed.Published != "" && feed.PublishedParsed == nil { + f.RecordResult("feedPublished", false, map[string]interface{}{ + "error": "Invalid published date", + "rawdate": feed.Published, + }) + } + + if feed.Items == nil || len(feed.Items) == 0 { + f.RecordResult("feedItems", false, map[string]interface{}{ + "error": "No items found", + }) + } else { + guidMap := make(map[string]int) + for i, item := range feed.Items { + if item.Title == "" { + f.RecordResult("feedItemTitle", false, map[string]interface{}{ + "error": "Missing title", + "index": i, + }) + } + if item.Description == "" { + f.RecordResult("feedItemDescription", false, map[string]interface{}{ + "error": "Missing description", + "index": i, + }) + } + if item.Link == "" { + f.RecordResult("feedItemLink", false, map[string]interface{}{ + "error": "Missing link", + "index": i, + }) + } + if item.Published != "" && item.PublishedParsed == nil { + f.RecordResult("feedItemPublished", false, map[string]interface{}{ + "error": "Invalid published date", + "index": i, + "rawdate": item.Published, + }) + } + if item.Updated != "" && item.UpdatedParsed == nil { + f.RecordResult("feedItemUpdated", false, map[string]interface{}{ + "error": "Invalid updated date", + "index": i, + "rawdate": item.Updated, + }) + } + if item.GUID == "" { + f.RecordResult("feedItemGUID", false, map[string]interface{}{ + "error": "Missing GUID", + "index": i, + }) + } else if originalIndex, ok := guidMap[item.GUID]; ok { + f.RecordResult("feedItemGUID", false, map[string]interface{}{ + "error": "Duplicate GUID", + "originalIndex": originalIndex, + "duplicateIndex": i, + "guid": item.GUID, + }) + } else { + guidMap[item.GUID] = i + } + } + } +} + +func feedInit(cmd *cobra.Command, args []string) error { + return nil +} + +func feedCleanup(cmd *cobra.Command, args []string) error { + return nil +} + +func IsValidUrl(target string) error { + if target == "" { + return fmt.Errorf("URL not set") + } + _, err := url.ParseRequestURI(target) + if err != nil { + return err + } + return nil +} + +// UTF8-safe substring +func Substr(input string, start int, length int) string { + + if start == 0 && length >= len(input) { + return input + } + + asRunes := []rune(input) + if start >= len(asRunes) { + return "" + } + + if start+length > len(asRunes) { + length = len(asRunes) - start + } + + return string(asRunes[start : start+length]) +}