yarr notation

event.go

@@ -193,6 +193,12 @@ func deleteRemoteFollow(pubkeyHex string) nostr.Tags {
 
 // TRUE if feed exists in bookmark event
 func feedExists(pubkeyHex, privKeyHex, feedUrl string) (bool, error) {
+
+	if feedUrl == "" {
+		log.Printf("[ERROR] feedURL is empty")
+		return false, fmt.Errorf("feedURL is empty")
+	}
+
 	var bookMarkTags nostr.Tags
 
 	var bookmarkFilter nostr.Filter = nostr.Filter{

feed.go

@@ -17,7 +17,6 @@ import (
 	"time"
 
 	md "github.com/JohannesKaufmann/html-to-markdown"
-	"github.com/PuerkitoBio/goquery"
 	"github.com/microcosm-cc/bluemonday"
 	"github.com/mmcdole/gofeed"
 	"github.com/nbd-wtf/go-nostr"
@@ -36,53 +35,6 @@ var (
 	}
 )
 
-var rss_types = []string{
-	"rss+xml",
-	"atom+xml",
-	"feed+json",
-	"text/xml",
-	"application/xml",
-}
-
-func getFeedUrl(url string) string {
-	resp, err := client.Get(url)
-	if err != nil {
-		log.Printf("[ERROR] %s", err)
-		return ""
-	} else if resp.StatusCode >= 300 {
-		log.Printf("[ERROR] status code: %d", resp.StatusCode)
-		return ""
-	}
-
-	ct := resp.Header.Get("Content-Type")
-	for _, typ := range rss_types {
-		if strings.Contains(ct, typ) {
-			return url
-		}
-	}
-
-	if strings.Contains(ct, "text/html") {
-		doc, err := goquery.NewDocumentFromReader(resp.Body)
-		if err != nil {
-			log.Printf("[ERROR] %s", err)
-			return ""
-		}
-
-		for _, typ := range rss_types {
-			href, _ := doc.Find(fmt.Sprintf("link[type*='%s']", typ)).Attr("href")
-			if href == "" {
-				continue
-			}
-			if !strings.HasPrefix(href, "http") && !strings.HasPrefix(href, "https") {
-				href, _ = UrlJoin(url, href)
-			}
-			return href
-		}
-	}
-
-	return ""
-}
-
 func parseFeedForUrl(url string) (*gofeed.Feed, error) {
 	//metrics.CacheMiss.Inc()
 	fp.RSSTranslator = NewCustomTranslator()

handlers.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 	"rssnotes/metrics"
 
-	"rssnotes/rssworker"
+	"rssnotes/yarrworker"
 	"strconv"
 	"strings"
 	"text/template"
@@ -140,32 +140,15 @@ func createFeed(r *http.Request, secret *string) *GUIEntry {
 		Error: false,
 	}
 
-	discFeed, err := rssworker.DiscoverRssFeed(urlParam)
-	if err != nil {
+	discFeed, err := yarrworker.DiscoverRssFeed(urlParam)
+	if err != nil || discFeed.FeedLink == "" {
 		guientry.ErrorCode = http.StatusBadRequest
 		guientry.Error = true
 		guientry.ErrorMessage = "Could not find a feed URL in there..."
 		return &guientry
 	}
-
 	feedUrl := discFeed.FeedLink
 
-	// if !IsValidHttpUrl(urlParam) {
-	// 	log.Printf("[DEBUG] tried to create feed from invalid feed url '%q' skipping...", urlParam)
-	// 	guientry.ErrorCode = http.StatusBadRequest
-	// 	guientry.Error = true
-	// 	guientry.ErrorMessage = "Invalid URL provided (must be in absolute format and with https or https scheme)..."
-	// 	return &guientry
-	// }
-
-	// feedUrl := getFeedUrl(urlParam)
-	// if feedUrl == "" {
-	// 	guientry.ErrorCode = http.StatusBadRequest
-	// 	guientry.Error = true
-	// 	guientry.ErrorMessage = "Could not find a feed URL in there..."
-	// 	return &guientry
-	// }
-
 	sk := getPrivateKeyFromFeedUrl(feedUrl, *secret)
 	publicKey, err := nostr.GetPublicKey(sk)
 	if err != nil {
@@ -205,7 +188,7 @@ func createFeed(r *http.Request, secret *string) *GUIEntry {
 	guientry.NPubKey, _ = nip19.EncodePublicKey(publicKey)
 	guientry.BookmarkEntity.ImageURL = s.DefaultProfilePicUrl
 
-	faviconUrl, err := rssworker.FindFaviconURL(parsedFeed.Link, feedUrl)
+	faviconUrl, err := yarrworker.FindFaviconURL(parsedFeed.Link, feedUrl)
 	if err != nil {
 		log.Print("[ERROR] FindFavicon", err)
 	} else if faviconUrl != "" {
@@ -323,8 +306,8 @@ func importFeeds(opmlUrls []opml.Outline, secret *string) []*GUIEntry {
 			continue
 		}
 
-		discFeed, err := rssworker.DiscoverRssFeed(urlParam.XMLURL)
-		if err != nil {
+		discFeed, err := yarrworker.DiscoverRssFeed(urlParam.XMLURL)
+		if err != nil || discFeed.FeedLink == "" {
 			importedEntries = append(importedEntries, &GUIEntry{
 				BookmarkEntity: Entity{URL: urlParam.XMLURL},
 				ErrorMessage:   "Could not find a feed URL in there...",
@@ -337,19 +320,6 @@ func importFeeds(opmlUrls []opml.Outline, secret *string) []*GUIEntry {
 		}
 		feedUrl := discFeed.FeedLink
 
-		// feedUrl := getFeedUrl(urlParam.XMLURL)
-		// if feedUrl == "" {
-		// 	importedEntries = append(importedEntries, &GUIEntry{
-		// 		BookmarkEntity: Entity{URL: urlParam.XMLURL},
-		// 		ErrorMessage:   "Could not find a feed URL in there...",
-		// 		Error:          true,
-		// 		ErrorCode:      http.StatusBadRequest,
-		// 	})
-		// 	importProgressCh <- ImportProgressStruct{entryIndex: urlIndex, totalEntries: len(opmlUrls)}
-		// 	log.Printf("[DEBUG] Could not find a feed URL in %s", feedUrl)
-		// 	continue
-		// }
-
 		sk := getPrivateKeyFromFeedUrl(feedUrl, *secret)
 		publicKey, err := nostr.GetPublicKey(sk)
 		if err != nil {
@@ -416,7 +386,7 @@ func importFeeds(opmlUrls []opml.Outline, secret *string) []*GUIEntry {
 		}
 
 		localImageURL := s.DefaultProfilePicUrl
-		faviconUrl, err := rssworker.FindFaviconURL(parsedFeed.Link, feedUrl)
+		faviconUrl, err := yarrworker.FindFaviconURL(parsedFeed.Link, feedUrl)
 		if err != nil {
 			log.Print("[ERROR] FindFavicon", err)
 		} else if faviconUrl != "" {

main.go

@@ -168,7 +168,7 @@ func main() {
 	mux.HandleFunc(" /delete", handleDeleteFeed)
 	mux.Handle("GET /metrics", promhttp.Handler())
 
-	mux.HandleFunc(" /metricsDisplay", handleMetricsDisplay)
+	mux.HandleFunc("GET /metricsDisplay", handleMetricsDisplay)
 
 	mux.HandleFunc("GET /health", handleHealth)
 	mux.HandleFunc("GET /log", handleLog)

rssparser/atom.go → yarrparser/atom.go

@@ -1,12 +1,12 @@
 // Atom 1.0 parser
-package rssparser
+package yarrparser
 
 import (
 	"encoding/xml"
 	"io"
 	"strings"
 
-	htmlutil "rssnotes/rsshtmlutil"
+	htmlutil "rssnotes/yarrhtmlutil"
 )
 
 type atomFeed struct {

rssparser/date.go → yarrparser/date.go

@@ -1,4 +1,4 @@
-package rssparser
+package yarrparser
 
 import "time"
 

rssparser/feed.go → yarrparser/feed.go

@@ -1,4 +1,4 @@
-package rssparser
+package yarrparser
 
 import (
 	"bytes"
@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	htmlutil "rssnotes/rsshtmlutil"
+	htmlutil "rssnotes/yarrhtmlutil"
 
 	"golang.org/x/net/html/charset"
 )

rssparser/json.go → yarrparser/json.go

@@ -1,5 +1,5 @@
 // JSON 1.0 parser
-package rssparser
+package yarrparser
 
 import (
 	"encoding/json"

rssparser/media.go → yarrparser/media.go

@@ -1,4 +1,4 @@
-package rssparser
+package yarrparser
 
 type media struct {
 	MediaGroups       []mediaGroup       `xml:"http://search.yahoo.com/mrss/ group"`

rssparser/models.go → yarrparser/models.go

@@ -1,4 +1,4 @@
-package rssparser
+package yarrparser
 
 import "time"
 

rssparser/rdf.go → yarrparser/rdf.go

@@ -1,7 +1,7 @@
 // Parser for RSS versions:
 // - 0.90
 // - 1.0
-package rssparser
+package yarrparser
 
 import (
 	"encoding/xml"

rssparser/rss.go → yarrparser/rss.go

@@ -2,7 +2,7 @@
 // - 0.91 netscape
 // - 0.91 userland
 // - 2.0
-package rssparser
+package yarrparser
 
 import (
 	"encoding/xml"

rssparser/util.go → yarrparser/util.go

@@ -1,4 +1,4 @@
-package rssparser
+package yarrparser
 
 import (
 	"bufio"

rssscraper/finder.go → yarrscraper/finder.go

@@ -1,9 +1,9 @@
-package rssscraper
+package yarrscraper
 
 import (
 	"strings"
 
-	htmlutil "rssnotes/rsshtmlutil"
+	htmlutil "rssnotes/yarrhtmlutil"
 
 	"golang.org/x/net/html"
 )

rssworker/client.go → yarrworker/client.go

@@ -1,4 +1,4 @@
-package rssworker
+package yarrworker
 
 import (
 	"net"

rssworker/crawler.go → yarrworker/crawler.go

@@ -1,11 +1,11 @@
-package rssworker
+package yarrworker
 
 import (
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"rssnotes/rssscraper"
+	"rssnotes/yarrscraper"
 )
 
 var imageTypes = map[string]bool{
@@ -30,7 +30,7 @@ func FindFaviconURL(siteUrl, feedUrl string) (string, error) {
 		if res, err := client.get(siteUrl); err == nil {
 			defer res.Body.Close()
 			if body, err := io.ReadAll(res.Body); err == nil {
-				urls = append(urls, rssscraper.FindIcons(string(body), siteUrl)...)
+				urls = append(urls, yarrscraper.FindIcons(string(body), siteUrl)...)
 				if c := favicon(siteUrl); c != "" {
 					urls = append(urls, c)
 				}

rssworker/discoverfeed.go → yarrworker/discoverfeed.go

@@ -1,4 +1,4 @@
-package rssworker
+package yarrworker
 
 import (
 	"bytes"
@@ -8,8 +8,8 @@ import (
 	"mime"
 	"net/http"
 
-	"rssnotes/rssparser"
-	"rssnotes/rssscraper"
+	"rssnotes/yarrparser"
+	"rssnotes/yarrscraper"
 
 	"golang.org/x/net/html/charset"
 )
@@ -20,7 +20,7 @@ type FeedSource struct {
 }
 
 type DiscoverResult struct {
-	Feed     *rssparser.Feed
+	Feed     *yarrparser.Feed
 	FeedLink string
 	Sources  []FeedSource
 }
@@ -44,7 +44,7 @@ func DiscoverRssFeed(candidateUrl string) (*DiscoverResult, error) {
 	}
 
 	// Try to feed into parser
-	feed, err := rssparser.ParseAndFix(bytes.NewReader(body), candidateUrl, cs)
+	feed, err := yarrparser.ParseAndFix(bytes.NewReader(body), candidateUrl, cs)
 	if err == nil {
 		result.Feed = feed
 		result.FeedLink = candidateUrl
@@ -61,7 +61,7 @@ func DiscoverRssFeed(candidateUrl string) (*DiscoverResult, error) {
 		}
 	}
 	sources := make([]FeedSource, 0)
-	for url, title := range rssscraper.FindFeeds(content, candidateUrl) {
+	for url, title := range yarrscraper.FindFeeds(content, candidateUrl) {
 		sources = append(sources, FeedSource{Title: title, Url: url})
 	}
 	switch {