From 8d840b09c27ef2f0f4b4ca467efd207dbf9209f4 Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Mon, 9 Oct 2023 13:04:16 -0400 Subject: [PATCH 1/9] Add support for refreshing feed icons --- internal/database/migrations.go | 7 ++++++ internal/reader/handler/handler.go | 31 ++++++++++++++++++++------ internal/storage/feed.go | 12 ++++++---- internal/storage/feed_query_builder.go | 4 +++- internal/storage/icon.go | 6 +++++ 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/internal/database/migrations.go b/internal/database/migrations.go index be3bef84c89..2b6b878763e 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -834,4 +834,11 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE feeds ADD COLUMN icon_url text default ''; + ` + _, err = tx.Exec(sql) + return err + }, } diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index 36cd2561533..7ffde137e3e 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -256,6 +256,8 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool return localizedError } + refreshIcon := forceRefresh + if originalFeed.IgnoreHTTPCache || responseHandler.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) { slog.Debug("Feed modified", slog.Int64("user_id", userID), @@ -321,13 +323,10 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool originalFeed.EtagHeader = responseHandler.ETag() originalFeed.LastModifiedHeader = responseHandler.LastModified() - checkFeedIcon( - store, - requestBuilder, - originalFeed.ID, - originalFeed.SiteURL, - updatedFeed.IconURL, - ) + if updatedFeed.IconURL != originalFeed.IconURL { + originalFeed.IconURL = updatedFeed.IconURL + refreshIcon = true + } } else { slog.Debug("Feed not modified", slog.Int64("user_id", userID), @@ -335,6 +334,24 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool ) } + if refreshIcon { + if err := store.RemoveFeedIcon(feedID); err != nil { + slog.Debug("Unable to remove Feed Icon", + slog.Int64("feed_id", feedID), + ) + } + } + + checkFeedIcon( + store, + originalFeed.ID, + originalFeed.SiteURL, + originalFeed.IconURL, + originalFeed.UserAgent, + originalFeed.FetchViaProxy, + originalFeed.AllowSelfSignedCertificates, + ) + originalFeed.ResetErrorCounter() if storeErr := store.UpdateFeed(originalFeed); storeErr != nil { diff --git a/internal/storage/feed.go b/internal/storage/feed.go index 1a40102c0f2..11e81e290dc 100644 --- a/internal/storage/feed.go +++ b/internal/storage/feed.go @@ -235,10 +235,11 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { hide_globally, url_rewrite_rules, no_media_player, - apprise_service_urls + apprise_service_urls, + icon_url ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25) RETURNING id ` @@ -268,6 +269,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { feed.UrlRewriteRules, feed.NoMediaPlayer, feed.AppriseServiceURLs, + feed.IconURL, ).Scan(&feed.ID) if err != nil { return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err) @@ -339,9 +341,10 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { hide_globally=$24, url_rewrite_rules=$25, no_media_player=$26, - apprise_service_urls=$27 + apprise_service_urls=$27, + icon_url=$28 WHERE - id=$28 AND user_id=$29 + id=$29 AND user_id=$30 ` _, err = s.db.Exec(query, feed.FeedURL, @@ -371,6 +374,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { feed.UrlRewriteRules, feed.NoMediaPlayer, feed.AppriseServiceURLs, + feed.IconURL, feed.ID, feed.UserID, ) diff --git a/internal/storage/feed_query_builder.go b/internal/storage/feed_query_builder.go index 29926b81959..2d2df66974c 100644 --- a/internal/storage/feed_query_builder.go +++ b/internal/storage/feed_query_builder.go @@ -163,7 +163,8 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) { c.hide_globally as category_hidden, fi.icon_id, u.timezone, - f.apprise_service_urls + f.apprise_service_urls, + f.icon_url FROM feeds f LEFT JOIN @@ -230,6 +231,7 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) { &iconID, &tz, &feed.AppriseServiceURLs, + &feed.IconURL, ) if err != nil { diff --git a/internal/storage/icon.go b/internal/storage/icon.go index bd407d92a26..6ef760a808d 100644 --- a/internal/storage/icon.go +++ b/internal/storage/icon.go @@ -114,6 +114,12 @@ func (s *Storage) CreateFeedIcon(feedID int64, icon *model.Icon) error { return nil } +func (s *Storage) RemoveFeedIcon(feedID int64) error { + if _, err := s.db.Exec(`DELETE FROM feed_icons WHERE feed_id=$1`, feedID); err != nil { + return fmt.Errorf(`store: unable to remove feed icon: %v`, err) + } + return nil +} // Icons returns all icons that belongs to a user. func (s *Storage) Icons(userID int64) (model.Icons, error) { From 1b92ce460fc0c3d44e229cc30f0ffd1c0c9b09fa Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Wed, 25 Oct 2023 19:31:39 -0400 Subject: [PATCH 2/9] only update icon_url if successful --- internal/model/feed.go | 10 ++++----- internal/model/icon.go | 1 + internal/reader/handler/handler.go | 33 +++++++++++++++--------------- internal/reader/icon/finder.go | 1 + internal/storage/feed.go | 12 ++++------- internal/storage/icon.go | 4 ++++ 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/internal/model/feed.go b/internal/model/feed.go index c9562ac9325..46740c012e7 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -51,17 +51,17 @@ type Feed struct { FetchViaProxy bool `json:"fetch_via_proxy"` HideGlobally bool `json:"hide_globally"` AppriseServiceURLs string `json:"apprise_service_urls"` + IconURL string `json:"icon_url"` // Non persisted attributes Category *Category `json:"category,omitempty"` Icon *FeedIcon `json:"icon"` Entries Entries `json:"entries,omitempty"` - TTL int `json:"-"` - IconURL string `json:"-"` - UnreadCount int `json:"-"` - ReadCount int `json:"-"` - NumberOfVisibleEntries int `json:"-"` + TTL int `json:"-"` + UnreadCount int `json:"-"` + ReadCount int `json:"-"` + NumberOfVisibleEntries int `json:"-"` } type FeedCounters struct { diff --git a/internal/model/icon.go b/internal/model/icon.go index 7a38b75c90f..51257106c1d 100644 --- a/internal/model/icon.go +++ b/internal/model/icon.go @@ -14,6 +14,7 @@ type Icon struct { Hash string `json:"hash"` MimeType string `json:"mime_type"` Content []byte `json:"-"` + URL string `json:"-"` } // DataURL returns the data URL of the icon. diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index 7ffde137e3e..9b17f41891f 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -97,6 +97,7 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f subscription.ID, subscription.SiteURL, subscription.IconURL, + false, ) return subscription, nil @@ -189,6 +190,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model subscription.ID, subscription.SiteURL, subscription.IconURL, + false, ) return subscription, nil } @@ -256,8 +258,6 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool return localizedError } - refreshIcon := forceRefresh - if originalFeed.IgnoreHTTPCache || responseHandler.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) { slog.Debug("Feed modified", slog.Int64("user_id", userID), @@ -325,7 +325,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool if updatedFeed.IconURL != originalFeed.IconURL { originalFeed.IconURL = updatedFeed.IconURL - refreshIcon = true + forceRefresh = true } } else { slog.Debug("Feed not modified", @@ -334,22 +334,13 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool ) } - if refreshIcon { - if err := store.RemoveFeedIcon(feedID); err != nil { - slog.Debug("Unable to remove Feed Icon", - slog.Int64("feed_id", feedID), - ) - } - } - checkFeedIcon( store, + requestBuilder, originalFeed.ID, originalFeed.SiteURL, originalFeed.IconURL, - originalFeed.UserAgent, - originalFeed.FetchViaProxy, - originalFeed.AllowSelfSignedCertificates, + forceRefresh, ) originalFeed.ResetErrorCounter() @@ -364,8 +355,8 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool return nil } -func checkFeedIcon(store *storage.Storage, requestBuilder *fetcher.RequestBuilder, feedID int64, websiteURL, feedIconURL string) { - if !store.HasIcon(feedID) { +func checkFeedIcon(store *storage.Storage, requestBuilder *fetcher.RequestBuilder, feedID int64, websiteURL, feedIconURL string, forceRefresh bool) { + if !store.HasIcon(feedID) || forceRefresh { iconFinder := icon.NewIconFinder(requestBuilder, websiteURL, feedIconURL) if icon, err := iconFinder.FindIcon(); err != nil { slog.Debug("Unable to find feed icon", @@ -381,6 +372,16 @@ func checkFeedIcon(store *storage.Storage, requestBuilder *fetcher.RequestBuilde slog.String("feed_icon_url", feedIconURL), ) } else { + if icon.URL == "" { + icon.URL = feedIconURL + } + if forceRefresh { + if err := store.RemoveFeedIcon(feedID); err != nil { + slog.Debug("Unable to remove Feed Icon", + slog.Int64("feed_id", feedID), + ) + } + } if err := store.CreateFeedIcon(feedID, icon); err != nil { slog.Error("Unable to store feed icon", slog.Int64("feed_id", feedID), diff --git a/internal/reader/icon/finder.go b/internal/reader/icon/finder.go index cb9c7d49cab..dececdca4e1 100644 --- a/internal/reader/icon/finder.go +++ b/internal/reader/icon/finder.go @@ -177,6 +177,7 @@ func (f *IconFinder) DownloadIcon(iconURL string) (*model.Icon, error) { Hash: crypto.HashFromBytes(responseBody), MimeType: responseHandler.ContentType(), Content: responseBody, + URL: iconURL, } return icon, nil diff --git a/internal/storage/feed.go b/internal/storage/feed.go index 11e81e290dc..1a40102c0f2 100644 --- a/internal/storage/feed.go +++ b/internal/storage/feed.go @@ -235,11 +235,10 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { hide_globally, url_rewrite_rules, no_media_player, - apprise_service_urls, - icon_url + apprise_service_urls ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25) + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) RETURNING id ` @@ -269,7 +268,6 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { feed.UrlRewriteRules, feed.NoMediaPlayer, feed.AppriseServiceURLs, - feed.IconURL, ).Scan(&feed.ID) if err != nil { return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err) @@ -341,10 +339,9 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { hide_globally=$24, url_rewrite_rules=$25, no_media_player=$26, - apprise_service_urls=$27, - icon_url=$28 + apprise_service_urls=$27 WHERE - id=$29 AND user_id=$30 + id=$28 AND user_id=$29 ` _, err = s.db.Exec(query, feed.FeedURL, @@ -374,7 +371,6 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { feed.UrlRewriteRules, feed.NoMediaPlayer, feed.AppriseServiceURLs, - feed.IconURL, feed.ID, feed.UserID, ) diff --git a/internal/storage/icon.go b/internal/storage/icon.go index 6ef760a808d..88e91fd6d38 100644 --- a/internal/storage/icon.go +++ b/internal/storage/icon.go @@ -111,6 +111,10 @@ func (s *Storage) CreateFeedIcon(feedID int64, icon *model.Icon) error { if err != nil { return fmt.Errorf(`store: unable to create feed icon: %v`, err) } + _, err = s.db.Exec(`UPDATE feeds SET icon_url=$1 WHERE ID=$2`, icon.URL, feedID) + if err != nil { + return fmt.Errorf(`store: unable to save icon_url: %v`, err) + } return nil } From d65b966ff836c801a64283163e833ee98a7d9e5e Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Sun, 29 Oct 2023 13:18:57 -0400 Subject: [PATCH 3/9] URLHasChanged --- internal/reader/handler/handler.go | 11 +++++++---- internal/reader/icon/finder.go | 8 ++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index 9b17f41891f..e5b2355861d 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -323,9 +323,15 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool originalFeed.EtagHeader = responseHandler.ETag() originalFeed.LastModifiedHeader = responseHandler.LastModified() - if updatedFeed.IconURL != originalFeed.IconURL { + if icon.URLHasChanged(originalFeed.IconURL, updatedFeed.IconURL, updatedFeed.SiteURL) { originalFeed.IconURL = updatedFeed.IconURL forceRefresh = true + } else { + slog.Debug("Feed icon not modified", + slog.Int64("user_id", userID), + slog.Int64("feed_id", feedID), + slog.String("feed_icon_url", originalFeed.IconURL), + ) } } else { slog.Debug("Feed not modified", @@ -372,9 +378,6 @@ func checkFeedIcon(store *storage.Storage, requestBuilder *fetcher.RequestBuilde slog.String("feed_icon_url", feedIconURL), ) } else { - if icon.URL == "" { - icon.URL = feedIconURL - } if forceRefresh { if err := store.RemoveFeedIcon(feedID); err != nil { slog.Debug("Unable to remove Feed Icon", diff --git a/internal/reader/icon/finder.go b/internal/reader/icon/finder.go index dececdca4e1..00d21839a60 100644 --- a/internal/reader/icon/finder.go +++ b/internal/reader/icon/finder.go @@ -288,3 +288,11 @@ func parseImageDataURL(value string) (*model.Icon, error) { return icon, nil } + +func URLHasChanged(original, updated, siteURL string) bool { + if updated == "" { + return false + } + updated, _ = urllib.AbsoluteURL(siteURL, updated) + return original == updated +} From 5279b77b59b47c9d0b4496184553c72d035e2f6d Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Tue, 2 Jan 2024 11:15:10 -0500 Subject: [PATCH 4/9] fix logic error --- internal/reader/icon/finder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/reader/icon/finder.go b/internal/reader/icon/finder.go index 00d21839a60..13dd617ff27 100644 --- a/internal/reader/icon/finder.go +++ b/internal/reader/icon/finder.go @@ -294,5 +294,5 @@ func URLHasChanged(original, updated, siteURL string) bool { return false } updated, _ = urllib.AbsoluteURL(siteURL, updated) - return original == updated + return original != updated } From ff1b5df056e1a2ece7d82fb175670e38d93bb0e6 Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Tue, 2 Jan 2024 13:05:15 -0500 Subject: [PATCH 5/9] support icons older than the feed --- internal/reader/handler/handler.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index e5b2355861d..d58ee424a31 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -340,6 +340,16 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool ) } + // new request without LastTimeout or ETag to allow for icons that are older than the feed + requestBuilder = fetcher.NewRequestBuilder() + requestBuilder.WithUsernameAndPassword(originalFeed.Username, originalFeed.Password) + requestBuilder.WithUserAgent(originalFeed.UserAgent, config.Opts.HTTPClientUserAgent()) + requestBuilder.WithCookie(originalFeed.Cookie) + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.UseProxy(originalFeed.FetchViaProxy) + requestBuilder.IgnoreTLSErrors(originalFeed.AllowSelfSignedCertificates) + checkFeedIcon( store, requestBuilder, From e3713c68dfb59209842aa3442858acb4a280924f Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Thu, 11 Jan 2024 09:43:03 -0500 Subject: [PATCH 6/9] moved icon requestbuilder into checkFeedIcon, updated logging, set IconURL during feed create/update --- internal/reader/handler/handler.go | 67 +++++++++++++----------------- internal/storage/feed.go | 12 ++++-- internal/storage/icon.go | 4 -- 3 files changed, 37 insertions(+), 46 deletions(-) diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index d58ee424a31..e8ceb627d62 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -93,10 +93,7 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f checkFeedIcon( store, - requestBuilder, - subscription.ID, - subscription.SiteURL, - subscription.IconURL, + subscription, false, ) @@ -186,10 +183,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model checkFeedIcon( store, - requestBuilder, - subscription.ID, - subscription.SiteURL, - subscription.IconURL, + subscription, false, ) return subscription, nil @@ -340,22 +334,9 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool ) } - // new request without LastTimeout or ETag to allow for icons that are older than the feed - requestBuilder = fetcher.NewRequestBuilder() - requestBuilder.WithUsernameAndPassword(originalFeed.Username, originalFeed.Password) - requestBuilder.WithUserAgent(originalFeed.UserAgent, config.Opts.HTTPClientUserAgent()) - requestBuilder.WithCookie(originalFeed.Cookie) - requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) - requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.UseProxy(originalFeed.FetchViaProxy) - requestBuilder.IgnoreTLSErrors(originalFeed.AllowSelfSignedCertificates) - checkFeedIcon( store, - requestBuilder, - originalFeed.ID, - originalFeed.SiteURL, - originalFeed.IconURL, + originalFeed, forceRefresh, ) @@ -371,35 +352,45 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool return nil } -func checkFeedIcon(store *storage.Storage, requestBuilder *fetcher.RequestBuilder, feedID int64, websiteURL, feedIconURL string, forceRefresh bool) { - if !store.HasIcon(feedID) || forceRefresh { - iconFinder := icon.NewIconFinder(requestBuilder, websiteURL, feedIconURL) +func checkFeedIcon(store *storage.Storage, feed *model.Feed, forceRefresh bool) { + if !store.HasIcon(feed.ID) || forceRefresh { + requestBuilder := fetcher.NewRequestBuilder() + requestBuilder.WithUsernameAndPassword(feed.Username, feed.Password) + requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent()) + requestBuilder.WithCookie(feed.Cookie) + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.UseProxy(feed.FetchViaProxy) + requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates) + + iconFinder := icon.NewIconFinder(requestBuilder, feed.SiteURL, feed.IconURL) if icon, err := iconFinder.FindIcon(); err != nil { slog.Debug("Unable to find feed icon", - slog.Int64("feed_id", feedID), - slog.String("website_url", websiteURL), - slog.String("feed_icon_url", feedIconURL), + slog.Int64("feed_id", feed.ID), + slog.String("website_url", feed.SiteURL), + slog.String("feed_icon_url", feed.IconURL), slog.Any("error", err), ) } else if icon == nil { slog.Debug("No icon found", - slog.Int64("feed_id", feedID), - slog.String("website_url", websiteURL), - slog.String("feed_icon_url", feedIconURL), + slog.Int64("feed_id", feed.ID), + slog.String("website_url", feed.SiteURL), + slog.String("feed_icon_url", feed.IconURL), ) } else { if forceRefresh { - if err := store.RemoveFeedIcon(feedID); err != nil { - slog.Debug("Unable to remove Feed Icon", - slog.Int64("feed_id", feedID), + if err := store.RemoveFeedIcon(feed.ID); err != nil { + slog.Error("Unable to remove Feed Icon", + slog.Int64("feed_id", feed.ID), + slog.Any("error", err), ) } } - if err := store.CreateFeedIcon(feedID, icon); err != nil { + if err := store.CreateFeedIcon(feed.ID, icon); err != nil { slog.Error("Unable to store feed icon", - slog.Int64("feed_id", feedID), - slog.String("website_url", websiteURL), - slog.String("feed_icon_url", feedIconURL), + slog.Int64("feed_id", feed.ID), + slog.String("website_url", feed.SiteURL), + slog.String("feed_icon_url", feed.IconURL), slog.Any("error", err), ) } diff --git a/internal/storage/feed.go b/internal/storage/feed.go index 1a40102c0f2..11e81e290dc 100644 --- a/internal/storage/feed.go +++ b/internal/storage/feed.go @@ -235,10 +235,11 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { hide_globally, url_rewrite_rules, no_media_player, - apprise_service_urls + apprise_service_urls, + icon_url ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25) RETURNING id ` @@ -268,6 +269,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { feed.UrlRewriteRules, feed.NoMediaPlayer, feed.AppriseServiceURLs, + feed.IconURL, ).Scan(&feed.ID) if err != nil { return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err) @@ -339,9 +341,10 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { hide_globally=$24, url_rewrite_rules=$25, no_media_player=$26, - apprise_service_urls=$27 + apprise_service_urls=$27, + icon_url=$28 WHERE - id=$28 AND user_id=$29 + id=$29 AND user_id=$30 ` _, err = s.db.Exec(query, feed.FeedURL, @@ -371,6 +374,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { feed.UrlRewriteRules, feed.NoMediaPlayer, feed.AppriseServiceURLs, + feed.IconURL, feed.ID, feed.UserID, ) diff --git a/internal/storage/icon.go b/internal/storage/icon.go index 88e91fd6d38..6ef760a808d 100644 --- a/internal/storage/icon.go +++ b/internal/storage/icon.go @@ -111,10 +111,6 @@ func (s *Storage) CreateFeedIcon(feedID int64, icon *model.Icon) error { if err != nil { return fmt.Errorf(`store: unable to create feed icon: %v`, err) } - _, err = s.db.Exec(`UPDATE feeds SET icon_url=$1 WHERE ID=$2`, icon.URL, feedID) - if err != nil { - return fmt.Errorf(`store: unable to save icon_url: %v`, err) - } return nil } From 24e846d6003053dbc7e490dd6763aa0b39a4ce32 Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Thu, 11 Jan 2024 10:05:02 -0500 Subject: [PATCH 7/9] removed absoluteURL check. only care if it changed --- internal/reader/handler/handler.go | 2 +- internal/reader/icon/finder.go | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index e8ceb627d62..9b6ad4e019f 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -317,7 +317,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool originalFeed.EtagHeader = responseHandler.ETag() originalFeed.LastModifiedHeader = responseHandler.LastModified() - if icon.URLHasChanged(originalFeed.IconURL, updatedFeed.IconURL, updatedFeed.SiteURL) { + if originalFeed.IconURL != updatedFeed.IconURL { originalFeed.IconURL = updatedFeed.IconURL forceRefresh = true } else { diff --git a/internal/reader/icon/finder.go b/internal/reader/icon/finder.go index 13dd617ff27..dececdca4e1 100644 --- a/internal/reader/icon/finder.go +++ b/internal/reader/icon/finder.go @@ -288,11 +288,3 @@ func parseImageDataURL(value string) (*model.Icon, error) { return icon, nil } - -func URLHasChanged(original, updated, siteURL string) bool { - if updated == "" { - return false - } - updated, _ = urllib.AbsoluteURL(siteURL, updated) - return original != updated -} From 89e1a6389be2e47175663f3c4f7f2fb95faed4b4 Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Thu, 11 Jan 2024 12:22:33 -0500 Subject: [PATCH 8/9] remove icon if no other feeds are using it --- internal/storage/icon.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/internal/storage/icon.go b/internal/storage/icon.go index 6ef760a808d..b1c6cc1ee96 100644 --- a/internal/storage/icon.go +++ b/internal/storage/icon.go @@ -114,10 +114,29 @@ func (s *Storage) CreateFeedIcon(feedID int64, icon *model.Icon) error { return nil } + +// RemoveFeedIcon removes the feed_icon relation and the icon if no other feed is using it func (s *Storage) RemoveFeedIcon(feedID int64) error { + var iconID int64 + err := s.db.QueryRow(`SELECT icon_id FROM feed_icons WHERE feed_id=$1`, feedID).Scan(&iconID) + if err == sql.ErrNoRows { + return nil + } else if err != nil { + return fmt.Errorf(`store: unable to fetch iconID: %v`, err) + } + if _, err := s.db.Exec(`DELETE FROM feed_icons WHERE feed_id=$1`, feedID); err != nil { return fmt.Errorf(`store: unable to remove feed icon: %v`, err) } + + query := ` + DELETE FROM icons + WHERE + id=$1 AND NOT EXISTS(SELECT icon_id FROM feed_icons WHERE icon_id=$1); + ` + if _, err := s.db.Exec(query, iconID); err != nil { + return fmt.Errorf(`store: unable to remove feed icon: %v`, err) + } return nil } From 6905dd68f8630e3f06db8c8f79d646fb81b15f1b Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Thu, 1 Feb 2024 05:38:57 -0500 Subject: [PATCH 9/9] updated log message/condition --- internal/reader/handler/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index 9b6ad4e019f..50d1a977069 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -320,8 +320,8 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool if originalFeed.IconURL != updatedFeed.IconURL { originalFeed.IconURL = updatedFeed.IconURL forceRefresh = true - } else { - slog.Debug("Feed icon not modified", + } else if originalFeed.IconURL != "" { + slog.Debug("Feed icon URL not modified", slog.Int64("user_id", userID), slog.Int64("feed_id", feedID), slog.String("feed_icon_url", originalFeed.IconURL),