-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initial feed creation and feed_item population
- Loading branch information
Showing
6 changed files
with
359 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| package feeds | ||
|
|
||
| import ( | ||
| "net/http" | ||
| "time" | ||
|
|
||
| "github.com/labstack/echo/v5" | ||
| "github.com/mmcdole/gofeed" | ||
| "github.com/pocketbase/pocketbase/apis" | ||
| "github.com/pocketbase/pocketbase/core" | ||
| "github.com/pocketbase/pocketbase/models" | ||
| ) | ||
|
|
||
| // FeedResult contains the parsed feed and the ETag and Last-Modified | ||
| // headers received from the remote server | ||
| type FeedResult struct { | ||
| Feed *gofeed.Feed | ||
| ETag string | ||
| LastModified string | ||
| } | ||
|
|
||
| // LoadFeedFromURL fetches and parses a feed from the given URL. | ||
| // It optionally uses etag and ifModifiedSince for conditional requests. | ||
| func LoadFeedFromURL(url string, etag string, ifModifiedSince time.Time) (*FeedResult, error) { | ||
| fp := gofeed.NewParser() | ||
|
|
||
| client := &http.Client{} | ||
| req, err := http.NewRequest("GET", url, nil) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if etag != "" { | ||
| req.Header.Set("If-None-Match", etag) | ||
| } | ||
| if !ifModifiedSince.IsZero() { | ||
| req.Header.Set("If-Modified-Since", ifModifiedSince.Format(http.TimeFormat)) | ||
| } | ||
|
|
||
| resp, err := client.Do(req) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| if resp.StatusCode == http.StatusNotModified { | ||
| return &FeedResult{}, nil | ||
| } | ||
|
|
||
| feed, err := fp.Parse(resp.Body) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &FeedResult{ | ||
| Feed: feed, | ||
| ETag: resp.Header.Get("ETag"), | ||
| LastModified: resp.Header.Get("Last-Modified"), | ||
| }, nil | ||
| } | ||
|
|
||
| func SaveNewFeedItems(app core.App, feed *gofeed.Feed, user string, feedId string, lastArticlePubDate time.Time) error { | ||
| collection, err := app.Dao().FindCollectionByNameOrId("feed_items") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| for _, item := range feed.Items { | ||
| if item.PublishedParsed != nil && !item.PublishedParsed.After(lastArticlePubDate) { | ||
| continue | ||
| } | ||
| existingItem, _ := app.Dao().FindFirstRecordByFilter( | ||
| "feed_items", | ||
| "feed = {:feed} && guid = {:guid}", | ||
| map[string]interface{}{"feed": feedId, "guid": item.GUID}, | ||
| ) | ||
| if existingItem == nil { | ||
| newItem := models.NewRecord(collection) | ||
| newItem.Set("user", user) | ||
| newItem.Set("feed", feedId) | ||
| newItem.Set("title", item.Title) | ||
| newItem.Set("pub_date", item.PublishedParsed) | ||
| newItem.Set("guid", item.GUID) | ||
| newItem.Set("description", item.Description) | ||
| newItem.Set("url", item.Link) | ||
| if err := app.Dao().SaveRecord(newItem); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // SaveNewFeed extracts the URL from the request, loads the | ||
| // feed, and saves it to the database along with the first | ||
| // set of feed items. | ||
| func SaveNewFeed(app core.App, c echo.Context) error { | ||
| authRecord, _ := c.Get(apis.ContextAuthRecordKey).(*models.Record) | ||
| if authRecord == nil { | ||
| return apis.NewForbiddenError("Not authenticated", nil) | ||
| } | ||
|
|
||
| url := c.FormValue("url") | ||
| if url == "" { | ||
| return apis.NewBadRequestError("URL is required", nil) | ||
| } | ||
|
|
||
| feedResult, err := LoadFeedFromURL(url, "", time.Time{}) | ||
| if err != nil { | ||
| return apis.NewBadRequestError("Error parsing feed", err) | ||
| } | ||
|
|
||
| collection, err := app.Dao().FindCollectionByNameOrId("feeds") | ||
| if err != nil { | ||
| return apis.NewBadRequestError("Failed to find feeds collection", err) | ||
| } | ||
|
|
||
| record := models.NewRecord(collection) | ||
| record.Set("user", authRecord.Id) | ||
| record.Set("feed_url", url) | ||
| record.Set("name", feedResult.Feed.Title) | ||
| record.Set("description", feedResult.Feed.Description) | ||
| if feedResult.Feed.Image != nil { | ||
| record.Set("image_url", feedResult.Feed.Image.URL) | ||
| } | ||
| record.Set("etag", feedResult.ETag) | ||
| record.Set("modified", feedResult.LastModified) | ||
| record.Set("last_fetched_at", time.Now().UTC().Format(time.RFC3339)) | ||
| record.Set("auto_add_feed_items_to_library", false) | ||
|
|
||
| if err := app.Dao().SaveRecord(record); err != nil { | ||
| return apis.NewBadRequestError("Failed to save feed", err) | ||
| } | ||
|
|
||
| if err := SaveNewFeedItems(app, feedResult.Feed, authRecord.Id, record.Id, time.Time{}); err != nil { | ||
| return apis.NewBadRequestError("Failed to save feed items", err) | ||
| } | ||
|
|
||
| return c.JSON(200, map[string]interface{}{ | ||
| "id": record.Id, | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| package feeds | ||
|
|
||
| import ( | ||
| "net/http" | ||
| "net/http/httptest" | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| func TestLoadFeedFromURL(t *testing.T) { | ||
| // Set up a mock server | ||
| server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| if r.Header.Get("If-None-Match") == "some-etag" { | ||
| w.WriteHeader(http.StatusNotModified) | ||
| return | ||
| } | ||
| w.Header().Set("ETag", "new-etag") | ||
| w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") | ||
| w.Write([]byte(`<?xml version="1.0" encoding="UTF-8"?> | ||
| <rss version="2.0"> | ||
| <channel> | ||
| <title>Sample Feed</title> | ||
| <description>A sample feed</description> | ||
| <item> | ||
| <title>Sample Item</title> | ||
| <link>http://example.com/item</link> | ||
| <pubDate>Wed, 21 Oct 2015 07:28:00 GMT</pubDate> | ||
| </item> | ||
| </channel> | ||
| </rss>`)) | ||
| })) | ||
| defer server.Close() | ||
|
|
||
| testCases := []struct { | ||
| name string | ||
| etag string | ||
| ifModifiedSince time.Time | ||
| expectedResult func(*testing.T, *FeedResult) | ||
| }{ | ||
| { | ||
| name: "LoadFeedFromURL works with no etag", | ||
| etag: "", | ||
| ifModifiedSince: time.Time{}, | ||
| expectedResult: func(t *testing.T, result *FeedResult) { | ||
| if result.Feed.Title != "Sample Feed" { | ||
| t.Errorf("Expected feed title to be 'Sample Feed', got '%s'", result.Feed.Title) | ||
| } | ||
| if result.Feed.Description != "A sample feed" { | ||
| t.Errorf("Expected feed description to be 'A sample feed', got '%s'", result.Feed.Description) | ||
| } | ||
| if result.Feed.Items[0].Title != "Sample Item" { | ||
| t.Errorf("Expected feed item title to be 'Sample Item', got '%s'", result.Feed.Items[0].Title) | ||
| } | ||
| if result.ETag != "new-etag" { | ||
| t.Errorf("Expected etag to be 'new-etag', got '%s'", result.ETag) | ||
| } | ||
| }, | ||
| }, | ||
| { | ||
| name: "LoadFeedFromURL returns empty response on 304", | ||
| etag: "some-etag", | ||
| ifModifiedSince: time.Time{}, | ||
| expectedResult: func(t *testing.T, result *FeedResult) { | ||
| if result.Feed != nil { | ||
| t.Errorf("Expected feed to be nil, got '%+v'", result.Feed) | ||
| } | ||
| }, | ||
| }, | ||
| } | ||
| for _, tc := range testCases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| result, err := LoadFeedFromURL(server.URL, tc.etag, tc.ifModifiedSince) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| tc.expectedResult(t, result) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.