Skip to content

Commit

Permalink
Merge pull request #1 from trinidz/avgposttime
Browse files Browse the repository at this point in the history 
Avgposttime
  • Loading branch information
@trinidz
trinidz authored Nov 22, 2024
2 parents 0ade936 + 23ae159 commit 759a915
Show file tree
Hide file tree
Showing 6 changed files with 100 additions and 52 deletions.
37 changes: 16 additions & 21 deletions event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (

const KIND_BOOKMARKS int = 10003 //NIP-51

// gets notes from the local db
func getLocalEvents(localFilter nostr.Filter) ([]*nostr.Event, error) {
ctx := context.TODO()

Expand Down Expand Up @@ -191,7 +190,7 @@ func deleteRemoteFollow(pubkeyHex string) nostr.Tags {
return nil
}

// TRUE if a rsslay feedUrl already exists in bookmark event
// TRUE if feed exists in bookmark event
func feedExists(pubkeyHex, privKeyHex, feedUrl string) (bool, error) {
var bookMarkTags nostr.Tags

Expand Down Expand Up @@ -220,7 +219,6 @@ func feedExists(pubkeyHex, privKeyHex, feedUrl string) (bool, error) {
return false, nil
}

// add a feed entity to the bookmark event
func addEntityToBookmarkEvent(entitiesToAdd []Entity) error {
if len(entitiesToAdd) == 0 {
return nil
Expand Down Expand Up @@ -273,10 +271,10 @@ func addEntityToBookmarkEvent(entitiesToAdd []Entity) error {
return nil
}

// update an existing feed entity 'last' property in the bookmark event
func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
// update an existing feed entity time properties
func updateEntityTimesInBookmarkEvent(updatedEntity Entity) error {
var bookMarkTags nostr.Tags
var rsslayEntity Entity
var rssnotesEntity Entity

var bookmarkFilter nostr.Filter = nostr.Filter{
Kinds: []int{KIND_BOOKMARKS},
Expand All @@ -292,18 +290,20 @@ func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
if len(bookMarkEvts) > 0 {
bookMarkTags = bookMarkEvts[0].Tags.GetAll([]string{s.RsslayTagKey})
for i, tag := range bookMarkTags {
if strings.Contains(tag.Value(), pubKeyHex) {
if err := json.Unmarshal([]byte(tag.Value()), &rsslayEntity); err != nil {
if strings.Contains(tag.Value(), updatedEntity.PubKey) {
if err := json.Unmarshal([]byte(tag.Value()), &rssnotesEntity); err != nil {
log.Printf("[ERROR] %s", err)
}

copy(bookMarkTags[i:], bookMarkTags[i+1:])
bookMarkTags[len(bookMarkTags)-1] = nostr.Tag{}
bookMarkTags = bookMarkTags[:len(bookMarkTags)-1]

rsslayEntity.LastUpdate = lastUpdate
rssnotesEntity.LastPostTime = updatedEntity.LastPostTime
rssnotesEntity.LastCheckedTime = updatedEntity.LastCheckedTime
rssnotesEntity.AvgPostTime = updatedEntity.AvgPostTime

jsonentArr, err := json.Marshal(rsslayEntity)
jsonentArr, err := json.Marshal(rssnotesEntity)
if err != nil {
log.Printf("[ERROR] %s", err)
}
Expand All @@ -313,7 +313,7 @@ func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
evt := nostr.Event{
CreatedAt: nostr.Now(),
Kind: KIND_BOOKMARKS,
Content: "{rsslay, pubkey, privkey, url, last_update}",
Content: "",
Tags: bookMarkTags,
}

Expand All @@ -326,7 +326,7 @@ func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
store(context.TODO(), &evt)
}

log.Printf("[DEBUG] entity %s last update %d in event ID %s", rsslayEntity.URL, rsslayEntity.LastUpdate, evt.ID)
log.Printf("[DEBUG] entity %s last post time %d in event ID %s", rssnotesEntity.URL, rssnotesEntity.LastPostTime, evt.ID)
break
}
}
Expand All @@ -337,7 +337,6 @@ func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
return nil
}

// delete a feed entity from a local bookmark event
func deleteEntityInBookmarkEvent(pubKeyORfeedUrl string) error {
var bookMarkTags nostr.Tags
var rsslayEntity Entity
Expand Down Expand Up @@ -412,7 +411,6 @@ func deleteEntityInBookmarkEvent(pubKeyORfeedUrl string) error {
return nil
}

// return all saved FeedURL entries
func getSavedEntries() ([]GUIEntry, error) {

var bookMarkTags nostr.Tags
Expand Down Expand Up @@ -441,10 +439,10 @@ func getSavedEntries() ([]GUIEntry, error) {
npub, _ := nip19.EncodePublicKey(rsslayEntity.PubKey)
localEntries = append(localEntries, GUIEntry{
BookmarkEntity: Entity{
PubKey: rsslayEntity.PubKey,
URL: rsslayEntity.URL,
ImageURL: rsslayEntity.ImageURL,
LastUpdate: rsslayEntity.LastUpdate},
PubKey: rsslayEntity.PubKey,
URL: rsslayEntity.URL,
ImageURL: rsslayEntity.ImageURL,
LastPostTime: rsslayEntity.LastPostTime},
NPubKey: npub,
})
}
Expand All @@ -454,7 +452,6 @@ func getSavedEntries() ([]GUIEntry, error) {
return localEntries, nil
}

// find the saved Entity with the given pubkey
func getSavedEntity(pubkeyHex string) (Entity, error) {
var bookMarkTags nostr.Tags
var rsslayEntity Entity
Expand Down Expand Up @@ -487,7 +484,6 @@ func getSavedEntity(pubkeyHex string) (Entity, error) {
return Entity{}, nil
}

// return all feedURL Entities
func getSavedEntities() ([]Entity, error) {
var bookMarkTags nostr.Tags
var rsslayEntity Entity
Expand Down Expand Up @@ -518,7 +514,6 @@ func getSavedEntities() ([]Entity, error) {
return entities, nil
}

// delete event from relay db
func deleteLocalEvents(filter nostr.Filter) error {
ctx := context.TODO()

Expand Down
47 changes: 30 additions & 17 deletions feed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,15 +348,21 @@ func GetConverterRules() []md.Rule {
}
}

func updateAllFeeds() {
var latestCreatedAt int64
//metrics.ListeningFiltersOps.Inc()
func checkAllFeeds() {
currentEntities, err := getSavedEntities()
if err != nil {
log.Print("[ERROR] could not retrieve entities")
return
}
for _, currentEntity := range currentEntities {
if !TimetoUpdateFeed(currentEntity) {
//log.Printf("[DEBUG] not time to update %s. Time since last check: %d Avg post time: %d", currentEntity.URL, time.Now().Unix()-currentEntity.LastCheckedTime, currentEntity.AvgPostTime)
continue
}

lastPostTime := int64(0)
allPostTimes := make([]int64, 0)

parsedFeed, entity := parseFeedForPubkey(currentEntity.PubKey, s.DeleteFailingFeeds)
if parsedFeed == nil {
return
Expand All @@ -369,7 +375,7 @@ func updateAllFeeds() {
for _, item := range parsedFeed.Items {
defaultCreatedAt := time.Unix(time.Now().Unix(), 0)
evt := feedItemToNote(currentEntity.PubKey, item, parsedFeed, defaultCreatedAt, entity.URL, s.MaxContentLength)
if entity.LastUpdate < evt.CreatedAt.Time().Unix() {
if entity.LastPostTime < evt.CreatedAt.Time().Unix() {
if err := evt.Sign(entity.PrivateKey); err != nil {
log.Printf("[ERROR] %s", err)
return
Expand All @@ -381,25 +387,30 @@ func updateAllFeeds() {
for _, store := range relay.StoreEvent {
store(context.TODO(), &evt)
}
} else {
log.Printf("[DEBUG] event id %s created at %d older than last update %d", evt.ID, evt.CreatedAt.Time().Unix(), entity.LastUpdate)
}
if evt.CreatedAt.Time().Unix() > latestCreatedAt {
latestCreatedAt = evt.CreatedAt.Time().Unix()

if evt.CreatedAt.Time().Unix() > lastPostTime {
lastPostTime = evt.CreatedAt.Time().Unix()
}

allPostTimes = append(allPostTimes, evt.CreatedAt.Time().Unix())
}

if err := updateEntityInBookmarkEvent(entity.PubKey, latestCreatedAt); err != nil {
if err := updateEntityTimesInBookmarkEvent(Entity{
PubKey: entity.PubKey,
LastPostTime: lastPostTime,
LastCheckedTime: time.Now().Unix(),
AvgPostTime: CalcAvgPostTime(allPostTimes),
}); err != nil {
log.Printf("[ERROR] feed entity %s not updated", entity.URL)
}

latestCreatedAt = 0
}
}

// init feed by creating kind 1's from rss feed
func initFeed(pubkey string, privkey string, feedURL string, parsedFeed *gofeed.Feed) int64 {
var latestCreatedAt int64
func initFeed(pubkey string, privkey string, feedURL string, parsedFeed *gofeed.Feed) (int64, []int64) {
var lastPostTime int64
postTimes := make([]int64, 0)

for _, item := range parsedFeed.Items {
defaultCreatedAt := time.Unix(time.Now().Unix(), 0)
evt := feedItemToNote(pubkey, item, parsedFeed, defaultCreatedAt, feedURL, s.MaxContentLength)
Expand All @@ -415,10 +426,12 @@ func initFeed(pubkey string, privkey string, feedURL string, parsedFeed *gofeed.
store(context.TODO(), &evt)
}

if evt.CreatedAt.Time().Unix() > latestCreatedAt {
latestCreatedAt = evt.CreatedAt.Time().Unix()
if evt.CreatedAt.Time().Unix() > lastPostTime {
lastPostTime = evt.CreatedAt.Time().Unix()
}

postTimes = append(postTimes, evt.CreatedAt.Time().Unix())
}

return latestCreatedAt
return lastPostTime, postTimes
}
26 changes: 18 additions & 8 deletions handlers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,16 @@ func createFeed(r *http.Request, secret *string) *GUIEntry {
publishNostrEventCh <- metadataEvent
}

latestCreatedAt := initFeed(publicKey, sk, feedUrl, parsedFeed)
lastPostTime, allPostTimes := initFeed(publicKey, sk, feedUrl, parsedFeed)

if err := addEntityToBookmarkEvent([]Entity{
{PubKey: publicKey,
PrivateKey: sk,
URL: feedUrl,
ImageURL: guientry.BookmarkEntity.ImageURL,
LastUpdate: latestCreatedAt}}); err != nil {
PrivateKey: sk,
URL: feedUrl,
ImageURL: guientry.BookmarkEntity.ImageURL,
LastPostTime: lastPostTime,
LastCheckedTime: time.Now().Unix(),
AvgPostTime: CalcAvgPostTime(allPostTimes)}}); err != nil {
log.Printf("[ERROR] feed entity %s not added to bookmark", feedUrl)
}

Expand Down Expand Up @@ -383,8 +385,16 @@ func importFeeds(opmlUrls []opml.Outline, secret *string) []*GUIEntry {
localImageURL = parsedFeed.Image.URL
}

latestCreatedAt := initFeed(publicKey, sk, feedUrl, parsedFeed)
bookmarkEntities = append(bookmarkEntities, Entity{PubKey: publicKey, PrivateKey: sk, URL: feedUrl, ImageURL: localImageURL, LastUpdate: latestCreatedAt})
lastPostTime, allPostTimes := initFeed(publicKey, sk, feedUrl, parsedFeed)
bookmarkEntities = append(bookmarkEntities, Entity{
PubKey: publicKey,
PrivateKey: sk,
URL: feedUrl,
ImageURL: localImageURL,
LastPostTime: lastPostTime,
LastCheckedTime: time.Now().Unix(),
AvgPostTime: CalcAvgPostTime(allPostTimes),
})

importedEntries = append(importedEntries, &guiEntry)
importProgressCh <- ImportProgressStruct{entryIndex: urlIndex, totalEntries: len(opmlUrls)}
Expand Down Expand Up @@ -586,7 +596,7 @@ func updateRssNotesState() {
blastEvent(&nostrEvent)
}()
case <-tickerUpdateFeeds.C:
updateAllFeeds()
checkAllFeeds()
case <-tickerDeleteOldNotes.C:
deleteOldEvents()
case <-quitChannel:
Expand Down
25 changes: 25 additions & 0 deletions helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"os"
"path"
"slices"
"sort"
"strings"
"time"
)
Expand Down Expand Up @@ -82,3 +83,27 @@ func GetRelayListFromFile(filePath string) []string {
}
return relayList
}

func CalcAvgPostTime(feedPostTimes []int64) int64 {
if len(feedPostTimes) < s.MinPostPeriodSamples {
return int64(s.MaxAvgPostPeriodHrs * 60 * 60)
}

sort.SliceStable(feedPostTimes, func(i, j int) bool {
return feedPostTimes[i] > feedPostTimes[j]
})

avgposttimesecs := (feedPostTimes[0] - feedPostTimes[len(feedPostTimes)-1]) / int64(len(feedPostTimes))

if avgposttimesecs < int64(s.MinAvgPostPeriodMins*60) {
return int64(s.MinAvgPostPeriodMins * 60)
} else if avgposttimesecs > int64(s.MaxAvgPostPeriodHrs*60*60) {
return int64(s.MaxAvgPostPeriodHrs * 60 * 60)
}

return avgposttimesecs
}

func TimetoUpdateFeed(rssfeed Entity) bool {
return time.Now().Unix()-rssfeed.LastCheckedTime >= rssfeed.AvgPostTime
}
5 changes: 4 additions & 1 deletion main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ type Settings struct {
RelayIcon string `envconfig:"RELAY_ICON" default:"https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons/png/commafeed.png"`
RandomSecret string `envconfig:"RANDOM_SECRET" required:"true"`
OwnerPubkey string `envconfig:"OWNER_PUBKEY"`
Version string `envconfig:"VERSION" default:"0.0.7"`
Version string `envconfig:"VERSION" default:"0.0.8"`

LogLevel string `envconfig:"LOG_LEVEL" default:"WARN"`
Port string `envconfig:"PORT" default:"3334"`
Expand All @@ -50,6 +50,9 @@ type Settings struct {
FeedItemsRefreshMinutes int `envconfig:"FEED_ITEMS_REFRESH_MINUTES" default:"30"`
FeedMetadataRefreshDays int `envconfig:"METADATA_REFRESH_DAYS" default:"7"`
MaxNoteAgeDays int `envconfig:"MAX_NOTE_AGE_DAYS" default:"0"`
MaxAvgPostPeriodHrs int64 `envconfig:"MAX_AVG_POST_PERIOD_HRS" default:"4"`
MinAvgPostPeriodMins int64 `envconfig:"MIN_AVG_POST_PERIOD_MINS" default:"10"`
MinPostPeriodSamples int `envconfig:"MIN_POST_PERIOD_SAMPLES" default:"5"`
}

var (
Expand Down
12 changes: 7 additions & 5 deletions models.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@ type ImportProgressStruct struct {
}

type Entity struct {
PubKey string
PrivateKey string
URL string
ImageURL string
LastUpdate int64
PubKey string
PrivateKey string
URL string
ImageURL string
LastPostTime int64
AvgPostTime int64
LastCheckedTime int64
}

type GUIEntry struct {
Expand Down

0 comments on commit 759a915

Please sign in to comment.