feat!(source): Increase backlog limit defaults and add config control #…

…153 * Increase default backlog limits to maximum pagination supported by each source * Add scrobbleBacklogCount config property so user can explicitly set # of tracks to backlog
FoxxMD · May 16, 2024 · 57dff9d · 57dff9d
1 parent 963fc79
commit 57dff9d
Show file tree

Hide file tree

Showing 9 changed files with 57 additions and 8 deletions.
diff --git a/src/backend/common/infrastructure/config/source/index.ts b/src/backend/common/infrastructure/config/source/index.ts
@@ -88,6 +88,14 @@ export interface CommonSourceOptions extends SourceRetryOptions {
      * Set thresholds for when multi-scrobbler should consider a tracked play to be "scrobbable". If both duration and percent are defined then if either condition is met the track is scrobbled.
      * */
     scrobbleThresholds?: ScrobbleThresholds
+
+    /**
+     * The number of listens to fetch when scrobbling from backlog
+     *
+     * * Only applies if this source supports fetching a listen history
+     * * If not specified it defaults to the maximum number of listens the source API supports
+     * */
+    scrobbleBacklogCount?: number
 }
 
 export interface CommonSourceData extends CommonData {

diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json
@@ -283,6 +283,11 @@
                     "title": "scrobbleBacklog",
                     "type": "boolean"
                 },
+                "scrobbleBacklogCount": {
+                    "description": "The number of listens to fetch when scrobbling from backlog\n\n* Only applies if this source supports fetching a listen history\n* If not specified it defaults to the maximum number of listens the source API supports",
+                    "title": "scrobbleBacklogCount",
+                    "type": "number"
+                },
                 "scrobbleThresholds": {
                     "$ref": "#/definitions/ScrobbleThresholds",
                     "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.",

diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json
@@ -506,6 +506,11 @@
                     "title": "scrobbleBacklog",
                     "type": "boolean"
                 },
+                "scrobbleBacklogCount": {
+                    "description": "The number of listens to fetch when scrobbling from backlog\n\n* Only applies if this source supports fetching a listen history\n* If not specified it defaults to the maximum number of listens the source API supports",
+                    "title": "scrobbleBacklogCount",
+                    "type": "number"
+                },
                 "scrobbleThresholds": {
                     "$ref": "#/definitions/ScrobbleThresholds",
                     "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.",
@@ -2284,6 +2289,11 @@
                     "title": "scrobbleBacklog",
                     "type": "boolean"
                 },
+                "scrobbleBacklogCount": {
+                    "description": "The number of listens to fetch when scrobbling from backlog\n\n* Only applies if this source supports fetching a listen history\n* If not specified it defaults to the maximum number of listens the source API supports",
+                    "title": "scrobbleBacklogCount",
+                    "type": "number"
+                },
                 "scrobbleThresholds": {
                     "$ref": "#/definitions/ScrobbleThresholds",
                     "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.",

diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json
@@ -322,6 +322,11 @@
                     "title": "scrobbleBacklog",
                     "type": "boolean"
                 },
+                "scrobbleBacklogCount": {
+                    "description": "The number of listens to fetch when scrobbling from backlog\n\n* Only applies if this source supports fetching a listen history\n* If not specified it defaults to the maximum number of listens the source API supports",
+                    "title": "scrobbleBacklogCount",
+                    "type": "number"
+                },
                 "scrobbleThresholds": {
                     "$ref": "#/definitions/ScrobbleThresholds",
                     "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.",

diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts
@@ -79,6 +79,8 @@ export default abstract class AbstractSource implements Authenticatable {
 
     emitter: EventEmitter;
 
+    protected SCROBBLE_BACKLOG_COUNT: number = 20;
+
     protected recentDiscoveredPlays: GroupedFixedPlays = new TupleMap<DeviceId, PlayUserId, FixedSizeList<ProgressAwarePlayObject>>();
 
     constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) {
@@ -314,8 +316,17 @@ export default abstract class AbstractSource implements Authenticatable {
         if (this.canBacklog) {
             this.logger.info('Discovering backlogged tracks from recently played API...');
             let backlogPlays: PlayObject[] = [];
+            const {
+                scrobbleBacklogCount = this.SCROBBLE_BACKLOG_COUNT
+            } = this.config.options || {};
+            let backlogLimit = scrobbleBacklogCount;
+            if(backlogLimit > this.SCROBBLE_BACKLOG_COUNT) {
+                this.logger.warn(`scrobbleBacklogCount (${scrobbleBacklogCount}) cannot be greater than max API limit (${this.SCROBBLE_BACKLOG_COUNT}), reverting to max...`);
+                backlogLimit = this.SCROBBLE_BACKLOG_COUNT;
+            }
             try {
-                backlogPlays = await this.getBackloggedPlays();
+                this.logger.verbose(`Fetching the last ${backlogLimit}${backlogLimit === this.SCROBBLE_BACKLOG_COUNT ? ' (max) ' : ''} listens to check for backlogging...`);
+                backlogPlays = await this.getBackloggedPlays({limit: backlogLimit});
             } catch (e) {
                 throw new Error('Error occurred while fetching backlogged plays', {cause: e});
             }
@@ -341,7 +352,7 @@ export default abstract class AbstractSource implements Authenticatable {
         }
     }
 
-    protected getBackloggedPlays = async (): Promise<PlayObject[]> => {
+    protected getBackloggedPlays = async (options: RecentlyPlayedOptions): Promise<PlayObject[]> => {
         this.logger.debug('Backlogging not implemented');
         return [];
     }

diff --git a/src/backend/sources/DeezerSource.ts b/src/backend/sources/DeezerSource.ts
@@ -48,6 +48,9 @@ export default class DeezerSource extends AbstractSource {
         this.canPoll = true;
         this.canBacklog = true;
         this.supportsUpstreamRecentlyPlayed = true;
+        // https://developers.deezer.com/api/user/history
+        // https://stackoverflow.com/a/19497151/1469797
+        this.SCROBBLE_BACKLOG_COUNT = 50;
     }
 
     static formatPlayObj(obj: any, options: FormatPlayObjectOptions = {}): PlayObject {
@@ -134,7 +137,7 @@ export default class DeezerSource extends AbstractSource {
     getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => this.getRecentlyPlayed(options)
 
     getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}) => {
-        const resp = await this.callApi(request.get(`${this.baseUrl}/user/me/history?limit=20`));
+        const resp = await this.callApi(request.get(`${this.baseUrl}/user/me/history?limit=${options.limit || 20}`));
         return resp.data.map((x: any) => DeezerSource.formatPlayObj(x)).sort(sortByOldestPlayDate);
     }
 
@@ -248,5 +251,5 @@ export default class DeezerSource extends AbstractSource {
         }
     }
 
-    protected getBackloggedPlays = async () => await this.getRecentlyPlayed({formatted: true})
+    protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getRecentlyPlayed({formatted: true, ...options})
 }
diff --git a/src/backend/sources/LastfmSource.ts b/src/backend/sources/LastfmSource.ts
@@ -34,6 +34,8 @@ export default class LastfmSource extends MemorySource {
         this.supportsUpstreamNowPlaying = true;
         this.api = new LastfmApiClient(name, {...config.data, configDir: internal.configDir, localUrl: internal.localUrl}, {logger: this.logger});
         this.playerSourceOfTruth = SOURCE_SOT.HISTORY;
+        // https://www.last.fm/api/show/user.getRecentTracks
+        this.SCROBBLE_BACKLOG_COUNT = 200;
         this.logger.info(`Note: The player for this source is an analogue for the 'Now Playing' status exposed by ${this.type} which is NOT used for scrobbling. Instead, the 'recently played' or 'history' information provided by this source is used for scrobbles.`)
     }
 
@@ -151,5 +153,5 @@ export default class LastfmSource extends MemorySource {
         }
     }
 
-    protected getBackloggedPlays = async () => await this.getRecentlyPlayed({formatted: true})
+    protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getRecentlyPlayed({formatted: true, ...options})
 }
diff --git a/src/backend/sources/ListenbrainzSource.ts b/src/backend/sources/ListenbrainzSource.ts
@@ -29,7 +29,10 @@ export default class ListenbrainzSource extends MemorySource {
         this.canBacklog = true;
         this.api = new ListenbrainzApiClient(name, config.data, {logger: this.logger});
         this.playerSourceOfTruth = SOURCE_SOT.HISTORY;
-        this.supportsUpstreamRecentlyPlayed = true;
+        this.supportsUpstreamRecentlyPlayed = true
+        // https://listenbrainz.readthedocs.io/en/latest/users/api/core.html#get--1-user-(user_name)-listens
+        // 1000 is way too high. maxing at 100
+        this.SCROBBLE_BACKLOG_COUNT = 100;
         this.logger.info(`Note: The player for this source is an analogue for the 'Now Playing' status exposed by ${this.type} which is NOT used for scrobbling. Instead, the 'recently played' or 'history' information provided by this source is used for scrobbles.`)
     }
 
@@ -77,5 +80,5 @@ export default class ListenbrainzSource extends MemorySource {
         }
     }
 
-    protected getBackloggedPlays = async () => await this.getRecentlyPlayed({formatted: true})
+    protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) =>  await this.getRecentlyPlayed({formatted: true, ...options})
 }
diff --git a/src/backend/sources/SpotifySource.ts b/src/backend/sources/SpotifySource.ts
@@ -68,6 +68,8 @@ export default class SpotifySource extends MemorySource {
         this.canPoll = true;
         this.canBacklog = true;
         this.supportsUpstreamRecentlyPlayed = true;
+        // https://developer.spotify.com/documentation/web-api/reference/get-recently-played
+        this.SCROBBLE_BACKLOG_COUNT = 50
     }
 
     static formatPlayObj(obj: PlayHistoryObject | CurrentlyPlayingObject, options: FormatPlayObjectOptions = {}): PlayObject {
@@ -502,7 +504,7 @@ export default class SpotifySource extends MemorySource {
         return true;
     }
 
-    protected getBackloggedPlays = async () => await this.getPlayHistory({formatted: true})
+    protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getPlayHistory({formatted: true, ...options})
 }
 
 const asPlayHistoryObject = (obj: object): obj is PlayHistoryObject => 'played_at' in obj