From 1240f7dc81d78f26ef40a665e1203f05aa9b644c Mon Sep 17 00:00:00 2001 From: Revadike Date: Wed, 9 Mar 2022 04:02:46 +0100 Subject: [PATCH 01/25] Added savePicsCache --- README.md | 10 ++++++++++ components/apps.js | 27 +++++++++++++++++++++++++++ resources/default_options.js | 1 + 3 files changed, 38 insertions(+) diff --git a/README.md b/README.md index 83649c4b..056ab134 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,16 @@ Added in 3.3.0. Defaults to `false`. +### savePicsCache + +If `enablePicsCache` is enabled, save product info to disk. (TODO: Improve description) + +Added in 4.24.0 + +Defaults to `false`. + +**Warning:** This will significantly increase the storage space that is needed! + ### picsCacheAll If `picsCacheAll` is enabled, `enablePicsCache` is enabled, and `changelistUpdateInterval` is nonzero, then apps and diff --git a/components/apps.js b/components/apps.js index 8b027905..818b0d30 100644 --- a/components/apps.js +++ b/components/apps.js @@ -156,6 +156,30 @@ class SteamUserApps extends SteamUserAppAuth { }); } + _saveProductInfo({ apps, packages }) { + let toSave = []; + + for (let appid in apps) { + if (apps[appid].missingToken) { + continue; + } + let filename = `app_info_${appid}.json`; + let content = JSON.stringify(apps[appid]); + toSave.push({filename, content}); + } + + for (let packageid in packages) { + if (packages[packageid].missingToken) { + continue; + } + let filename = `package_info_${packageid}.json`; + let content = JSON.stringify(packages[packageid]); + toSave.push({filename, content}); + } + + return Promise.all(toSave.map(({filename, content}) => this._saveFile(filename, content))); + } + /** * Get info about some apps and/or packages from Steam. * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} @@ -313,6 +337,7 @@ class SteamUserApps extends SteamUserAppAuth { // appids and packageids contain the list of IDs that we're still waiting on data for if (appids.length === 0 && packageids.length === 0) { if (!inclTokens) { + this._saveProductInfo(response); return resolve(response); } @@ -334,6 +359,7 @@ class SteamUserApps extends SteamUserAppAuth { if (tokenlessAppids.length == 0 && tokenlessPackages.length == 0) { // No tokens needed + this._saveProductInfo(response); return resolve(response); } @@ -374,6 +400,7 @@ class SteamUserApps extends SteamUserAppAuth { } } + this._saveProductInfo(response); resolve(response); } catch (ex) { return reject(ex); diff --git a/resources/default_options.js b/resources/default_options.js index 098ec39c..9eb8d480 100644 --- a/resources/default_options.js +++ b/resources/default_options.js @@ -28,6 +28,7 @@ module.exports = { machineIdType: EMachineIDType.AccountNameGenerated, machineIdFormat: ['SteamUser Hash BB3 {account_name}', 'SteamUser Hash FF2 {account_name}', 'SteamUser Hash 3B3 {account_name}'], enablePicsCache: false, + savePicsCache: false, picsCacheAll: false, changelistUpdateInterval: 60000, additionalHeaders: {}, From 0321d67076f1f43b74c2da48b7d0714fe87e5b71 Mon Sep 17 00:00:00 2001 From: Revadike Date: Fri, 11 Mar 2022 05:11:02 +0100 Subject: [PATCH 02/25] update jsdoc --- resources/default_options.js | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/default_options.js b/resources/default_options.js index 9eb8d480..1042fdf8 100644 --- a/resources/default_options.js +++ b/resources/default_options.js @@ -9,6 +9,7 @@ const EMachineIDType = require('./EMachineIDType.js'); * @property {EMachineIDType} [machineIdType] * @property {string[]} [machineIdFormat] * @property {boolean} [enablePicsCache=false] + * @property {boolean} [savePicsCache=false] * @property {boolean} [picsCacheAll=false] * @property {number} [changelistUpdateInterval=60000] * @property {PackageFilter|PackageFilterFunction|null} [ownershipFilter=null] From aa2cc26f2b5770909f48c11a154803e24cdaa4db Mon Sep 17 00:00:00 2001 From: Revadike Date: Fri, 20 May 2022 05:43:57 +0200 Subject: [PATCH 03/25] fix jsdoc --- components/apps.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/apps.js b/components/apps.js index 818b0d30..0c5d31db 100644 --- a/components/apps.js +++ b/components/apps.js @@ -137,7 +137,7 @@ class SteamUserApps extends SteamUserAppAuth { * Get a list of apps or packages which have changed since a particular changenumber. * @param {int} sinceChangenumber - Changenumber to get changes since. Use 0 to get the latest changenumber, but nothing else * @param {function} [callback] - * @returns {Promise<{currentChangeNumber: number, appChanges: number[], packageChanges: number[]}>} + * @returns {Promise<{currentChangeNumber: number, appChanges: object[], packageChanges: object[]}>} */ getProductChanges(sinceChangenumber, callback) { let args = ['currentChangeNumber', 'appChanges', 'packageChanges']; From a9bd0cf96a94248ca47e5a0838d23fd7f3d54f25 Mon Sep 17 00:00:00 2001 From: Revadike Date: Fri, 20 May 2022 05:59:38 +0200 Subject: [PATCH 04/25] issue warning for picsCacheAll+savePicsCache --- index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.js b/index.js index 439bfec0..74122825 100644 --- a/index.js +++ b/index.js @@ -88,6 +88,10 @@ class SteamUser extends SteamUserTwoFactor { this.storage = new FileManager(this.options.dataDirectory); } + if (this.options.picsCacheAll && this.options.savePicsCache) { + this._warn('Both picsCacheAll and savePicsCache are enabled, beware that this will cause a lot of disk IO and space usage!'); + } + this._initialized = true; } From c0afe0480f52512a8da81881137e5b1a033eeacd Mon Sep 17 00:00:00 2001 From: Revadike Date: Fri, 20 May 2022 06:10:41 +0200 Subject: [PATCH 05/25] _saveProductInfo jsdoc --- components/apps.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/apps.js b/components/apps.js index 0c5d31db..29808b1f 100644 --- a/components/apps.js +++ b/components/apps.js @@ -156,6 +156,10 @@ class SteamUserApps extends SteamUserAppAuth { }); } + /** + * @param {apps: Object, packages: Object} - Response from getProductInfo + * @protected + */ _saveProductInfo({ apps, packages }) { let toSave = []; From 488a74ac63020496f33354f1c5342a8f68d2d5e8 Mon Sep 17 00:00:00 2001 From: Revadike Date: Fri, 20 May 2022 06:15:11 +0200 Subject: [PATCH 06/25] savePicsCache functionality --- components/apps.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/apps.js b/components/apps.js index 29808b1f..0b41ec41 100644 --- a/components/apps.js +++ b/components/apps.js @@ -161,6 +161,10 @@ class SteamUserApps extends SteamUserAppAuth { * @protected */ _saveProductInfo({ apps, packages }) { + if (!this.options.savePicsCache) { + return Promise.resolve([]); + } + let toSave = []; for (let appid in apps) { From 24eb205f5a9fd9c7eb30274af185b5d6411cd5ac Mon Sep 17 00:00:00 2001 From: Revadike Date: Fri, 20 May 2022 17:39:05 +0200 Subject: [PATCH 07/25] current changenumber to disk --- components/apps.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/apps.js b/components/apps.js index 0b41ec41..fd8f601c 100644 --- a/components/apps.js +++ b/components/apps.js @@ -518,6 +518,7 @@ class SteamUserApps extends SteamUserAppAuth { } cache.changenumber = currentChangeNumber; + this._saveFile('changenumber.txt', currentChangeNumber); this._resetChangelistUpdateTimer(); return; } @@ -535,6 +536,7 @@ class SteamUserApps extends SteamUserAppAuth { let {appTokens, packageTokens} = result; cache.changenumber = currentChangeNumber; + this._saveFile('changenumber.txt', currentChangeNumber); this._resetChangelistUpdateTimer(); let index = -1; @@ -957,9 +959,7 @@ SteamUserBase.prototype._handlerManager.add(EMsg.ClientLicenseList, function(bod this.licenses = body.licenses; // Request info for our licenses - if (this.options.enablePicsCache) { - this._getLicenseInfo(); - } + this._getLicenseInfo(); }); SteamUserBase.prototype._handlerManager.add(EMsg.ClientPlayingSessionState, function(body) { From 3d45029554e97747c5bda6a491888e88da2fa331 Mon Sep 17 00:00:00 2001 From: Revadike Date: Fri, 20 May 2022 19:10:35 +0200 Subject: [PATCH 08/25] read changenumber on startup --- index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/index.js b/index.js index 74122825..6c2ea021 100644 --- a/index.js +++ b/index.js @@ -86,6 +86,11 @@ class SteamUser extends SteamUserTwoFactor { if (this.options.dataDirectory) { this.storage = new FileManager(this.options.dataDirectory); + this._readFile('changenumber.txt').then((changenumber) => { + if (changenumber && isFinite(changenumber)) { + this.picsCache.changenumber = parseInt(changenumber, 10); + } + }); } if (this.options.picsCacheAll && this.options.savePicsCache) { From ec7aa5d5e5d8d1fb3e3545591b39e7e8758484f2 Mon Sep 17 00:00:00 2001 From: Revadike Date: Sat, 21 May 2022 18:07:15 +0200 Subject: [PATCH 09/25] altered warning --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 6c2ea021..c507b6ac 100644 --- a/index.js +++ b/index.js @@ -94,7 +94,7 @@ class SteamUser extends SteamUserTwoFactor { } if (this.options.picsCacheAll && this.options.savePicsCache) { - this._warn('Both picsCacheAll and savePicsCache are enabled, beware that this will cause a lot of disk IO and space usage!'); + this._warn('Both picsCacheAll and savePicsCache are enabled, unless a custom storage engine is used, beware that this will cause a lot of disk IO and space usage!'); } this._initialized = true; From 84bfec3e44f073c98c6beca2b0ff2d412602253a Mon Sep 17 00:00:00 2001 From: Revadike Date: Sat, 21 May 2022 18:08:18 +0200 Subject: [PATCH 10/25] spelling --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index c507b6ac..b6c87a7b 100644 --- a/index.js +++ b/index.js @@ -94,7 +94,7 @@ class SteamUser extends SteamUserTwoFactor { } if (this.options.picsCacheAll && this.options.savePicsCache) { - this._warn('Both picsCacheAll and savePicsCache are enabled, unless a custom storage engine is used, beware that this will cause a lot of disk IO and space usage!'); + this._warn('Both picsCacheAll and savePicsCache are enabled. Unless a custom storage engine is used, beware that this will cause a lot of disk IO and space usage!'); } this._initialized = true; From 5de1d3538d195aaaa70f54de5700928de24e7b59 Mon Sep 17 00:00:00 2001 From: Revadike Date: Sat, 21 May 2022 19:19:15 +0200 Subject: [PATCH 11/25] implement product info cache --- components/apps.js | 137 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 130 insertions(+), 7 deletions(-) diff --git a/components/apps.js b/components/apps.js index fd8f601c..9a79e7f2 100644 --- a/components/apps.js +++ b/components/apps.js @@ -188,6 +188,87 @@ class SteamUserApps extends SteamUserAppAuth { return Promise.all(toSave.map(({filename, content}) => this._saveFile(filename, content))); } + /** + * Get cached product info. + * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} + * @param {int[]|object[]} packages - Array of package IDs. May be empty. May also contain objects with keys {packageid, access_token} + * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} + * @protected + */ + async _getProductInfo(apps, packages) { + let response = { + apps: {}, + packages: {}, + unknownApps: [], + unknownPackages: [] + }; + + // With pics cache disabled, we cannot assure cache is up to date. + if (!this.options.enablePicsCache) { + return response; + } + + // From this point, we can assume pics cache is up to date (via changelist updates). + let appids = apps.map(app => typeof app === 'object' ? app.appid : app); + let packageids = packages.map(pkg => typeof pkg === 'object' ? pkg.packageid : pkg); + for (let appid of appids) { + if (this.picsCache.apps[appid]) { + response.apps[appid] = this.picsCache.apps[appid]; + } else { + response.unknownApps.push(appid); + } + } + for (let packageid of packageids) { + if (this.picsCache.packages[packageid]) { + response.packages[packageid] = this.picsCache.packages[packageid]; + } else { + response.unknownPackages.push(packageid); + } + } + + // If everything was already in memory cache, we're done. + if (response.unknownApps.length === 0 && response.unknownPackages.length === 0) { + return response; + } + + // Otherwise, we try loading the missing apps & packages from disk. + appids = response.unknownApps; + packageids = response.unknownPackages; + response.unknownApps = []; + response.unknownPackages = []; + let appFiles = {}; + let packageFiles = {}; + for (let appid of appids) { + let filename = `app_info_${appid}.json`; + appFiles[filename] = appid; + } + for (let packageid of packageids) { + let filename = `package_info_${packageid}.json`; + packageFiles[filename] = packageid; + } + let files = await this._loadFiles(Object.keys(appFiles).concat(Object.keys(packageFiles))); + for (let file of files) { + let {filename, error, content} = file; + let appid = appFiles[filename]; + let packageid = packageFiles[filename]; + if (appid) { + if (error) { + response.unknownApps.push(appid); + } else { + response.apps[appid] = JSON.parse(content); + } + } else if (packageid) { + if (error) { + response.unknownPackages.push(packageid); + } else { + response.packages[packageid] = JSON.parse(content); + } + } + } + + return response; + } + /** * Get info about some apps and/or packages from Steam. * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} @@ -205,8 +286,8 @@ class SteamUserApps extends SteamUserAppAuth { inclTokens = false; } - // This one actually can take a while, so allow it to go as long as 60 minutes - return StdLib.Promises.timeoutCallbackPromise(3600000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, (resolve, reject) => { + // This one actually can take a while, so allow it to go as long as 120 minutes + return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { requestType = requestType || PICSRequestType.User; // Steam can send us the full response in multiple responses, so we need to buffer them into one callback @@ -218,8 +299,25 @@ class SteamUserApps extends SteamUserAppAuth { unknownApps: [], unknownPackages: [] }; + let cached = response; // Same format as response, but with cached data + + // Changelist requests always require fresh product info + if (this.options.enablePicsCache && requestType !== PICSRequestType.Changelist) { + cached = await this._getCachedProductInfo(apps, packages); + // If we already have all the info, we can return it immediately + if (cached.unknownApps.length === 0 && cached.unknownPackages.length === 0) { + return resolve(cached); + } + } - apps = apps.map((app) => { + // Filter out any apps we already have cached + apps = apps.filter((app) => { + if (typeof app === 'object') { + return cached.unknownApps.includes(app.appid); + } else { + return cached.unknownApps.includes(app); + } + }).map((app) => { if (typeof app === 'object') { appids.push(app.appid); return app; @@ -229,7 +327,14 @@ class SteamUserApps extends SteamUserAppAuth { } }); - packages = packages.map((pkg) => { + // Filter out any packages we already have cached + packages = packages.filter((pkg) => { + if (typeof pkg === 'object') { + return cached.unknownPackages.includes(pkg.packageid); + } else { + return cached.unknownPackages.includes(pkg); + } + }).map((pkg) => { if (typeof pkg === 'object') { packageids.push(pkg.packageid); return pkg; @@ -346,7 +451,13 @@ class SteamUserApps extends SteamUserAppAuth { if (appids.length === 0 && packageids.length === 0) { if (!inclTokens) { this._saveProductInfo(response); - return resolve(response); + let combined = { + apps: Object.assign({}, response.apps, cached.apps), + packages: Object.assign({}, response.packages, cached.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + return resolve(combined); } // We want tokens @@ -368,7 +479,13 @@ class SteamUserApps extends SteamUserAppAuth { if (tokenlessAppids.length == 0 && tokenlessPackages.length == 0) { // No tokens needed this._saveProductInfo(response); - return resolve(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + return resolve(combined); } try { @@ -409,7 +526,13 @@ class SteamUserApps extends SteamUserAppAuth { } this._saveProductInfo(response); - resolve(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + resolve(combined); } catch (ex) { return reject(ex); } From 1a4e46de788b97a90f59c0241b3cb63c6266a28e Mon Sep 17 00:00:00 2001 From: Revadike Date: Sun, 22 May 2022 02:44:45 +0200 Subject: [PATCH 12/25] a lot of bugfixes --- components/05-filestorage.js | 13 ++++++ components/apps.js | 91 +++++++++++++++++++++--------------- index.js | 3 ++ package.json | 3 +- 4 files changed, 72 insertions(+), 38 deletions(-) diff --git a/components/05-filestorage.js b/components/05-filestorage.js index 10144668..cde26d23 100644 --- a/components/05-filestorage.js +++ b/components/05-filestorage.js @@ -17,6 +17,19 @@ class SteamUserFileStorage extends SteamUserUtility { } }; + /** + * @param {Object} files - Keys are filenames, values are Buffer objects containing the file contents + * @return {Promise} + * @protected + */ + async _saveFiles(files) { + if (!this.storage) { + return Promise.reject(new Error('Storage system disabled')); + } + + return await this.storage.saveFiles(files); + }; + /** * @param {string} filename * @returns {Promise} diff --git a/components/apps.js b/components/apps.js index 9a79e7f2..d5a30de7 100644 --- a/components/apps.js +++ b/components/apps.js @@ -158,6 +158,7 @@ class SteamUserApps extends SteamUserAppAuth { /** * @param {apps: Object, packages: Object} - Response from getProductInfo + * @returns {Promise} * @protected */ _saveProductInfo({ apps, packages }) { @@ -165,15 +166,16 @@ class SteamUserApps extends SteamUserAppAuth { return Promise.resolve([]); } - let toSave = []; + let toSave = {}; for (let appid in apps) { - if (apps[appid].missingToken) { + // Public only apps are weird... + if (apps[appid].missingToken && !apps[appid].appinfo.public_only) { continue; } let filename = `app_info_${appid}.json`; - let content = JSON.stringify(apps[appid]); - toSave.push({filename, content}); + let contents = JSON.stringify(apps[appid]); + toSave[filename] = contents; } for (let packageid in packages) { @@ -181,29 +183,29 @@ class SteamUserApps extends SteamUserAppAuth { continue; } let filename = `package_info_${packageid}.json`; - let content = JSON.stringify(packages[packageid]); - toSave.push({filename, content}); + let contents = JSON.stringify(packages[packageid]); + toSave[filename] = contents; } - return Promise.all(toSave.map(({filename, content}) => this._saveFile(filename, content))); + return this._saveFiles(toSave); } /** * Get cached product info. * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} * @param {int[]|object[]} packages - Array of package IDs. May be empty. May also contain objects with keys {packageid, access_token} - * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} + * @returns {Promise<{apps: Object, packages: Object, notCachedApps: number[], notCachedPackages: number[]}>} * @protected */ - async _getProductInfo(apps, packages) { + async _getCachedProductInfo(apps, packages) { let response = { apps: {}, packages: {}, - unknownApps: [], - unknownPackages: [] + notCachedApps: [], + notCachedPackages: [] }; - // With pics cache disabled, we cannot assure cache is up to date. + // With pics cache disabled, we cannot assure pics cache is up to date. if (!this.options.enablePicsCache) { return response; } @@ -215,27 +217,32 @@ class SteamUserApps extends SteamUserAppAuth { if (this.picsCache.apps[appid]) { response.apps[appid] = this.picsCache.apps[appid]; } else { - response.unknownApps.push(appid); + response.notCachedApps.push(appid); } } for (let packageid of packageids) { if (this.picsCache.packages[packageid]) { response.packages[packageid] = this.picsCache.packages[packageid]; } else { - response.unknownPackages.push(packageid); + response.notCachedPackages.push(packageid); } } // If everything was already in memory cache, we're done. - if (response.unknownApps.length === 0 && response.unknownPackages.length === 0) { + if (response.notCachedApps.length === 0 && response.notCachedPackages.length === 0) { + return response; + } + + // If pics cache is not being saved to disk, we're done. + if (!this.options.savePicsCache) { return response; } // Otherwise, we try loading the missing apps & packages from disk. - appids = response.unknownApps; - packageids = response.unknownPackages; - response.unknownApps = []; - response.unknownPackages = []; + appids = response.notCachedApps; + packageids = response.notCachedPackages; + response.notCachedApps = []; + response.notCachedPackages = []; let appFiles = {}; let packageFiles = {}; for (let appid of appids) { @@ -246,22 +253,28 @@ class SteamUserApps extends SteamUserAppAuth { let filename = `package_info_${packageid}.json`; packageFiles[filename] = packageid; } - let files = await this._loadFiles(Object.keys(appFiles).concat(Object.keys(packageFiles))); + let files = await this._readFiles(Object.keys(appFiles).concat(Object.keys(packageFiles))); for (let file of files) { - let {filename, error, content} = file; + let {filename, error, contents} = file; + if (Buffer.isBuffer(contents)) { + contents = contents.toString('utf8'); + } let appid = appFiles[filename]; let packageid = packageFiles[filename]; + if (appid) { - if (error) { - response.unknownApps.push(appid); + if (error || !contents) { + response.notCachedApps.push(appid); } else { - response.apps[appid] = JSON.parse(content); + this.picsCache.apps[appid] = JSON.parse(contents); + response.apps[appid] = this.picsCache.apps[appid]; } } else if (packageid) { - if (error) { - response.unknownPackages.push(packageid); + if (error || !contents) { + response.notCachedPackages.push(packageid); } else { - response.packages[packageid] = JSON.parse(content); + this.picsCache.packages[packageid] = JSON.parse(contents); + response.packages[packageid] = this.picsCache.packages[packageid]; } } } @@ -305,7 +318,7 @@ class SteamUserApps extends SteamUserAppAuth { if (this.options.enablePicsCache && requestType !== PICSRequestType.Changelist) { cached = await this._getCachedProductInfo(apps, packages); // If we already have all the info, we can return it immediately - if (cached.unknownApps.length === 0 && cached.unknownPackages.length === 0) { + if (cached.notCachedApps.length === 0 && cached.notCachedPackages.length === 0) { return resolve(cached); } } @@ -313,9 +326,9 @@ class SteamUserApps extends SteamUserAppAuth { // Filter out any apps we already have cached apps = apps.filter((app) => { if (typeof app === 'object') { - return cached.unknownApps.includes(app.appid); + return cached.notCachedApps.includes(app.appid); } else { - return cached.unknownApps.includes(app); + return cached.notCachedApps.includes(app); } }).map((app) => { if (typeof app === 'object') { @@ -330,9 +343,9 @@ class SteamUserApps extends SteamUserAppAuth { // Filter out any packages we already have cached packages = packages.filter((pkg) => { if (typeof pkg === 'object') { - return cached.unknownPackages.includes(pkg.packageid); + return cached.notCachedPackages.includes(pkg.packageid); } else { - return cached.unknownPackages.includes(pkg); + return cached.notCachedPackages.includes(pkg); } }).map((pkg) => { if (typeof pkg === 'object') { @@ -452,8 +465,8 @@ class SteamUserApps extends SteamUserAppAuth { if (!inclTokens) { this._saveProductInfo(response); let combined = { - apps: Object.assign({}, response.apps, cached.apps), - packages: Object.assign({}, response.packages, cached.packages), + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), unknownApps: response.unknownApps, unknownPackages: response.unknownPackages } @@ -641,7 +654,9 @@ class SteamUserApps extends SteamUserAppAuth { } cache.changenumber = currentChangeNumber; - this._saveFile('changenumber.txt', currentChangeNumber); + if (this.options.savePicsCache) { + this._saveFile('changenumber.txt', currentChangeNumber); + } this._resetChangelistUpdateTimer(); return; } @@ -659,7 +674,9 @@ class SteamUserApps extends SteamUserAppAuth { let {appTokens, packageTokens} = result; cache.changenumber = currentChangeNumber; - this._saveFile('changenumber.txt', currentChangeNumber); + if (this.options.savePicsCache) { + this._saveFile('changenumber.txt', currentChangeNumber); + } this._resetChangelistUpdateTimer(); let index = -1; @@ -694,7 +711,7 @@ class SteamUserApps extends SteamUserAppAuth { return; } - this.getProductInfo([appid], [], false, null, PICSRequestType.AddToCache).catch(() => { + this.getProductInfo([appid], [], true, null, PICSRequestType.AddToCache).catch(() => { }); } diff --git a/index.js b/index.js index b6c87a7b..6212b86b 100644 --- a/index.js +++ b/index.js @@ -86,6 +86,9 @@ class SteamUser extends SteamUserTwoFactor { if (this.options.dataDirectory) { this.storage = new FileManager(this.options.dataDirectory); + } + + if (this.options.savePicsCache) { this._readFile('changenumber.txt').then((changenumber) => { if (changenumber && isFinite(changenumber)) { this.picsCache.changenumber = parseInt(changenumber, 10); diff --git a/package.json b/package.json index 5fb8b294..091ac915 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "wtfnode": "^0.8.4" }, "scripts": { + "basicbot": "node examples/basicbot.js", "prepublishOnly": "node scripts/prepublish.js", "generate-enums": "node scripts/generate-enums.js", "generate-protos": "node scripts/generate-protos.js" @@ -50,4 +51,4 @@ "engines": { "node": ">=8.0.0" } -} +} \ No newline at end of file From e514b50035d1ab5c7928c017f8ab8ce359030195 Mon Sep 17 00:00:00 2001 From: Revadike Date: Sun, 22 May 2022 03:05:21 +0200 Subject: [PATCH 13/25] corrected cached object --- components/apps.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/apps.js b/components/apps.js index d5a30de7..13e59dbd 100644 --- a/components/apps.js +++ b/components/apps.js @@ -312,7 +312,12 @@ class SteamUserApps extends SteamUserAppAuth { unknownApps: [], unknownPackages: [] }; - let cached = response; // Same format as response, but with cached data + let cached = { + apps: {}, + packages: {}, + notCachedApps: [], + notCachedPackages: [] + }; // Changelist requests always require fresh product info if (this.options.enablePicsCache && requestType !== PICSRequestType.Changelist) { From 61bf5ca1d747b2a4df7edc1b3fa40d92baf2a155 Mon Sep 17 00:00:00 2001 From: Revadike Date: Sun, 22 May 2022 17:05:37 +0200 Subject: [PATCH 14/25] file-manager@2.0.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 091ac915..9ed88d7e 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "appdirectory": "^0.1.0", "binarykvparser": "^2.2.0", "bytebuffer": "^5.0.0", - "file-manager": "^2.0.0", + "file-manager": "^2.0.1", "lzma": "^2.3.2", "protobufjs": "^6.8.8", "steam-appticket": "^1.0.1", From 571b5d35498e1bb8ce72972ad0c0a342f9b6a10f Mon Sep 17 00:00:00 2001 From: Revadike Date: Thu, 2 Jun 2022 15:54:54 +0200 Subject: [PATCH 15/25] split product info requests --- components/apps.js | 484 ++++++++++++++++++++++++--------------------- 1 file changed, 264 insertions(+), 220 deletions(-) diff --git a/components/apps.js b/components/apps.js index 13e59dbd..97d57b3a 100644 --- a/components/apps.js +++ b/components/apps.js @@ -290,8 +290,9 @@ class SteamUserApps extends SteamUserAppAuth { * @param {function} [callback] * @param {int} [requestType] - Don't touch * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} + * @protected */ - getProductInfo(apps, packages, inclTokens, callback, requestType) { + async _getProductInfo(apps, packages, inclTokens, callback, requestType) { // Adds support for the previous syntax if (typeof inclTokens !== 'boolean' && typeof inclTokens === 'function') { requestType = callback; @@ -299,263 +300,306 @@ class SteamUserApps extends SteamUserAppAuth { inclTokens = false; } - // This one actually can take a while, so allow it to go as long as 120 minutes - return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { - requestType = requestType || PICSRequestType.User; - - // Steam can send us the full response in multiple responses, so we need to buffer them into one callback - let appids = []; - let packageids = []; - let response = { - apps: {}, - packages: {}, - unknownApps: [], - unknownPackages: [] - }; - let cached = { - apps: {}, - packages: {}, - notCachedApps: [], - notCachedPackages: [] - }; + requestType = requestType || PICSRequestType.User; - // Changelist requests always require fresh product info - if (this.options.enablePicsCache && requestType !== PICSRequestType.Changelist) { - cached = await this._getCachedProductInfo(apps, packages); - // If we already have all the info, we can return it immediately - if (cached.notCachedApps.length === 0 && cached.notCachedPackages.length === 0) { - return resolve(cached); - } + // Steam can send us the full response in multiple responses, so we need to buffer them into one callback + let appids = []; + let packageids = []; + let response = { + apps: {}, + packages: {}, + unknownApps: [], + unknownPackages: [] + }; + let cached = { + apps: {}, + packages: {}, + notCachedApps: [], + notCachedPackages: [] + }; + + // Changelist requests always require fresh product info + if (this.options.enablePicsCache && requestType !== PICSRequestType.Changelist) { + cached = await this._getCachedProductInfo(apps, packages); + // If we already have all the info, we can return it immediately + if (cached.notCachedApps.length === 0 && cached.notCachedPackages.length === 0) { + response.apps = cached.apps; + response.packages = cached.packages; + return response; } + } - // Filter out any apps we already have cached - apps = apps.filter((app) => { - if (typeof app === 'object') { - return cached.notCachedApps.includes(app.appid); - } else { - return cached.notCachedApps.includes(app); - } - }).map((app) => { - if (typeof app === 'object') { - appids.push(app.appid); - return app; - } else { - appids.push(app); - return {appid: app}; - } - }); + // Filter out any apps we already have cached + apps = apps.filter((app) => { + if (typeof app === 'object') { + return cached.notCachedApps.includes(app.appid); + } else { + return cached.notCachedApps.includes(app); + } + }).map((app) => { + if (typeof app === 'object') { + appids.push(app.appid); + return app; + } else { + appids.push(app); + return {appid: app}; + } + }); - // Filter out any packages we already have cached - packages = packages.filter((pkg) => { - if (typeof pkg === 'object') { - return cached.notCachedPackages.includes(pkg.packageid); - } else { - return cached.notCachedPackages.includes(pkg); - } - }).map((pkg) => { - if (typeof pkg === 'object') { - packageids.push(pkg.packageid); - return pkg; - } else { - packageids.push(pkg); - return {packageid: pkg}; + // Filter out any packages we already have cached + packages = packages.filter((pkg) => { + if (typeof pkg === 'object') { + return cached.notCachedPackages.includes(pkg.packageid); + } else { + return cached.notCachedPackages.includes(pkg); + } + }).map((pkg) => { + if (typeof pkg === 'object') { + packageids.push(pkg.packageid); + return pkg; + } else { + packageids.push(pkg); + return {packageid: pkg}; + } + }); + + if (inclTokens) { + packages.filter(pkg => !pkg.access_token).forEach((pkg) => { + // Check if we have a license for this package which includes a token + let license = this.licenses.find(lic => lic.package_id == pkg.packageid && lic.access_token != 0); + if (license) { + this.emit('debug', `Using token "${license.access_token}" from license for package ${pkg.packageid}`); + pkg.access_token = license.access_token; } }); + } + + this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, async (body) => { + // If we're using the PICS cache, then add the items in this response to it + if (this.options.enablePicsCache) { + let cache = this.picsCache; + cache.apps = cache.apps || {}; + cache.packages = cache.packages || {}; + + (body.apps || []).forEach((app) => { + let data = { + changenumber: app.change_number, + missingToken: !!app.missing_token, + appinfo: VDF.parse(app.buffer.toString('utf8')).appinfo + }; - if (inclTokens) { - packages.filter(pkg => !pkg.access_token).forEach((pkg) => { - // Check if we have a license for this package which includes a token - let license = this.licenses.find(lic => lic.package_id == pkg.packageid && lic.access_token != 0); - if (license) { - this.emit('debug', `Using token "${license.access_token}" from license for package ${pkg.packageid}`); - pkg.access_token = license.access_token; + if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { + // Only emit the event if we previously didn't have the appinfo, or if the changenumber changed + this.emit('appUpdate', app.appid, data); } - }); - } - this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, async (body) => { - // If we're using the PICS cache, then add the items in this response to it - if (this.options.enablePicsCache) { - let cache = this.picsCache; - cache.apps = cache.apps || {}; - cache.packages = cache.packages || {}; + cache.apps[app.appid] = data; - (body.apps || []).forEach((app) => { - let data = { - changenumber: app.change_number, - missingToken: !!app.missing_token, - appinfo: VDF.parse(app.buffer.toString('utf8')).appinfo - }; + app._parsedData = data; + }); - if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { - // Only emit the event if we previously didn't have the appinfo, or if the changenumber changed - this.emit('appUpdate', app.appid, data); - } + (body.packages || []).forEach((pkg) => { + let data = { + changenumber: pkg.change_number, + missingToken: !!pkg.missing_token, + packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null + }; - cache.apps[app.appid] = data; + if ((!cache.packages[pkg.packageid] && requestType == PICSRequestType.Changelist) || (cache.packages[pkg.packageid] && cache.packages[pkg.packageid].changenumber != data.changenumber)) { + this.emit('packageUpdate', pkg.packageid, data); + } - app._parsedData = data; - }); + cache.packages[pkg.packageid] = data; - (body.packages || []).forEach((pkg) => { - let data = { - changenumber: pkg.change_number, - missingToken: !!pkg.missing_token, - packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null - }; + pkg._parsedData = data; - if ((!cache.packages[pkg.packageid] && requestType == PICSRequestType.Changelist) || (cache.packages[pkg.packageid] && cache.packages[pkg.packageid].changenumber != data.changenumber)) { - this.emit('packageUpdate', pkg.packageid, data); - } + // Request info for all the apps in this package, if this request didn't originate from the license list + if (requestType != PICSRequestType.Licenses) { + let appids = (pkg.packageinfo || {}).appids || []; + this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { + }); + } + }); + } - cache.packages[pkg.packageid] = data; + (body.unknown_appids || []).forEach((appid) => { + response.unknownApps.push(appid); + let index = appids.indexOf(appid); + if (index != -1) { + appids.splice(index, 1); + } + }); - pkg._parsedData = data; + (body.unknown_packageids || []).forEach((packageid) => { + response.unknownPackages.push(packageid); + let index = packageids.indexOf(packageid); + if (index != -1) { + packageids.splice(index, 1); + } + }); - // Request info for all the apps in this package, if this request didn't originate from the license list - if (requestType != PICSRequestType.Licenses) { - let appids = (pkg.packageinfo || {}).appids || []; - this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { - }); - } - }); + (body.apps || []).forEach((app) => { + // _parsedData will be populated if we have the PICS cache enabled. + // If we don't, we need to parse the data here. + response.apps[app.appid] = app._parsedData || { + "changenumber": app.change_number, + "missingToken": !!app.missing_token, + "appinfo": VDF.parse(app.buffer.toString('utf8')).appinfo + }; + + let index = appids.indexOf(app.appid); + if (index != -1) { + appids.splice(index, 1); } + }); - (body.unknown_appids || []).forEach((appid) => { - response.unknownApps.push(appid); - let index = appids.indexOf(appid); - if (index != -1) { - appids.splice(index, 1); - } - }); + (body.packages || []).forEach((pkg) => { + response.packages[pkg.packageid] = pkg._parsedData || { + "changenumber": pkg.change_number, + "missingToken": !!pkg.missing_token, + "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null + }; - (body.unknown_packageids || []).forEach((packageid) => { - response.unknownPackages.push(packageid); - let index = packageids.indexOf(packageid); - if (index != -1) { - packageids.splice(index, 1); + let index = packageids.indexOf(pkg.packageid); + if (index != -1) { + packageids.splice(index, 1); + } + }); + + // appids and packageids contain the list of IDs that we're still waiting on data for + if (appids.length === 0 && packageids.length === 0) { + if (!inclTokens) { + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages } - }); + return combined; + } - (body.apps || []).forEach((app) => { - // _parsedData will be populated if we have the PICS cache enabled. - // If we don't, we need to parse the data here. - response.apps[app.appid] = app._parsedData || { - "changenumber": app.change_number, - "missingToken": !!app.missing_token, - "appinfo": VDF.parse(app.buffer.toString('utf8')).appinfo - }; + // We want tokens + let tokenlessAppids = []; + let tokenlessPackages = []; - let index = appids.indexOf(app.appid); - if (index != -1) { - appids.splice(index, 1); + for (let appid in response.apps) { + if (response.apps[appid].missingToken) { + tokenlessAppids.push(parseInt(appid, 10)); } - }); - - (body.packages || []).forEach((pkg) => { - response.packages[pkg.packageid] = pkg._parsedData || { - "changenumber": pkg.change_number, - "missingToken": !!pkg.missing_token, - "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null - }; + } - let index = packageids.indexOf(pkg.packageid); - if (index != -1) { - packageids.splice(index, 1); + for (let packageid in response.packages) { + if (response.packages[packageid].missingToken) { + tokenlessPackages.push(parseInt(packageid, 10)); } - }); + } - // appids and packageids contain the list of IDs that we're still waiting on data for - if (appids.length === 0 && packageids.length === 0) { - if (!inclTokens) { - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages - } - return resolve(combined); + if (tokenlessAppids.length == 0 && tokenlessPackages.length == 0) { + // No tokens needed + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages } + return combined; + } - // We want tokens - let tokenlessAppids = []; - let tokenlessPackages = []; + let { + appTokens, + packageTokens + } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); + let tokenApps = []; + let tokenPackages = []; - for (let appid in response.apps) { - if (response.apps[appid].missingToken) { - tokenlessAppids.push(parseInt(appid, 10)); - } - } + for (let appid in appTokens) { + tokenApps.push({appid: parseInt(appid, 10), access_token: appTokens[appid]}) + } - for (let packageid in response.packages) { - if (response.packages[packageid].missingToken) { - tokenlessPackages.push(parseInt(packageid, 10)); - } + for (let packageid in packageTokens) { + tokenPackages.push({ + packageid: parseInt(packageid, 10), + access_token: packageTokens[packageid] + }) + } + + // Now we have the tokens. Request the data. + let {apps, packages} = await this.getProductInfo(tokenApps, tokenPackages, false); + for (let appid in apps) { + response.apps[appid] = apps[appid]; + let index = response.unknownApps.indexOf(parseInt(appid, 10)); + if (index != -1) { + response.unknownApps.splice(index, 1); } + } - if (tokenlessAppids.length == 0 && tokenlessPackages.length == 0) { - // No tokens needed - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages - } - return resolve(combined); + for (let packageid in packages) { + response.packages[packageid] = packages[packageid]; + let index = response.unknownPackages.indexOf(parseInt(packageid, 10)); + if (index != -1) { + response.unknownPackages.splice(index, 1); } + } - try { - let { - appTokens, - packageTokens - } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); - let tokenApps = []; - let tokenPackages = []; - - for (let appid in appTokens) { - tokenApps.push({appid: parseInt(appid, 10), access_token: appTokens[appid]}) - } - - for (let packageid in packageTokens) { - tokenPackages.push({ - packageid: parseInt(packageid, 10), - access_token: packageTokens[packageid] - }) - } - - // Now we have the tokens. Request the data. - let {apps, packages} = await this.getProductInfo(tokenApps, tokenPackages, false); - for (let appid in apps) { - response.apps[appid] = apps[appid]; - let index = response.unknownApps.indexOf(parseInt(appid, 10)); - if (index != -1) { - response.unknownApps.splice(index, 1); - } - } - - for (let packageid in packages) { - response.packages[packageid] = packages[packageid]; - let index = response.unknownPackages.indexOf(parseInt(packageid, 10)); - if (index != -1) { - response.unknownPackages.splice(index, 1); - } - } - - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages - } - resolve(combined); - } catch (ex) { - return reject(ex); + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + return combined; + } + }); + } + + /** + * Get info about some apps and/or packages from Steam, but first split it into chunks + * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} + * @param {int[]|object[]} packages - Array of package IDs. May be empty. May also contain objects with keys {packageid, access_token} + * @param {boolean} [inclTokens=false] - If true, automatically retrieve access tokens if needed + * @param {function} [callback] + * @param {int} [requestType] - Don't touch + * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} + */ + getProductInfo(apps, packages, inclTokens, callback, requestType) { + // This one actually can take a while, so allow it to go as long as 120 minutes + return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { + try { + let response = { + apps: {}, + packages: {}, + unknownApps: [], + unknownPackages: [] + }; + // split apps + packages into chunks of 1000 + let chunkSize = 1000; + for (let i = 0; i < packages.length; i += chunkSize) { + let packagesChunk = packages.slice(i, i + chunkSize); + let result = await this._getProductInfo([], packagesChunk, inclTokens, callback, requestType); + response = { + apps: Object.assign(response.apps, result.apps), + packages: Object.assign(response.packages, result.packages), + unknownApps: response.unknownApps.concat(result.unknownApps), + unknownPackages: response.unknownPackages.concat(result.unknownPackages) } } - }); + for (let i = 0; i < apps.length; i += chunkSize) { + let appsChunk = apps.slice(i, i + chunkSize); + let result = await this._getProductInfo(appsChunk, [], inclTokens, callback, requestType); + response = { + apps: Object.assign(response.apps, result.apps), + packages: Object.assign(response.packages, result.packages), + unknownApps: response.unknownApps.concat(result.unknownApps), + unknownPackages: response.unknownPackages.concat(result.unknownPackages) + } + } + resolve(response); + } catch (ex) { + return reject(ex); + } }); } From 58b0b70467ff594cb48ac2ac308c1b94b9339ae5 Mon Sep 17 00:00:00 2001 From: Revadike Date: Thu, 2 Jun 2022 18:39:03 +0200 Subject: [PATCH 16/25] fixed deadlock issue --- components/apps.js | 456 +++++++++++++++++++++++---------------------- 1 file changed, 235 insertions(+), 221 deletions(-) diff --git a/components/apps.js b/components/apps.js index 97d57b3a..8209eec4 100644 --- a/components/apps.js +++ b/components/apps.js @@ -292,266 +292,278 @@ class SteamUserApps extends SteamUserAppAuth { * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} * @protected */ - async _getProductInfo(apps, packages, inclTokens, callback, requestType) { + _getProductInfo(apps, packages, inclTokens, callback, requestType) { // Adds support for the previous syntax if (typeof inclTokens !== 'boolean' && typeof inclTokens === 'function') { requestType = callback; callback = inclTokens; inclTokens = false; } - - requestType = requestType || PICSRequestType.User; - - // Steam can send us the full response in multiple responses, so we need to buffer them into one callback - let appids = []; - let packageids = []; - let response = { - apps: {}, - packages: {}, - unknownApps: [], - unknownPackages: [] - }; - let cached = { - apps: {}, - packages: {}, - notCachedApps: [], - notCachedPackages: [] - }; - - // Changelist requests always require fresh product info - if (this.options.enablePicsCache && requestType !== PICSRequestType.Changelist) { - cached = await this._getCachedProductInfo(apps, packages); - // If we already have all the info, we can return it immediately - if (cached.notCachedApps.length === 0 && cached.notCachedPackages.length === 0) { - response.apps = cached.apps; - response.packages = cached.packages; - return response; - } + // Add support for optional callback + if (!requestType && typeof callback === 'number') { + requestType = callback; + callback = undefined; } + + return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { + requestType = requestType || PICSRequestType.User; + + // Steam can send us the full response in multiple responses, so we need to buffer them into one callback + let appids = []; + let packageids = []; + let response = { + apps: {}, + packages: {}, + unknownApps: [], + unknownPackages: [] + }; + let cached = { + apps: {}, + packages: {}, + notCachedApps: [], + notCachedPackages: [] + }; - // Filter out any apps we already have cached - apps = apps.filter((app) => { - if (typeof app === 'object') { - return cached.notCachedApps.includes(app.appid); - } else { - return cached.notCachedApps.includes(app); - } - }).map((app) => { - if (typeof app === 'object') { - appids.push(app.appid); - return app; - } else { - appids.push(app); - return {appid: app}; - } - }); - - // Filter out any packages we already have cached - packages = packages.filter((pkg) => { - if (typeof pkg === 'object') { - return cached.notCachedPackages.includes(pkg.packageid); - } else { - return cached.notCachedPackages.includes(pkg); - } - }).map((pkg) => { - if (typeof pkg === 'object') { - packageids.push(pkg.packageid); - return pkg; - } else { - packageids.push(pkg); - return {packageid: pkg}; + // Changelist requests always require fresh product info + if (this.options.enablePicsCache && requestType !== PICSRequestType.Changelist) { + cached = await this._getCachedProductInfo(apps, packages); + // If we already have all the info, we can return it immediately + if (cached.notCachedApps.length === 0 && cached.notCachedPackages.length === 0) { + response.apps = cached.apps; + response.packages = cached.packages; + return resolve(response); + } } - }); - if (inclTokens) { - packages.filter(pkg => !pkg.access_token).forEach((pkg) => { - // Check if we have a license for this package which includes a token - let license = this.licenses.find(lic => lic.package_id == pkg.packageid && lic.access_token != 0); - if (license) { - this.emit('debug', `Using token "${license.access_token}" from license for package ${pkg.packageid}`); - pkg.access_token = license.access_token; + // Filter out any apps we already have cached + apps = apps.filter((app) => { + if (typeof app === 'object') { + return cached.notCachedApps.includes(app.appid); + } else { + return cached.notCachedApps.includes(app); + } + }).map((app) => { + if (typeof app === 'object') { + appids.push(app.appid); + return app; + } else { + appids.push(app); + return {appid: app}; } }); - } - - this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, async (body) => { - // If we're using the PICS cache, then add the items in this response to it - if (this.options.enablePicsCache) { - let cache = this.picsCache; - cache.apps = cache.apps || {}; - cache.packages = cache.packages || {}; - - (body.apps || []).forEach((app) => { - let data = { - changenumber: app.change_number, - missingToken: !!app.missing_token, - appinfo: VDF.parse(app.buffer.toString('utf8')).appinfo - }; - - if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { - // Only emit the event if we previously didn't have the appinfo, or if the changenumber changed - this.emit('appUpdate', app.appid, data); - } - - cache.apps[app.appid] = data; - - app._parsedData = data; - }); - - (body.packages || []).forEach((pkg) => { - let data = { - changenumber: pkg.change_number, - missingToken: !!pkg.missing_token, - packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null - }; - - if ((!cache.packages[pkg.packageid] && requestType == PICSRequestType.Changelist) || (cache.packages[pkg.packageid] && cache.packages[pkg.packageid].changenumber != data.changenumber)) { - this.emit('packageUpdate', pkg.packageid, data); - } - cache.packages[pkg.packageid] = data; - - pkg._parsedData = data; + // Filter out any packages we already have cached + packages = packages.filter((pkg) => { + if (typeof pkg === 'object') { + return cached.notCachedPackages.includes(pkg.packageid); + } else { + return cached.notCachedPackages.includes(pkg); + } + }).map((pkg) => { + if (typeof pkg === 'object') { + packageids.push(pkg.packageid); + return pkg; + } else { + packageids.push(pkg); + return {packageid: pkg}; + } + }); - // Request info for all the apps in this package, if this request didn't originate from the license list - if (requestType != PICSRequestType.Licenses) { - let appids = (pkg.packageinfo || {}).appids || []; - this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { - }); + if (inclTokens) { + packages.filter(pkg => !pkg.access_token).forEach((pkg) => { + // Check if we have a license for this package which includes a token + let license = this.licenses.find(lic => lic.package_id == pkg.packageid && lic.access_token != 0); + if (license) { + this.emit('debug', `Using token "${license.access_token}" from license for package ${pkg.packageid}`); + pkg.access_token = license.access_token; } }); } - (body.unknown_appids || []).forEach((appid) => { - response.unknownApps.push(appid); - let index = appids.indexOf(appid); - if (index != -1) { - appids.splice(index, 1); - } - }); + // Note: This callback can be called multiple times + this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, async (body) => { + // If we're using the PICS cache, then add the items in this response to it + if (this.options.enablePicsCache) { + let cache = this.picsCache; + cache.apps = cache.apps || {}; + cache.packages = cache.packages || {}; + + (body.apps || []).forEach((app) => { + let data = { + changenumber: app.change_number, + missingToken: !!app.missing_token, + appinfo: VDF.parse(app.buffer.toString('utf8')).appinfo + }; + + if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { + // Only emit the event if we previously didn't have the appinfo, or if the changenumber changed + this.emit('appUpdate', app.appid, data); + } + + cache.apps[app.appid] = data; + + app._parsedData = data; + }); - (body.unknown_packageids || []).forEach((packageid) => { - response.unknownPackages.push(packageid); - let index = packageids.indexOf(packageid); - if (index != -1) { - packageids.splice(index, 1); - } - }); + (body.packages || []).forEach((pkg) => { + let data = { + changenumber: pkg.change_number, + missingToken: !!pkg.missing_token, + packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null + }; - (body.apps || []).forEach((app) => { - // _parsedData will be populated if we have the PICS cache enabled. - // If we don't, we need to parse the data here. - response.apps[app.appid] = app._parsedData || { - "changenumber": app.change_number, - "missingToken": !!app.missing_token, - "appinfo": VDF.parse(app.buffer.toString('utf8')).appinfo - }; + if ((!cache.packages[pkg.packageid] && requestType == PICSRequestType.Changelist) || (cache.packages[pkg.packageid] && cache.packages[pkg.packageid].changenumber != data.changenumber)) { + this.emit('packageUpdate', pkg.packageid, data); + } - let index = appids.indexOf(app.appid); - if (index != -1) { - appids.splice(index, 1); - } - }); + cache.packages[pkg.packageid] = data; - (body.packages || []).forEach((pkg) => { - response.packages[pkg.packageid] = pkg._parsedData || { - "changenumber": pkg.change_number, - "missingToken": !!pkg.missing_token, - "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null - }; + pkg._parsedData = data; - let index = packageids.indexOf(pkg.packageid); - if (index != -1) { - packageids.splice(index, 1); + // Request info for all the apps in this package, if this request didn't originate from the license list + if (requestType != PICSRequestType.Licenses) { + let appids = (pkg.packageinfo || {}).appids || []; + this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { + }); + } + }); } - }); - // appids and packageids contain the list of IDs that we're still waiting on data for - if (appids.length === 0 && packageids.length === 0) { - if (!inclTokens) { - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages + (body.unknown_appids || []).forEach((appid) => { + response.unknownApps.push(appid); + let index = appids.indexOf(appid); + if (index != -1) { + appids.splice(index, 1); } - return combined; - } - - // We want tokens - let tokenlessAppids = []; - let tokenlessPackages = []; + }); - for (let appid in response.apps) { - if (response.apps[appid].missingToken) { - tokenlessAppids.push(parseInt(appid, 10)); + (body.unknown_packageids || []).forEach((packageid) => { + response.unknownPackages.push(packageid); + let index = packageids.indexOf(packageid); + if (index != -1) { + packageids.splice(index, 1); } - } + }); - for (let packageid in response.packages) { - if (response.packages[packageid].missingToken) { - tokenlessPackages.push(parseInt(packageid, 10)); + (body.apps || []).forEach((app) => { + // _parsedData will be populated if we have the PICS cache enabled. + // If we don't, we need to parse the data here. + response.apps[app.appid] = app._parsedData || { + "changenumber": app.change_number, + "missingToken": !!app.missing_token, + "appinfo": VDF.parse(app.buffer.toString('utf8')).appinfo + }; + + let index = appids.indexOf(app.appid); + if (index != -1) { + appids.splice(index, 1); } - } + }); + + (body.packages || []).forEach((pkg) => { + response.packages[pkg.packageid] = pkg._parsedData || { + "changenumber": pkg.change_number, + "missingToken": !!pkg.missing_token, + "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null + }; - if (tokenlessAppids.length == 0 && tokenlessPackages.length == 0) { - // No tokens needed - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages + let index = packageids.indexOf(pkg.packageid); + if (index != -1) { + packageids.splice(index, 1); } - return combined; - } + }); - let { - appTokens, - packageTokens - } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); - let tokenApps = []; - let tokenPackages = []; + // appids and packageids contain the list of IDs that we're still waiting on data for + if (appids.length === 0 && packageids.length === 0) { + if (!inclTokens) { + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + return resolve(combined); + } - for (let appid in appTokens) { - tokenApps.push({appid: parseInt(appid, 10), access_token: appTokens[appid]}) - } + // We want tokens + let tokenlessAppids = []; + let tokenlessPackages = []; - for (let packageid in packageTokens) { - tokenPackages.push({ - packageid: parseInt(packageid, 10), - access_token: packageTokens[packageid] - }) - } + for (let appid in response.apps) { + if (response.apps[appid].missingToken) { + tokenlessAppids.push(parseInt(appid, 10)); + } + } - // Now we have the tokens. Request the data. - let {apps, packages} = await this.getProductInfo(tokenApps, tokenPackages, false); - for (let appid in apps) { - response.apps[appid] = apps[appid]; - let index = response.unknownApps.indexOf(parseInt(appid, 10)); - if (index != -1) { - response.unknownApps.splice(index, 1); + for (let packageid in response.packages) { + if (response.packages[packageid].missingToken) { + tokenlessPackages.push(parseInt(packageid, 10)); + } } - } - for (let packageid in packages) { - response.packages[packageid] = packages[packageid]; - let index = response.unknownPackages.indexOf(parseInt(packageid, 10)); - if (index != -1) { - response.unknownPackages.splice(index, 1); + if (tokenlessAppids.length == 0 && tokenlessPackages.length == 0) { + // No tokens needed + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + return resolve(combined); } - } - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages + try { + let { + appTokens, + packageTokens + } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); + let tokenApps = []; + let tokenPackages = []; + + for (let appid in appTokens) { + tokenApps.push({appid: parseInt(appid, 10), access_token: appTokens[appid]}) + } + + for (let packageid in packageTokens) { + tokenPackages.push({ + packageid: parseInt(packageid, 10), + access_token: packageTokens[packageid] + }) + } + + // Now we have the tokens. Request the data. + let {apps, packages} = await this.getProductInfo(tokenApps, tokenPackages, false); + for (let appid in apps) { + response.apps[appid] = apps[appid]; + let index = response.unknownApps.indexOf(parseInt(appid, 10)); + if (index != -1) { + response.unknownApps.splice(index, 1); + } + } + + for (let packageid in packages) { + response.packages[packageid] = packages[packageid]; + let index = response.unknownPackages.indexOf(parseInt(packageid, 10)); + if (index != -1) { + response.unknownPackages.splice(index, 1); + } + } + + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + resolve(combined); + } catch (ex) { + return reject(ex); + } } - return combined; - } + }); }); } @@ -578,7 +590,8 @@ class SteamUserApps extends SteamUserAppAuth { let chunkSize = 1000; for (let i = 0; i < packages.length; i += chunkSize) { let packagesChunk = packages.slice(i, i + chunkSize); - let result = await this._getProductInfo([], packagesChunk, inclTokens, callback, requestType); + // Do not include callback in the request, it will be called multiple times + let result = await this._getProductInfo([], packagesChunk, inclTokens, requestType); response = { apps: Object.assign(response.apps, result.apps), packages: Object.assign(response.packages, result.packages), @@ -588,7 +601,8 @@ class SteamUserApps extends SteamUserAppAuth { } for (let i = 0; i < apps.length; i += chunkSize) { let appsChunk = apps.slice(i, i + chunkSize); - let result = await this._getProductInfo(appsChunk, [], inclTokens, callback, requestType); + // Do not include callback in the request, it will be called multiple times + let result = await this._getProductInfo(appsChunk, [], inclTokens, requestType); response = { apps: Object.assign(response.apps, result.apps), packages: Object.assign(response.packages, result.packages), From 2fd5ee06e395cc5efbf137b2a97b3b5e3c480147 Mon Sep 17 00:00:00 2001 From: Revadike Date: Fri, 3 Jun 2022 03:30:47 +0200 Subject: [PATCH 17/25] bugfixes + small changes --- components/apps.js | 84 +++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 27 deletions(-) diff --git a/components/apps.js b/components/apps.js index 8209eec4..336a40de 100644 --- a/components/apps.js +++ b/components/apps.js @@ -262,20 +262,22 @@ class SteamUserApps extends SteamUserAppAuth { let appid = appFiles[filename]; let packageid = packageFiles[filename]; - if (appid) { + if (appid !== undefined) { if (error || !contents) { response.notCachedApps.push(appid); } else { this.picsCache.apps[appid] = JSON.parse(contents); response.apps[appid] = this.picsCache.apps[appid]; } - } else if (packageid) { + } else if (packageid !== undefined) { // Remember, package ID can be 0 if (error || !contents) { response.notCachedPackages.push(packageid); } else { this.picsCache.packages[packageid] = JSON.parse(contents); response.packages[packageid] = this.picsCache.packages[packageid]; } + } else { + this.emit('debug', `Error retrieving origins of file ${filename}`); } } @@ -299,15 +301,16 @@ class SteamUserApps extends SteamUserAppAuth { callback = inclTokens; inclTokens = false; } + // Add support for optional callback - if (!requestType && typeof callback === 'number') { + if (!requestType && typeof callback !== 'function') { requestType = callback; callback = undefined; } - return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { - requestType = requestType || PICSRequestType.User; + requestType = requestType || PICSRequestType.User; + return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { // Steam can send us the full response in multiple responses, so we need to buffer them into one callback let appids = []; let packageids = []; @@ -324,8 +327,8 @@ class SteamUserApps extends SteamUserAppAuth { notCachedPackages: [] }; - // Changelist requests always require fresh product info - if (this.options.enablePicsCache && requestType !== PICSRequestType.Changelist) { + // Changelist requests always require fresh product info anyway + if (this.options.enablePicsCache && requestType != PICSRequestType.Changelist) { cached = await this._getCachedProductInfo(apps, packages); // If we already have all the info, we can return it immediately if (cached.notCachedApps.length === 0 && cached.notCachedPackages.length === 0) { @@ -334,15 +337,14 @@ class SteamUserApps extends SteamUserAppAuth { return resolve(response); } } + + if (requestType != PICSRequestType.Changelist) { + // Filter out any apps & packages we already have cached + apps = apps.filter((app) => cached.notCachedApps.includes(typeof app === 'object' ? app.appid : app)); + packages = packages.filter((pkg) => cached.notCachedPackages.includes(typeof pkg === 'object' ? pkg.packageid : pkg)); + } - // Filter out any apps we already have cached - apps = apps.filter((app) => { - if (typeof app === 'object') { - return cached.notCachedApps.includes(app.appid); - } else { - return cached.notCachedApps.includes(app); - } - }).map((app) => { + apps = apps.map((app) => { if (typeof app === 'object') { appids.push(app.appid); return app; @@ -352,14 +354,7 @@ class SteamUserApps extends SteamUserAppAuth { } }); - // Filter out any packages we already have cached - packages = packages.filter((pkg) => { - if (typeof pkg === 'object') { - return cached.notCachedPackages.includes(pkg.packageid); - } else { - return cached.notCachedPackages.includes(pkg); - } - }).map((pkg) => { + packages = packages.map((pkg) => { if (typeof pkg === 'object') { packageids.push(pkg.packageid); return pkg; @@ -577,6 +572,21 @@ class SteamUserApps extends SteamUserAppAuth { * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} */ getProductInfo(apps, packages, inclTokens, callback, requestType) { + // Adds support for the previous syntax + if (typeof inclTokens !== 'boolean' && typeof inclTokens === 'function') { + requestType = callback; + callback = inclTokens; + inclTokens = false; + } + + // Add support for optional callback + if (!requestType && typeof callback !== 'function') { + requestType = callback; + callback = undefined; + } + + requestType = requestType || PICSRequestType.User; + // This one actually can take a while, so allow it to go as long as 120 minutes return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { try { @@ -586,12 +596,12 @@ class SteamUserApps extends SteamUserAppAuth { unknownApps: [], unknownPackages: [] }; - // split apps + packages into chunks of 1000 + // Split apps + packages into chunks of 1000 let chunkSize = 1000; for (let i = 0; i < packages.length; i += chunkSize) { let packagesChunk = packages.slice(i, i + chunkSize); // Do not include callback in the request, it will be called multiple times - let result = await this._getProductInfo([], packagesChunk, inclTokens, requestType); + let result = await this._getProductInfo([], packagesChunk, inclTokens, undefined, requestType); response = { apps: Object.assign(response.apps, result.apps), packages: Object.assign(response.packages, result.packages), @@ -602,7 +612,7 @@ class SteamUserApps extends SteamUserAppAuth { for (let i = 0; i < apps.length; i += chunkSize) { let appsChunk = apps.slice(i, i + chunkSize); // Do not include callback in the request, it will be called multiple times - let result = await this._getProductInfo(appsChunk, [], inclTokens, requestType); + let result = await this._getProductInfo(appsChunk, [], inclTokens, undefined, requestType); response = { apps: Object.assign(response.apps, result.apps), packages: Object.assign(response.packages, result.packages), @@ -688,6 +698,12 @@ class SteamUserApps extends SteamUserAppAuth { return; } + // First wait for ownership cache to be loaded, so we can calculate ourApps & ourPackages correctly. + if (!this.picsCache.ownershipModified) { + this._resetChangelistUpdateTimer(); + return; + } + let result = null; try { result = await this.getProductChanges(this.picsCache.changenumber); @@ -756,8 +772,22 @@ class SteamUserApps extends SteamUserAppAuth { } // Add a no-op catch in case there's some kind of error - this.getProductInfo(ourApps, ourPackages, false, null, PICSRequestType.Changelist).catch(() => { + let {packages} = await this.getProductInfo(ourApps, ourPackages, false, null, PICSRequestType.Changelist).catch(() => { }); + + // Request info for all the apps in these packages + let appids = []; + + for (let pkgid in packages) { + ((packages[pkgid].packageinfo || {}).appids || []).filter(appid => !appids.includes(appid)).forEach(appid => appids.push(appid)); + } + + try { + // Request type is changelist, because we need to refresh their pics cache + await this.getProductInfo(appids, [], true, undefined, PICSRequestType.Changelist); + } catch (ex) { + this.emit('debug', `Error retrieving product info of changed apps: ${ex.message}`); + } } /** From 3249f29649847c7e168658d4fcd35b5cf83a4e96 Mon Sep 17 00:00:00 2001 From: Revadike Date: Fri, 17 Jun 2022 06:00:43 +0200 Subject: [PATCH 18/25] Prevent any stale data with meta_data_only + sha check --- components/apps.js | 350 +++++++++++++++++++++++---------------------- 1 file changed, 176 insertions(+), 174 deletions(-) diff --git a/components/apps.js b/components/apps.js index 336a40de..44e5c953 100644 --- a/components/apps.js +++ b/components/apps.js @@ -326,43 +326,51 @@ class SteamUserApps extends SteamUserAppAuth { notCachedApps: [], notCachedPackages: [] }; + let shaList = { + apps: {}, + packages: {} + }; // Changelist requests always require fresh product info anyway if (this.options.enablePicsCache && requestType != PICSRequestType.Changelist) { cached = await this._getCachedProductInfo(apps, packages); - // If we already have all the info, we can return it immediately - if (cached.notCachedApps.length === 0 && cached.notCachedPackages.length === 0) { - response.apps = cached.apps; - response.packages = cached.packages; - return resolve(response); - } - } - - if (requestType != PICSRequestType.Changelist) { - // Filter out any apps & packages we already have cached - apps = apps.filter((app) => cached.notCachedApps.includes(typeof app === 'object' ? app.appid : app)); - packages = packages.filter((pkg) => cached.notCachedPackages.includes(typeof pkg === 'object' ? pkg.packageid : pkg)); } - apps = apps.map((app) => { + let _apps = []; + for (let app of apps) { + let appid = parseInt(typeof app === 'object' ? app.appid : app, 10); + // Ensure uniqueness to prevent nasty bugs + if (appids.includes(appid)) { + continue; + } if (typeof app === 'object') { + app.appid = appid; appids.push(app.appid); - return app; + _apps.push(app); } else { - appids.push(app); - return {appid: app}; + appids.push(appid); + _apps.push({appid}); + } + } + apps = _apps; + + let _packages = []; + for (let pkg of packages) { + let packageid = parseInt(typeof pkg === 'object' ? pkg.packageid : pkg, 10); + // Ensure uniqueness to prevent nasty bugs + if (packageids.includes(packageid)) { + continue; } - }); - - packages = packages.map((pkg) => { if (typeof pkg === 'object') { + pkg.packageid = packageid; packageids.push(pkg.packageid); - return pkg; + _packages.push(pkg); } else { - packageids.push(pkg); - return {packageid: pkg}; + packageids.push(packageid); + _packages.push({packageid}); } - }); + } + packages = _packages; if (inclTokens) { packages.filter(pkg => !pkg.access_token).forEach((pkg) => { @@ -376,129 +384,78 @@ class SteamUserApps extends SteamUserAppAuth { } // Note: This callback can be called multiple times - this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, async (body) => { - // If we're using the PICS cache, then add the items in this response to it - if (this.options.enablePicsCache) { - let cache = this.picsCache; - cache.apps = cache.apps || {}; - cache.packages = cache.packages || {}; - - (body.apps || []).forEach((app) => { - let data = { - changenumber: app.change_number, - missingToken: !!app.missing_token, - appinfo: VDF.parse(app.buffer.toString('utf8')).appinfo - }; - - if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { - // Only emit the event if we previously didn't have the appinfo, or if the changenumber changed - this.emit('appUpdate', app.appid, data); - } - - cache.apps[app.appid] = data; - - app._parsedData = data; - }); - - (body.packages || []).forEach((pkg) => { - let data = { - changenumber: pkg.change_number, - missingToken: !!pkg.missing_token, - packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null - }; - - if ((!cache.packages[pkg.packageid] && requestType == PICSRequestType.Changelist) || (cache.packages[pkg.packageid] && cache.packages[pkg.packageid].changenumber != data.changenumber)) { - this.emit('packageUpdate', pkg.packageid, data); - } - - cache.packages[pkg.packageid] = data; - - pkg._parsedData = data; - - // Request info for all the apps in this package, if this request didn't originate from the license list - if (requestType != PICSRequestType.Licenses) { - let appids = (pkg.packageinfo || {}).appids || []; - this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { - }); - } - }); - } - - (body.unknown_appids || []).forEach((appid) => { - response.unknownApps.push(appid); - let index = appids.indexOf(appid); - if (index != -1) { - appids.splice(index, 1); - } - }); - - (body.unknown_packageids || []).forEach((packageid) => { - response.unknownPackages.push(packageid); - let index = packageids.indexOf(packageid); - if (index != -1) { - packageids.splice(index, 1); - } - }); - + this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages, meta_data_only: true}, async (body) => { (body.apps || []).forEach((app) => { - // _parsedData will be populated if we have the PICS cache enabled. - // If we don't, we need to parse the data here. - response.apps[app.appid] = app._parsedData || { - "changenumber": app.change_number, - "missingToken": !!app.missing_token, - "appinfo": VDF.parse(app.buffer.toString('utf8')).appinfo - }; - - let index = appids.indexOf(app.appid); - if (index != -1) { - appids.splice(index, 1); - } + shaList.apps[app.appid] = app.sha.toString('hex'); }); (body.packages || []).forEach((pkg) => { - response.packages[pkg.packageid] = pkg._parsedData || { - "changenumber": pkg.change_number, - "missingToken": !!pkg.missing_token, - "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null - }; - - let index = packageids.indexOf(pkg.packageid); - if (index != -1) { - packageids.splice(index, 1); - } + shaList.packages[pkg.packageid] = pkg.sha.toString('hex'); }); - // appids and packageids contain the list of IDs that we're still waiting on data for - if (appids.length === 0 && packageids.length === 0) { - if (!inclTokens) { - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages - } - return resolve(combined); - } + response.unknownApps = response.unknownApps.concat(body.unknown_appids || []); + response.unknownPackages = response.unknownPackages.concat(body.unknown_packageids || []); + + let appTotal = Object.keys(shaList.apps).length + response.unknownApps.length; + let packageTotal = Object.keys(shaList.packages).length + response.unknownPackages.length; + + // If our collected totals match the total we requested + if (appTotal === appids.length && packageTotal === packageids.length) { + // Filter out any apps & packages we already have cached and do not need to be refreshed + apps = apps.filter((app) => !response.unknownApps.includes(app.appid) && (cached.apps[app.appid] || {}).sha !== shaList.apps[app.appid]); + packages = packages.filter((pkg) => !response.unknownPackages.includes(pkg.packageid) && (cached.packages[pkg.packageid] || {}).sha !== shaList.packages[pkg.packageid]); + // console.log( + // "requestType:", requestType, + // "app request:", appids.length, + // "pkg request:", packageids.length, + // "app unknown:", response.unknownApps.length, + // "pkg unknown:", response.unknownPackages.length, + // "app cache:", Object.keys(cached.apps).length, + // "pkg cache:", Object.keys(cached.packages).length, + // "app to refresh:", apps.length, + // "pkg to refresh:", packages.length, + // ); // We want tokens - let tokenlessAppids = []; - let tokenlessPackages = []; - - for (let appid in response.apps) { - if (response.apps[appid].missingToken) { - tokenlessAppids.push(parseInt(appid, 10)); - } - } - - for (let packageid in response.packages) { - if (response.packages[packageid].missingToken) { - tokenlessPackages.push(parseInt(packageid, 10)); + if (inclTokens) { + let _appids = apps.map(app => app.appid); + let _packageids = packages.map(pkg => pkg.packageid); + let tokenlessAppids = body.apps.filter(app => _appids.includes(app.appid) && !!app.missing_token).map(app => app.appid); + let tokenlessPackages = body.packages.filter(pkg => _packageids.includes(pkg.packageid) && !!pkg.missing_token).map(pkg => pkg.packageid); + if (tokenlessAppids.length > 0 || tokenlessPackages.length > 0) { + try { + let { + appTokens, + packageTokens + } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); + let tokenApps = []; + let tokenPackages = []; + + for (let appid in appTokens) { + tokenApps.push({appid: parseInt(appid, 10), access_token: appTokens[appid]}); + } + + for (let packageid in packageTokens) { + tokenPackages.push({ + packageid: parseInt(packageid, 10), + access_token: packageTokens[packageid] + }); + } + + // Replace products to request with included tokens + apps = apps.filter(app => !tokenlessAppids.includes(app.appid)).concat(tokenApps); + packages = packages.filter(pkg => !tokenlessPackages.includes(pkg.packageid)).concat(tokenPackages); + } catch (ex) { + return reject(ex); + } } } + + appids = apps.map(app => app.appid); + packageids = packages.map(pkg => pkg.packageid); - if (tokenlessAppids.length == 0 && tokenlessPackages.length == 0) { - // No tokens needed + // If we have nothing to refresh / no stale data + if (apps.length === 0 && packages.length === 0) { this._saveProductInfo(response); let combined = { apps: Object.assign(cached.apps, response.apps), @@ -509,54 +466,99 @@ class SteamUserApps extends SteamUserAppAuth { return resolve(combined); } - try { - let { - appTokens, - packageTokens - } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); - let tokenApps = []; - let tokenPackages = []; + // Note: This callback can be called multiple times + this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, async (body) => { + // If we're using the PICS cache, then add the items in this response to it + if (this.options.enablePicsCache) { + let cache = this.picsCache; + cache.apps = cache.apps || {}; + cache.packages = cache.packages || {}; + + (body.apps || []).forEach((app) => { + let data = { + sha: app.sha.toString('hex'), + changenumber: app.change_number, + missingToken: !!app.missing_token, + appinfo: app.buffer ? VDF.parse(app.buffer.toString('utf8')).appinfo : null + }; + + if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { + // Only emit the event if we previously didn't have the appinfo, or if the changenumber changed + this.emit('appUpdate', app.appid, data); + } + + cache.apps[app.appid] = data; + + app._parsedData = data; + }); + + (body.packages || []).forEach((pkg) => { + let data = { + sha: pkg.sha.toString('hex'), + changenumber: pkg.change_number, + missingToken: !!pkg.missing_token, + packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null + }; - for (let appid in appTokens) { - tokenApps.push({appid: parseInt(appid, 10), access_token: appTokens[appid]}) - } + if ((!cache.packages[pkg.packageid] && requestType == PICSRequestType.Changelist) || (cache.packages[pkg.packageid] && cache.packages[pkg.packageid].changenumber != data.changenumber)) { + this.emit('packageUpdate', pkg.packageid, data); + } - for (let packageid in packageTokens) { - tokenPackages.push({ - packageid: parseInt(packageid, 10), - access_token: packageTokens[packageid] - }) + cache.packages[pkg.packageid] = data; + + pkg._parsedData = data; + + // Request info for all the apps in this package, if this request didn't originate from the license list + if (requestType != PICSRequestType.Licenses) { + let appids = (pkg.packageinfo || {}).appids || []; + this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { + }); + } + }); } - // Now we have the tokens. Request the data. - let {apps, packages} = await this.getProductInfo(tokenApps, tokenPackages, false); - for (let appid in apps) { - response.apps[appid] = apps[appid]; - let index = response.unknownApps.indexOf(parseInt(appid, 10)); + (body.apps || []).forEach((app) => { + // _parsedData will be populated if we have the PICS cache enabled. + // If we don't, we need to parse the data here. + response.apps[app.appid] = app._parsedData || { + "sha": app.sha.toString('hex'), + "changenumber": app.change_number, + "missingToken": !!app.missing_token, + "appinfo": VDF.parse(app.buffer.toString('utf8')).appinfo + }; + + let index = appids.indexOf(app.appid); if (index != -1) { - response.unknownApps.splice(index, 1); + appids.splice(index, 1); } - } + }); - for (let packageid in packages) { - response.packages[packageid] = packages[packageid]; - let index = response.unknownPackages.indexOf(parseInt(packageid, 10)); + (body.packages || []).forEach((pkg) => { + response.packages[pkg.packageid] = pkg._parsedData || { + "sha": pkg.sha.toString('hex'), + "changenumber": pkg.change_number, + "missingToken": !!pkg.missing_token, + "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null + }; + + let index = packageids.indexOf(pkg.packageid); if (index != -1) { - response.unknownPackages.splice(index, 1); + packageids.splice(index, 1); } + }); + + // appids and packageids contain the list of IDs that we're still waiting on data for + if (appids.length === 0 && packageids.length === 0) { + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + return resolve(combined); } - - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages - } - resolve(combined); - } catch (ex) { - return reject(ex); - } + }); } }); }); @@ -597,7 +599,7 @@ class SteamUserApps extends SteamUserAppAuth { unknownPackages: [] }; // Split apps + packages into chunks of 1000 - let chunkSize = 1000; + let chunkSize = 2000; for (let i = 0; i < packages.length; i += chunkSize) { let packagesChunk = packages.slice(i, i + chunkSize); // Do not include callback in the request, it will be called multiple times From ff774422ef3b27ee0bfee1112b4a0bec77e9c07b Mon Sep 17 00:00:00 2001 From: Revadike Date: Sun, 19 Jun 2022 17:49:34 +0200 Subject: [PATCH 19/25] improve readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 056ab134..1891c554 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ user.storage.on('read', function(filename, callback) { }); ``` -In this manner, you can save data to a database, a cloud service, or anything else you choose. +In this manner, you can save data to a database, a cloud service, or anything else you choose. If both [`savePicsCache`](#savePicsCache) and [`picsCacheAll`](#picsCacheAll) are enabled, it is possible to maintain your own Steam database this way. ### autoRelogin @@ -181,13 +181,13 @@ Defaults to `false`. ### savePicsCache -If `enablePicsCache` is enabled, save product info to disk. (TODO: Improve description) +If `enablePicsCache` is enabled, saves all product info from [the PICS cache](#picscache) to disk (in [`dataDirectory`](#dataDirectory)) or to your [Custom Storage Engine](#custom-storage-engine). This will significantly speed up the [`appOwnershipCached`](#appOwnershipCache) event from firing and reduce the amount product info requests to Steam. It will only save product info if it's not missing its access token. -Added in 4.24.0 +Added in {{TODO}} Defaults to `false`. -**Warning:** This will significantly increase the storage space that is needed! +**Warning:** Mind that this will significantly increase the storage usage and space! ### picsCacheAll @@ -843,6 +843,7 @@ Requests a list of game servers from the master server. - `callback` - Called when requested data is available - `err` - An `Error` object on failure, or `null` on success - `apps` - An object whose keys are AppIDs and whose values are objects + - `sha` - The SHA hash of the app info in hex format (useful to compare to old data) - `changenumber` - The changenumber of the latest changelist in which this app changed - `missingToken` - `true` if you need to provide an access token to get more details about this app - `appinfo` - An object whose structure is identical to the output of `app_info_print` in the [Steam console](steam://nav/console) From fd813a76cba42d71bf54c36a5277e46a4f8a1aad Mon Sep 17 00:00:00 2001 From: Revadike Date: Sun, 19 Jun 2022 18:24:57 +0200 Subject: [PATCH 20/25] fixed anonymous usage --- components/apps.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/components/apps.js b/components/apps.js index 44e5c953..5811e334 100644 --- a/components/apps.js +++ b/components/apps.js @@ -375,7 +375,7 @@ class SteamUserApps extends SteamUserAppAuth { if (inclTokens) { packages.filter(pkg => !pkg.access_token).forEach((pkg) => { // Check if we have a license for this package which includes a token - let license = this.licenses.find(lic => lic.package_id == pkg.packageid && lic.access_token != 0); + let license = (this.licenses || []).find(lic => lic.package_id == pkg.packageid && lic.access_token != 0); if (license) { this.emit('debug', `Using token "${license.access_token}" from license for package ${pkg.packageid}`); pkg.access_token = license.access_token; @@ -833,7 +833,13 @@ class SteamUserApps extends SteamUserAppAuth { } // Get all owned lisense id's - let packageids = this.licenses.map(license => license.package_id); + let packageids; + // We're anonymous + if (this.steamID.type == SteamID.Type.ANON_USER) { + packageids = [17906]; + } else { + packageids = this.licenses.map(license => license.package_id); + } let result; try { From b8c3791725b23cbd59841f2ef827088943eb3cc0 Mon Sep 17 00:00:00 2001 From: Revadike Date: Tue, 5 Jul 2022 00:19:35 +0200 Subject: [PATCH 21/25] small changes --- components/apps.js | 57 +++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/components/apps.js b/components/apps.js index c75a84ba..7ce93bbb 100644 --- a/components/apps.js +++ b/components/apps.js @@ -293,30 +293,15 @@ class SteamUserApps extends SteamUserAppAuth { * Get info about some apps and/or packages from Steam. * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} * @param {int[]|object[]} packages - Array of package IDs. May be empty. May also contain objects with keys {packageid, access_token} - * @param {boolean} [inclTokens=false] - If true, automatically retrieve access tokens if needed - * @param {function} [callback] + * @param {boolean} inclTokens - If true, automatically retrieve access tokens if needed + * @param {function} callback * @param {int} [requestType] - Don't touch * @returns {Promise<{apps: Object, packages: Object, unknownApps: number[], unknownPackages: number[]}>} * @protected */ - _getProductInfo(apps, packages, inclTokens, callback, requestType) { - // Adds support for the previous syntax - if (typeof inclTokens !== 'boolean' && typeof inclTokens === 'function') { - requestType = callback; - callback = inclTokens; - inclTokens = false; - } - - // Add support for optional callback - if (!requestType && typeof callback !== 'function') { - requestType = callback; - callback = undefined; - } - - requestType = requestType || PICSRequestType.User; - + _getProductInfo(apps, packages, inclTokens, callback, requestType = PICSRequestType.User) { + // Steam can send us the full response in multiple responses, so we need to buffer them into one callback return StdLib.Promises.timeoutCallbackPromise(7200000, ['apps', 'packages', 'unknownApps', 'unknownPackages'], callback, async (resolve, reject) => { - // Steam can send us the full response in multiple responses, so we need to buffer them into one callback let appids = []; let packageids = []; let response = { @@ -336,11 +321,7 @@ class SteamUserApps extends SteamUserAppAuth { packages: {} }; - // Changelist requests always require fresh product info anyway - if (this.options.enablePicsCache && requestType != PICSRequestType.Changelist) { - cached = await this._getCachedProductInfo(apps, packages); - } - + // Preprocess input: apps and packages let _apps = []; for (let app of apps) { let appid = parseInt(typeof app === 'object' ? app.appid : app, 10); @@ -351,11 +332,11 @@ class SteamUserApps extends SteamUserAppAuth { if (typeof app === 'object') { app.appid = appid; appids.push(app.appid); - _apps.push(app); } else { appids.push(appid); - _apps.push({appid}); + app = {appid}; } + _apps.push(app); } apps = _apps; @@ -369,23 +350,31 @@ class SteamUserApps extends SteamUserAppAuth { if (typeof pkg === 'object') { pkg.packageid = packageid; packageids.push(pkg.packageid); - _packages.push(pkg); } else { packageids.push(packageid); - _packages.push({packageid}); + pkg = {packageid}; } - } - packages = _packages; - - if (inclTokens) { - packages.filter(pkg => !pkg.access_token).forEach((pkg) => { + if (inclTokens && !pkg.access_token) { // Check if we have a license for this package which includes a token let license = (this.licenses || []).find(lic => lic.package_id == pkg.packageid && lic.access_token != 0); if (license) { this.emit('debug', `Using token "${license.access_token}" from license for package ${pkg.packageid}`); pkg.access_token = license.access_token; } - }); + } + _packages.push(pkg); + } + packages = _packages; + + // If we have no apps or packages, we're done + if (appids.length === 0 && packageids.length === 0) { + resolve(response); + return; + } + + // Changelist requests always require fresh product info anyway + if (this.options.enablePicsCache && requestType != PICSRequestType.Changelist) { + cached = await this._getCachedProductInfo(apps, packages); } // Note: This callback can be called multiple times From 2b0fa8ad78ee9fd9a56e64078f700eff71c87661 Mon Sep 17 00:00:00 2001 From: Revadike Date: Thu, 7 Jul 2022 21:31:24 +0200 Subject: [PATCH 22/25] handle when `sha` is `null` --- components/apps.js | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/components/apps.js b/components/apps.js index 7ce93bbb..0c851c7e 100644 --- a/components/apps.js +++ b/components/apps.js @@ -380,11 +380,11 @@ class SteamUserApps extends SteamUserAppAuth { // Note: This callback can be called multiple times this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages, meta_data_only: true}, async (body) => { (body.apps || []).forEach((app) => { - shaList.apps[app.appid] = app.sha.toString('hex'); + shaList.apps[app.appid] = app.sha ? app.sha.toString('hex') : null; }); (body.packages || []).forEach((pkg) => { - shaList.packages[pkg.packageid] = pkg.sha.toString('hex'); + shaList.packages[pkg.packageid] = pkg.sha ? pkg.sha.toString('hex') : null; }); response.unknownApps = response.unknownApps.concat(body.unknown_appids || []); @@ -422,29 +422,32 @@ class SteamUserApps extends SteamUserAppAuth { appTokens, packageTokens } = await this.getProductAccessToken(tokenlessAppids, tokenlessPackages); - let tokenApps = []; - let tokenPackages = []; + let tokenApps = {}; + let tokenPackages = {}; for (let appid in appTokens) { - tokenApps.push({appid: parseInt(appid, 10), access_token: appTokens[appid]}); + tokenApps[appid] = { + appid: parseInt(appid, 10), + access_token: appTokens[appid] + }; } for (let packageid in packageTokens) { - tokenPackages.push({ + tokenPackages[packageid] = { packageid: parseInt(packageid, 10), access_token: packageTokens[packageid] - }); + }; } // Replace products to request with included tokens - apps = apps.filter(app => !tokenlessAppids.includes(app.appid)).concat(tokenApps); - packages = packages.filter(pkg => !tokenlessPackages.includes(pkg.packageid)).concat(tokenPackages); + apps = apps.filter(app => !appTokens[app.appid]).concat(Object.values(tokenApps)); + packages = packages.filter(pkg => !tokenPackages[pkg.packageid]).concat(Object.values(tokenPackages)); } catch (ex) { return reject(ex); } } } - + appids = apps.map(app => app.appid); packageids = packages.map(pkg => pkg.packageid); @@ -470,7 +473,7 @@ class SteamUserApps extends SteamUserAppAuth { (body.apps || []).forEach((app) => { let data = { - sha: app.sha.toString('hex'), + sha: app.sha ? app.sha.toString('hex') : null, changenumber: app.change_number, missingToken: !!app.missing_token, appinfo: app.buffer ? VDF.parse(app.buffer.toString('utf8')).appinfo : null @@ -488,7 +491,7 @@ class SteamUserApps extends SteamUserAppAuth { (body.packages || []).forEach((pkg) => { let data = { - sha: pkg.sha.toString('hex'), + sha: pkg.sha ? pkg.sha.toString('hex') : null, changenumber: pkg.change_number, missingToken: !!pkg.missing_token, packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null @@ -515,7 +518,7 @@ class SteamUserApps extends SteamUserAppAuth { // _parsedData will be populated if we have the PICS cache enabled. // If we don't, we need to parse the data here. response.apps[app.appid] = app._parsedData || { - "sha": app.sha.toString('hex'), + "sha": app.sha ? app.sha.toString('hex') : null, "changenumber": app.change_number, "missingToken": !!app.missing_token, "appinfo": VDF.parse(app.buffer.toString('utf8')).appinfo @@ -529,7 +532,7 @@ class SteamUserApps extends SteamUserAppAuth { (body.packages || []).forEach((pkg) => { response.packages[pkg.packageid] = pkg._parsedData || { - "sha": pkg.sha.toString('hex'), + "sha": pkg.sha ? pkg.sha.toString('hex') : null, "changenumber": pkg.change_number, "missingToken": !!pkg.missing_token, "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null From 037feacf30b450fdca5b8327207123db73350aac Mon Sep 17 00:00:00 2001 From: Revadike Date: Tue, 19 Jul 2022 16:47:21 +0200 Subject: [PATCH 23/25] minor bugfix --- components/apps.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/components/apps.js b/components/apps.js index 0c851c7e..1f0e2443 100644 --- a/components/apps.js +++ b/components/apps.js @@ -884,14 +884,15 @@ class SteamUserApps extends SteamUserAppAuth { let appids = {}; ownedPackages.forEach((pkg) => { - if (!this.picsCache.packages[pkg]) { - this._warn(`Failed to get owned apps for package ${pkg}`); + let pkgid = pkg; + if (!this.picsCache.packages[pkgid]) { + this._warn(`Failed to get owned apps for package ${pkgid}: not in cache`); return; } - pkg = this.picsCache.packages[pkg]; + pkg = this.picsCache.packages[pkgid]; if (!pkg.packageinfo) { - this._warn(`Failed to get owned apps for package ${pkg}`); + this._warn(`Failed to get owned apps for package ${pkgid}: no package info`); return; } @@ -932,14 +933,15 @@ class SteamUserApps extends SteamUserAppAuth { let depotids = {}; ownedPackages.forEach((pkg) => { - if (!this.picsCache.packages[pkg]) { - this._warn(`Failed to get owned depots for package ${pkg}`); + let pkgid = pkg; + if (!this.picsCache.packages[pkgid]) { + this._warn(`Failed to get owned depots for package ${pkgid}: not in cache`); return; } - pkg = this.picsCache.packages[pkg]; + pkg = this.picsCache.packages[pkgid]; if (!pkg.packageinfo) { - this._warn(`Failed to get owned depots for package ${pkg}`); + this._warn(`Failed to get owned depots for package ${pkgid}: no package info`); return; } From f3a725e20945d5374cd037dbb44a9598862d3325 Mon Sep 17 00:00:00 2001 From: Revadike Date: Sat, 23 Jul 2022 01:48:44 +0200 Subject: [PATCH 24/25] comments + refactor + handle enablePicsCache disabled --- .vscode/settings.json | 3 + components/apps.js | 226 +++++++++++++++++++++--------------------- 2 files changed, 117 insertions(+), 112 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..47488040 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "eslint.enable": false +} \ No newline at end of file diff --git a/components/apps.js b/components/apps.js index 1f0e2443..57aa1178 100644 --- a/components/apps.js +++ b/components/apps.js @@ -174,6 +174,7 @@ class SteamUserApps extends SteamUserAppAuth { let toSave = {}; for (let appid in apps) { + // We want to avoid saving apps that are missing a token // Public only apps are weird... if (apps[appid].missingToken && !apps[appid].appinfo.public_only) { continue; @@ -197,8 +198,8 @@ class SteamUserApps extends SteamUserAppAuth { /** * Get cached product info. - * @param {int[]|object[]} apps - Array of AppIDs. May be empty. May also contain objects with keys {appid, access_token} - * @param {int[]|object[]} packages - Array of package IDs. May be empty. May also contain objects with keys {packageid, access_token} + * @param {object[]} apps - Array of apps {appid, access_token} + * @param {object[]} packages - Array of packages {packageid, access_token} * @returns {Promise<{apps: Object, packages: Object, notCachedApps: number[], notCachedPackages: number[]}>} * @protected */ @@ -216,16 +217,14 @@ class SteamUserApps extends SteamUserAppAuth { } // From this point, we can assume pics cache is up to date (via changelist updates). - let appids = apps.map(app => typeof app === 'object' ? app.appid : app); - let packageids = packages.map(pkg => typeof pkg === 'object' ? pkg.packageid : pkg); - for (let appid of appids) { + for (let {appid} of apps) { if (this.picsCache.apps[appid]) { response.apps[appid] = this.picsCache.apps[appid]; } else { response.notCachedApps.push(appid); } } - for (let packageid of packageids) { + for (let {packageid} of packages) { if (this.picsCache.packages[packageid]) { response.packages[packageid] = this.picsCache.packages[packageid]; } else { @@ -244,10 +243,11 @@ class SteamUserApps extends SteamUserAppAuth { } // Otherwise, we try loading the missing apps & packages from disk. - appids = response.notCachedApps; - packageids = response.notCachedPackages; + let appids = response.notCachedApps; + let packageids = response.notCachedPackages; response.notCachedApps = []; response.notCachedPackages = []; + let appFiles = {}; let packageFiles = {}; for (let appid of appids) { @@ -259,8 +259,8 @@ class SteamUserApps extends SteamUserAppAuth { packageFiles[filename] = packageid; } let files = await this._readFiles(Object.keys(appFiles).concat(Object.keys(packageFiles))); - for (let file of files) { - let {filename, error, contents} = file; + + for (let {filename, error, contents} of files) { if (Buffer.isBuffer(contents)) { contents = contents.toString('utf8'); } @@ -271,6 +271,7 @@ class SteamUserApps extends SteamUserAppAuth { if (error || !contents) { response.notCachedApps.push(appid); } else { + // Save to memory cache this.picsCache.apps[appid] = JSON.parse(contents); response.apps[appid] = this.picsCache.apps[appid]; } @@ -372,11 +373,104 @@ class SteamUserApps extends SteamUserAppAuth { return; } - // Changelist requests always require fresh product info anyway - if (this.options.enablePicsCache && requestType != PICSRequestType.Changelist) { - cached = await this._getCachedProductInfo(apps, packages); + // Function to handle response of ClientPICSProductInfoRequest (may be called multiple times) + let onResponse = async (body) => { + // If we're using the PICS cache, then add the items in this response to it + if (this.options.enablePicsCache) { + let cache = this.picsCache; + cache.apps = cache.apps || {}; + cache.packages = cache.packages || {}; + + (body.apps || []).forEach((app) => { + let data = { + sha: app.sha ? app.sha.toString('hex') : null, + changenumber: app.change_number, + missingToken: !!app.missing_token, + appinfo: app.buffer ? VDF.parse(app.buffer.toString('utf8')).appinfo : null + }; + + if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { + // Only emit the event if we previously didn't have the appinfo, or if the changenumber changed + this.emit('appUpdate', app.appid, data); + } + + cache.apps[app.appid] = data; + app._parsedData = data; + }); + + (body.packages || []).forEach((pkg) => { + let data = { + sha: pkg.sha ? pkg.sha.toString('hex') : null, + changenumber: pkg.change_number, + missingToken: !!pkg.missing_token, + packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null + }; + + if ((!cache.packages[pkg.packageid] && requestType == PICSRequestType.Changelist) || (cache.packages[pkg.packageid] && cache.packages[pkg.packageid].changenumber != data.changenumber)) { + this.emit('packageUpdate', pkg.packageid, data); + } + + cache.packages[pkg.packageid] = data; + pkg._parsedData = data; + + // Request info for all the apps in this package, if this request didn't originate from the license list, because then we'll first process all packages before requesting all package contents + if (requestType != PICSRequestType.Licenses) { + let appids = (pkg.packageinfo || {}).appids || []; + this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { + }); + } + }); + } + + (body.apps || []).forEach((app) => { + // _parsedData will be populated if we have the PICS cache enabled. + // If we don't, we need to parse the data here. + response.apps[app.appid] = app._parsedData || { + "sha": app.sha ? app.sha.toString('hex') : null, + "changenumber": app.change_number, + "missingToken": !!app.missing_token, + "appinfo": VDF.parse(app.buffer.toString('utf8')).appinfo + }; + + let index = appids.indexOf(app.appid); + if (index != -1) { + appids.splice(index, 1); + } + }); + + (body.packages || []).forEach((pkg) => { + response.packages[pkg.packageid] = pkg._parsedData || { + "sha": pkg.sha ? pkg.sha.toString('hex') : null, + "changenumber": pkg.change_number, + "missingToken": !!pkg.missing_token, + "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null + }; + + let index = packageids.indexOf(pkg.packageid); + if (index != -1) { + packageids.splice(index, 1); + } + }); + + // appids and packageids contain the list of IDs that we're still waiting on data for + if (appids.length === 0 && packageids.length === 0) { + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + return resolve(combined); + } + }; + + // If we don't use PICS cache or require fresh product info for changelist request, then just perform a normal request + if (!this.options.enablePicsCache || requestType == PICSRequestType.Changelist) { + return this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, onResponse); } + cached = await this._getCachedProductInfo(apps, packages); // Note: This callback can be called multiple times this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages, meta_data_only: true}, async (body) => { (body.apps || []).forEach((app) => { @@ -393,7 +487,7 @@ class SteamUserApps extends SteamUserAppAuth { let appTotal = Object.keys(shaList.apps).length + response.unknownApps.length; let packageTotal = Object.keys(shaList.packages).length + response.unknownPackages.length; - // If our collected totals match the total we requested + // If our collected totals match the (unique) total we requested if (appTotal === appids.length && packageTotal === packageids.length) { // Filter out any apps & packages we already have cached and do not need to be refreshed apps = apps.filter((app) => !response.unknownApps.includes(app.appid) && (cached.apps[app.appid] || {}).sha !== shaList.apps[app.appid]); @@ -463,99 +557,8 @@ class SteamUserApps extends SteamUserAppAuth { return resolve(combined); } - // Note: This callback can be called multiple times - this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, async (body) => { - // If we're using the PICS cache, then add the items in this response to it - if (this.options.enablePicsCache) { - let cache = this.picsCache; - cache.apps = cache.apps || {}; - cache.packages = cache.packages || {}; - - (body.apps || []).forEach((app) => { - let data = { - sha: app.sha ? app.sha.toString('hex') : null, - changenumber: app.change_number, - missingToken: !!app.missing_token, - appinfo: app.buffer ? VDF.parse(app.buffer.toString('utf8')).appinfo : null - }; - - if ((!cache.apps[app.appid] && requestType == PICSRequestType.Changelist) || (cache.apps[app.appid] && cache.apps[app.appid].changenumber != data.changenumber)) { - // Only emit the event if we previously didn't have the appinfo, or if the changenumber changed - this.emit('appUpdate', app.appid, data); - } - - cache.apps[app.appid] = data; - - app._parsedData = data; - }); - - (body.packages || []).forEach((pkg) => { - let data = { - sha: pkg.sha ? pkg.sha.toString('hex') : null, - changenumber: pkg.change_number, - missingToken: !!pkg.missing_token, - packageinfo: pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null - }; - - if ((!cache.packages[pkg.packageid] && requestType == PICSRequestType.Changelist) || (cache.packages[pkg.packageid] && cache.packages[pkg.packageid].changenumber != data.changenumber)) { - this.emit('packageUpdate', pkg.packageid, data); - } - - cache.packages[pkg.packageid] = data; - - pkg._parsedData = data; - - // Request info for all the apps in this package, if this request didn't originate from the license list - if (requestType != PICSRequestType.Licenses) { - let appids = (pkg.packageinfo || {}).appids || []; - this.getProductInfo(appids, [], false, null, PICSRequestType.PackageContents).catch(() => { - }); - } - }); - } - - (body.apps || []).forEach((app) => { - // _parsedData will be populated if we have the PICS cache enabled. - // If we don't, we need to parse the data here. - response.apps[app.appid] = app._parsedData || { - "sha": app.sha ? app.sha.toString('hex') : null, - "changenumber": app.change_number, - "missingToken": !!app.missing_token, - "appinfo": VDF.parse(app.buffer.toString('utf8')).appinfo - }; - - let index = appids.indexOf(app.appid); - if (index != -1) { - appids.splice(index, 1); - } - }); - - (body.packages || []).forEach((pkg) => { - response.packages[pkg.packageid] = pkg._parsedData || { - "sha": pkg.sha ? pkg.sha.toString('hex') : null, - "changenumber": pkg.change_number, - "missingToken": !!pkg.missing_token, - "packageinfo": pkg.buffer ? BinaryKVParser.parse(pkg.buffer)[pkg.packageid] : null - }; - - let index = packageids.indexOf(pkg.packageid); - if (index != -1) { - packageids.splice(index, 1); - } - }); - - // appids and packageids contain the list of IDs that we're still waiting on data for - if (appids.length === 0 && packageids.length === 0) { - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages - } - return resolve(combined); - } - }); + // Request the apps & packages we need to refresh + this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, onResponse); } }); }); @@ -595,12 +598,12 @@ class SteamUserApps extends SteamUserAppAuth { unknownApps: [], unknownPackages: [] }; - // Split apps + packages into chunks of 1000 + // Split apps + packages into chunks of 2000 let chunkSize = 2000; for (let i = 0; i < packages.length; i += chunkSize) { let packagesChunk = packages.slice(i, i + chunkSize); // Do not include callback in the request, it will be called multiple times - let result = await this._getProductInfo([], packagesChunk, inclTokens, undefined, requestType); + let result = await this._getProductInfo([], packagesChunk, inclTokens, null, requestType); response = { apps: Object.assign(response.apps, result.apps), packages: Object.assign(response.packages, result.packages), @@ -610,8 +613,7 @@ class SteamUserApps extends SteamUserAppAuth { } for (let i = 0; i < apps.length; i += chunkSize) { let appsChunk = apps.slice(i, i + chunkSize); - // Do not include callback in the request, it will be called multiple times - let result = await this._getProductInfo(appsChunk, [], inclTokens, undefined, requestType); + let result = await this._getProductInfo(appsChunk, [], inclTokens, null, requestType); response = { apps: Object.assign(response.apps, result.apps), packages: Object.assign(response.packages, result.packages), @@ -843,7 +845,7 @@ class SteamUserApps extends SteamUserAppAuth { } let {packages} = result; - // Request info for all the apps in these packages + // Request info for all the apps in these packages, and only after all packages have been processed let appids = []; for (let pkgid in packages) { From 134ee79483411278ad7521bda520fa36dba4eec7 Mon Sep 17 00:00:00 2001 From: Revadike Date: Sun, 31 Jul 2022 03:08:36 +0200 Subject: [PATCH 25/25] Changed execution order --- components/apps.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/components/apps.js b/components/apps.js index 57aa1178..b5e62bac 100644 --- a/components/apps.js +++ b/components/apps.js @@ -504,6 +504,18 @@ class SteamUserApps extends SteamUserAppAuth { // "pkg to refresh:", packages.length, // ); + // If we have nothing to refresh / no stale data (e.g. all of them were unknown) + if (apps.length === 0 && packages.length === 0) { + this._saveProductInfo(response); + let combined = { + apps: Object.assign(cached.apps, response.apps), + packages: Object.assign(cached.packages, response.packages), + unknownApps: response.unknownApps, + unknownPackages: response.unknownPackages + } + return resolve(combined); + } + // We want tokens if (inclTokens) { let _appids = apps.map(app => app.appid); @@ -545,18 +557,6 @@ class SteamUserApps extends SteamUserAppAuth { appids = apps.map(app => app.appid); packageids = packages.map(pkg => pkg.packageid); - // If we have nothing to refresh / no stale data - if (apps.length === 0 && packages.length === 0) { - this._saveProductInfo(response); - let combined = { - apps: Object.assign(cached.apps, response.apps), - packages: Object.assign(cached.packages, response.packages), - unknownApps: response.unknownApps, - unknownPackages: response.unknownPackages - } - return resolve(combined); - } - // Request the apps & packages we need to refresh this._send(EMsg.ClientPICSProductInfoRequest, {apps, packages}, onResponse); }