From d05ff8ffebf8a35a9048c52837fa60a7a55f222f Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Mon, 30 Oct 2023 12:30:46 -0400 Subject: [PATCH] Add support for diff-patching filter lists Related discussion: https://github.com/ameshkov/diffupdates The benefits of diff-patching filter lists is much shorter update schedule and significantly less bandwidth consumed. At the moment, only default filter lists are subject to be diff-patched. External filter lists can make their lists diff-patchable by following the specification link above. Only filter lists fetched by the auto-updater are candidate for diff-patching. Forcing a manual update of the filter lists will prevent the diff-patcher from kicking in until one or more lists are auto-updated. Some back-of-the-envelop calculations regarding the load on free CDN solutions used by uBO to distribute its own filter lists: Currently, for each CDN (with lists updating after days): ~560 M req/month, ~78 TB/month With diff-patching lists on a 6-hour schedule: ~390 M req/month, 1 TB/month Those estimates were done according to statistics shown by jsDelivr, which is one of 4 CDNs picked randomly when a list updates: https://www.jsdelivr.com/package/gh/uBlockOrigin/uAssetsCDN?tab=stats --- assets/assets.dev.json | 6 +- src/js/assets.js | 280 ++++++++++++++++++++++++++++++++++------- src/js/diff-updater.js | 243 +++++++++++++++++++++++++++++++++++ src/js/storage.js | 39 ++---- 4 files changed, 490 insertions(+), 78 deletions(-) create mode 100644 src/js/diff-updater.js diff --git a/assets/assets.dev.json b/assets/assets.dev.json index 575ad539a23b0..33d4cd5acbe89 100644 --- a/assets/assets.dev.json +++ b/assets/assets.dev.json @@ -163,8 +163,7 @@ "cdnURLs": [ "https://cdn.jsdelivr.net/gh/uBlockOrigin/uAssetsCDN@latest/thirdparties/easylist.txt", "https://cdn.statically.io/gh/uBlockOrigin/uAssetsCDN/main/thirdparties/easylist.txt", - "https://ublockorigin.pages.dev/thirdparties/easylist.txt", - "https://easylist.to/easylist/easylist.txt" + "https://ublockorigin.pages.dev/thirdparties/easylist.txt" ], "supportURL": "https://easylist.to/" }, @@ -215,8 +214,7 @@ "cdnURLs": [ "https://cdn.jsdelivr.net/gh/uBlockOrigin/uAssetsCDN@latest/thirdparties/easyprivacy.txt", "https://cdn.statically.io/gh/uBlockOrigin/uAssetsCDN/main/thirdparties/easyprivacy.txt", - "https://ublockorigin.pages.dev/thirdparties/easyprivacy.txt", - "https://easylist.to/easylist/easyprivacy.txt" + "https://ublockorigin.pages.dev/thirdparties/easyprivacy.txt" ], "supportURL": "https://easylist.to/" }, diff --git a/src/js/assets.js b/src/js/assets.js index 73fec016c5597..29b14383733ca 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -35,6 +35,9 @@ import { ubolog } from './console.js'; const reIsExternalPath = /^(?:[a-z-]+):\/\//; const reIsUserAsset = /^user-/; const errorCantConnectTo = i18n$('errorCantConnectTo'); +const MS_PER_HOUR = 60 * 60 * 1000; +const MS_PER_DAY = 24 * MS_PER_HOUR; +const EXPIRES_DEFAULT = 7; const assets = {}; @@ -42,16 +45,57 @@ const assets = {}; // bandwidth of remote servers. let remoteServerFriendly = false; +/******************************************************************************/ + +const parseExpires = s => { + const matches = s.match(/(\d+)\s*([dh])?/i); + if ( matches === null ) { return 0; } + let updateAfter = parseInt(matches[1], 10); + if ( matches[2] === 'h' ) { + updateAfter = Math.ceil(updateAfter / 12) / 2; + } + return updateAfter; +}; + +const extractMetadataFromList = (content, fields) => { + const out = {}; + const head = content.slice(0, 1024); + for ( let field of fields ) { + field = field.replace(/\s+/g, '-'); + const re = new RegExp(`^(?:! *|# +)${field.replace(/-/g, '(?: +|-)')}: *(.+)$`, 'im'); + const match = re.exec(head); + let value = match && match[1].trim() || undefined; + if ( value !== undefined && /^%.+%$/.test(value) ) { + value = undefined; + } + field = field.toLowerCase().replace( + /-[a-z]/g, s => s.charAt(1).toUpperCase() + ); + out[field] = value; + } + // Pre-process known fields + if ( out.lastModified ) { + out.lastModified = (new Date(out.lastModified)).getTime() || 0; + } + if ( out.expires ) { + out.expires = Math.max(parseExpires(out.expires), 0.5); + } + if ( out.diffExpires ) { + out.diffExpires = Math.max(parseExpires(out.diffExpires), 0.25); + } + return out; +}; +assets.extractMetadataFromList = extractMetadataFromList; + const resourceTimeFromXhr = xhr => { try { // First lookup timestamp from content let assetTime = 0; if ( typeof xhr.response === 'string' ) { - const head = xhr.response.slice(0, 512); - const match = /^! Last modified: (.+)$/m.exec(head); - if ( match ) { - assetTime = (new Date(match[1])).getTime() || 0; - } + const metadata = extractMetadataFromList(xhr.response, [ + 'Last-Modified' + ]); + assetTime = metadata.lastModified || 0; } // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Age // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date @@ -91,6 +135,40 @@ const resourceIsStale = (networkDetails, cacheDetails) => { return false; }; +const getUpdateAfterTime = (assetKey, diff = false) => { + const entry = assetCacheRegistry[assetKey]; + if ( entry ) { + if ( diff && typeof entry.diffExpires === 'number' ) { + return entry.diffExpires * MS_PER_DAY; + } + if ( typeof entry.expires === 'number' ) { + return entry.expires * MS_PER_DAY; + } + } + if ( assetSourceRegistry ) { + const entry = assetSourceRegistry[assetKey]; + if ( entry && typeof entry.updateAfter === 'number' ) { + return entry.updateAfter * MS_PER_DAY; + } + } + return EXPIRES_DEFAULT * MS_PER_DAY; // default to 7-day +}; + +const getWriteTime = assetKey => { + const entry = assetCacheRegistry[assetKey]; + if ( entry ) { return entry.writeTime || 0; } + return 0; +}; + +const isDiffUpdatableAsset = content => { + if ( typeof content !== 'string' ) { return false; } + const data = extractMetadataFromList(content, [ + 'Diff-Path', + ]); + return typeof data.diffPath === 'string' && + /^[^%].*[^%]$/.test(data.diffPath); +}; + const stringIsNotEmpty = s => typeof s === 'string' && s !== ''; /******************************************************************************/ @@ -252,14 +330,6 @@ assets.fetchText = async function(url) { details.content = ''; details.error = 'assets.fetchText(): Not a text file'; } - - // Important: Non empty text resource must always end with a newline - if ( - details.content.length !== 0 && - details.content.endsWith('\n') === false - ) { - details.content += '\n'; - } } catch(ex) { details = ex; } @@ -374,6 +444,13 @@ assets.fetchFilterList = async function(mainlistURL) { return { url: mainlistURL, content: '', error: part.error }; } resourceTime = resourceTimeFromParts(allParts, resourceTime); + // Skip pre-parser directives for diff-updatable assets + if ( allParts.length === 1 && allParts[0] instanceof Object ) { + if ( isDiffUpdatableAsset(allParts[0].content) ) { + allParts[0] = allParts[0].content; + break; + } + } allParts = processIncludeDirectives(allParts); } while ( allParts.some(part => typeof part !== 'string') ); // If we reach this point, this means all fetches were successful. @@ -457,9 +534,6 @@ function registerAssetSource(assetKey, newDict) { } else if ( currentDict.contentURL === undefined ) { currentDict.contentURL = []; } - if ( typeof currentDict.updateAfter !== 'number' ) { - currentDict.updateAfter = 7; - } if ( currentDict.submitter ) { currentDict.submitTime = Date.now(); // To detect stale entries } @@ -607,7 +681,7 @@ async function assetCacheRead(assetKey, updateReadTime = false) { const reportBack = function(content) { if ( content instanceof Blob ) { content = ''; } - const details = { assetKey: assetKey, content: content }; + const details = { assetKey, content }; if ( content === '' ) { details.error = 'ENOTFOUND'; } return details; }; @@ -979,6 +1053,18 @@ async function getRemote(assetKey) { url: contentURL, resourceTime: result.resourceTime || 0, }); + + if ( assetDetails.content === 'filters' ) { + const metadata = extractMetadataFromList(result.content, [ + 'Last-Modified', + 'Expires', + 'Diff-Name', + 'Diff-Path', + 'Diff-Expires', + ]); + assetCacheSetDetails(assetKey, metadata); + } + registerAssetSource(assetKey, { birthtime: undefined, error: undefined }); return reportBack(result.content); } @@ -1029,8 +1115,7 @@ assets.metadata = async function() { if ( cacheEntry ) { assetEntry.cached = true; assetEntry.writeTime = cacheEntry.writeTime; - const obsoleteAfter = - cacheEntry.writeTime + assetEntry.updateAfter * 86400000; + const obsoleteAfter = cacheEntry.writeTime + getUpdateAfterTime(assetKey); assetEntry.obsolete = obsoleteAfter < now; assetEntry.remoteURL = cacheEntry.remoteURL; } else if ( @@ -1074,7 +1159,7 @@ assets.getUpdateAges = async function(conditions = {}) { out.push({ assetKey, age, - ageNormalized: age / (asset.updateAfter * 86400000), + ageNormalized: age / getUpdateAfterTime(assetKey), }); } return out; @@ -1091,33 +1176,122 @@ let updaterStatus; let updaterAssetDelay = updaterAssetDelayDefault; let updaterAuto = false; -const updateFirst = function() { +const getAssetDiffDetails = assetKey => { + const out = { name: assetKey }; + const cacheEntry = assetCacheRegistry[assetKey]; + if ( cacheEntry === undefined ) { return; } + if ( cacheEntry.diffPath === undefined ) { return; } + if ( cacheEntry.diffName === undefined ) { return; } + out.diffName = cacheEntry.diffName; + out.patchPath = cacheEntry.diffPath; + const assetEntry = assetSourceRegistry[assetKey]; + if ( assetEntry === undefined ) { return; } + if ( Array.isArray(assetEntry.cdnURLs) === false ) { return; } + out.cdnURLs = assetEntry.cdnURLs.slice(); + return out; +}; + +async function diffUpdater() { + const toUpdate = await getUpdateCandidates(true); + const now = Date.now(); + const toHardUpdate = []; + const toSoftUpdate = []; + while ( toUpdate.length !== 0 ) { + const assetKey = toUpdate.shift(); + const assetDetails = getAssetDiffDetails(assetKey); + if ( assetDetails === undefined ) { continue; } + if ( assetDetails.patchPath === undefined ) { continue; } + if ( assetDetails.diffName === undefined ) { continue; } + assetDetails.what = 'update'; + if ( (getWriteTime(assetKey) + getUpdateAfterTime(assetKey, true)) > now ) { + toSoftUpdate.push(assetDetails); + } else { + toHardUpdate.push(assetDetails); + } + } + if ( toHardUpdate.length === 0 ) { return; } + ubolog('Diff updater: cycle start'); + return new Promise(resolve => { + let pendingOps = 0; + const bc = new globalThis.BroadcastChannel('diffUpdater'); + bc.onmessage = ev => { + const data = ev.data; + if ( data.what === 'ready' ) { + ubolog('Diff updater: hard updating', toHardUpdate.map(v => v.name).join()); + while ( toHardUpdate.length !== 0 ) { + const assetDetails = toHardUpdate.shift(); + assetDetails.fetch = true; + bc.postMessage(assetDetails); + pendingOps += 1; + } + return; + } + if ( data.status === 'needtext' ) { + ubolog('Diff updater: need text for', data.name); + assetCacheRead(data.name).then(result => { + data.text = result.content; + data.status = undefined; + bc.postMessage(data); + }); + return; + } + if ( data.status === 'updated' ) { + ubolog(`Diff updater: successfully patched ${data.name} using ${data.diffURL}`); + const metadata = extractMetadataFromList(data.text, [ + 'Last-Modified', + 'Expires', + 'Diff-Name', + 'Diff-Path', + 'Diff-Expires', + ]); + assetCacheWrite(data.name, { + content: data.text, + resourceTime: metadata.lastModified || 0, + }); + assetCacheSetDetails(data.name, metadata); + } else if ( data.error ) { + ubolog(`Diff updater: failed to diff-update ${data.name}, reason: ${data.error}`); + } + pendingOps -= 1; + if ( pendingOps === 0 && toSoftUpdate.length !== 0 ) { + ubolog('Diff updater: soft updating', toSoftUpdate.map(v => v.name).join()); + while ( toSoftUpdate.length !== 0 ) { + bc.postMessage(toSoftUpdate.shift()); + pendingOps += 1; + } + } + if ( pendingOps !== 0 ) { return; } + ubolog('Diff updater: cycle complete'); + worker.terminate(); + bc.close(); + resolve(); + }; + const worker = new Worker('js/diff-updater.js'); + }); +} + +function updateFirst() { updaterStatus = 'updating'; updaterFetched.clear(); updaterUpdated.length = 0; - fireNotification('before-assets-updated'); - updateNext(); -}; + diffUpdater().catch(reason => { + ubolog(reason); + }).finally(( ) => { + updateNext(); + }); +} -const updateNext = async function() { +async function getUpdateCandidates() { const [ assetDict, cacheDict ] = await Promise.all([ getAssetSourceRegistry(), getAssetCacheRegistry(), ]); - - const now = Date.now(); const toUpdate = []; for ( const assetKey in assetDict ) { const assetEntry = assetDict[assetKey]; if ( assetEntry.hasRemoteURL !== true ) { continue; } if ( updaterFetched.has(assetKey) ) { continue; } const cacheEntry = cacheDict[assetKey]; - if ( - (cacheEntry instanceof Object) && - (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now - ) { - continue; - } if ( fireNotification('before-asset-updated', { assetKey, @@ -1132,9 +1306,6 @@ const updateNext = async function() { assetCacheRemove(assetKey); } } - if ( toUpdate.length === 0 ) { - return updateDone(); - } // https://github.com/uBlockOrigin/uBlock-issues/issues/1165 // Update most obsolete asset first. toUpdate.sort((a, b) => { @@ -1142,17 +1313,34 @@ const updateNext = async function() { const tb = cacheDict[b] !== undefined ? cacheDict[b].writeTime : 0; return ta - tb; }); - updaterFetched.add(toUpdate[0]); + return toUpdate; +} + +async function updateNext() { + const toUpdate = await getUpdateCandidates(); + const now = Date.now(); + const toHardUpdate = []; + + while ( toUpdate.length !== 0 ) { + const assetKey = toUpdate.shift(); + const writeTime = getWriteTime(assetKey); + const updateDelay = getUpdateAfterTime(assetKey); + if ( (writeTime + updateDelay) > now ) { continue; } + toHardUpdate.push(assetKey); + } + if ( toHardUpdate.length === 0 ) { + return updateDone(); + } + + const assetKey = toHardUpdate.pop(); + updaterFetched.add(assetKey); // In auto-update context, be gentle on remote servers. remoteServerFriendly = updaterAuto; let result; - if ( - toUpdate[0] !== 'assets.json' || - µb.hiddenSettings.debugAssetsJson !== true - ) { - result = await getRemote(toUpdate[0]); + if ( assetKey !== 'assets.json' || µb.hiddenSettings.debugAssetsJson !== true ) { + result = await getRemote(assetKey); } else { result = await assets.fetchText(µb.assetsJsonPath); result.assetKey = 'assets.json'; @@ -1161,8 +1349,10 @@ const updateNext = async function() { remoteServerFriendly = false; if ( result.error ) { + ubolog(`Full updater: failed to update ${assetKey}`); fireNotification('asset-update-failed', { assetKey: result.assetKey }); } else { + ubolog(`Full updater: successfully updated ${assetKey}`); updaterUpdated.push(result.assetKey); if ( result.assetKey === 'assets.json' && result.content !== '' ) { updateAssetSourceRegistry(result.content); @@ -1170,18 +1360,18 @@ const updateNext = async function() { } updaterTimer.on(updaterAssetDelay); -}; +} const updaterTimer = vAPI.defer.create(updateNext); -const updateDone = function() { +function updateDone() { const assetKeys = updaterUpdated.slice(0); updaterFetched.clear(); updaterUpdated.length = 0; updaterStatus = undefined; updaterAssetDelay = updaterAssetDelayDefault; fireNotification('after-assets-updated', { assetKeys }); -}; +} assets.updateStart = function(details) { const oldUpdateDelay = updaterAssetDelay; diff --git a/src/js/diff-updater.js b/src/js/diff-updater.js new file mode 100644 index 0000000000000..3467dbf22d3fa --- /dev/null +++ b/src/js/diff-updater.js @@ -0,0 +1,243 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +// This module can be dynamically loaded or spun off as a worker. + +/******************************************************************************/ + +const patches = new Map(); +const encoder = new TextEncoder(); +const reFileName = /[^\/]+$/; +const EMPTYLINE = ''; + +/******************************************************************************/ + +const suffleArray = arr => { + const out = arr.slice(); + for ( let i = 0, n = out.length; i < n; i++ ) { + const j = Math.floor(Math.random() * n); + if ( j === i ) { continue; } + [ out[j], out[i] ] = [ out[i], out[j] ]; + } + return out; +}; + +const basename = url => { + const match = reFileName.exec(url); + return match && match[0] || ''; +}; + +const resolveURL = (path, url) => { + try { + const urlAfter = new URL(path, url); + return urlAfter.href; + } + catch(_) { + } +}; + +function parsePatch(patch) { + const patchDetails = new Map(); + const diffLines = patch.split('\n'); + let i = 0, n = diffLines.length; + while ( i < n ) { + const line = diffLines[i++]; + if ( line.startsWith('diff ') === false ) { continue; } + const fields = line.split(/\s+/); + const diffBlock = {}; + for ( let j = 0; j < fields.length; j++ ) { + const field = fields[j]; + const pos = field.indexOf(':'); + if ( pos === -1 ) { continue; } + const name = field.slice(0, pos); + if ( name === '' ) { continue; } + const value = field.slice(pos+1); + switch ( name ) { + case 'name': + case 'checksum': + diffBlock[name] = value; + break; + case 'lines': + diffBlock.lines = parseInt(value, 10); + break; + default: + break; + } + } + if ( diffBlock.name === undefined ) { return; } + if ( isNaN(diffBlock.lines) || diffBlock.lines <= 0 ) { return; } + if ( diffBlock.checksum === undefined ) { return; } + patchDetails.set(diffBlock.name, diffBlock); + diffBlock.diff = diffLines.slice(i, i + diffBlock.lines).join('\n'); + i += diffBlock.lines; + } + if ( patchDetails.size === 0 ) { return; } + return patchDetails; +} + +function applyPatch(text, diff) { + // Inspired from (Perl) "sub _patch" at: + // https://twiki.org/p/pub/Codev/RcsLite/RcsLite.pm + // Apparently authored by John Talintyre in Jan. 2002 + // https://twiki.org/cgi-bin/view/Codev/RcsLite + const lines = text.split('\n'); + const diffLines = diff.split('\n'); + let iAdjust = 0; + let iDiff = 0, nDiff = diffLines.length; + while ( iDiff < nDiff ) { + const diffParsed = /^([ad])(\d+) (\d+)$/.exec(diffLines[iDiff++]); + if ( diffParsed === null ) { return; } + const op = diffParsed[1]; + const iOp = parseInt(diffParsed[2], 10); + const nOp = parseInt(diffParsed[3], 10); + const iOpAdj = iOp + iAdjust; + if ( iOpAdj > lines.length ) { return; } + // Delete lines + if ( op === 'd' ) { + lines.splice(iOpAdj-1, nOp); + iAdjust -= nOp; + continue; + } + // Add lines: Don't use splice() to avoid stack limit issues + for ( let i = 0; i < nOp; i++ ) { + lines.push(EMPTYLINE); + } + lines.copyWithin(iOpAdj+nOp, iOpAdj); + for ( let i = 0; i < nOp; i++ ) { + lines[iOpAdj+i] = diffLines[iDiff+i]; + } + iAdjust += nOp; + iDiff += nOp; + } + return lines.join('\n'); +} + +/******************************************************************************/ + +// Async + +async function applyPatchAndValidate(assetDetails, diffDetails) { + const { text } = assetDetails; + const { diff, checksum } = diffDetails; + const textAfter = applyPatch(text, diff); + if ( typeof textAfter !== 'string' ) { + assetDetails.error = 'baddiff'; + return false; + } + const crypto = globalThis.crypto; + if ( typeof crypto !== 'object' ) { + assetDetails.error = 'nocrypto'; + return false; + } + const arrayin = encoder.encode(textAfter); + const arraybuffer = await crypto.subtle.digest('SHA-1', arrayin); + const arrayout = new Uint8Array(arraybuffer); + const sha1Full = Array.from(arrayout).map(i => + i.toString(16).padStart(2, '0') + ).join(''); + if ( sha1Full.startsWith(checksum) === false ) { + assetDetails.error = 'badchecksum'; + return false; + } + assetDetails.text = textAfter; + const head = textAfter.slice(0, 1024); + let match = /^! Diff-Path: (\S+)/m.exec(head); + assetDetails.patchPath = match ? match[1] : undefined; + match = /^! Diff-Name: (\S+)/m.exec(head); + assetDetails.diffName = match ? match[1] : undefined; + return true; +} + +async function fetchPatchDetailsFromCDNs(assetDetails) { + const { patchPath, cdnURLs } = assetDetails; + if ( Array.isArray(cdnURLs) === false ) { return null; } + if ( cdnURLs.length === 0 ) { return null; } + for ( const cdnURL of suffleArray(cdnURLs) ) { + const diffURL = resolveURL(patchPath, cdnURL); + if ( diffURL === undefined ) { continue; } + const response = await fetch(diffURL).catch(reason => { + console.error(reason); + }); + if ( response === undefined ) { continue; } + if ( response.ok !== true ) { continue; } + const patchText = await response.text(); + const patchDetails = parsePatch(patchText); + if ( patchDetails === undefined ) { continue; } + return { diffURL, patchDetails }; + } + return null; +} + +async function fetchPatchDetails(assetDetails) { + const { patchPath } = assetDetails; + const patchFile = basename(patchPath); + if ( patchFile === '' ) { return null; } + if ( patches.has(patchFile) ) { + return patches.get(patchFile); + } + if ( assetDetails.fetch === false ) { return null; } + const patchDetailsPromise = fetchPatchDetailsFromCDNs(assetDetails); + patches.set(patchFile, patchDetailsPromise); + return patchDetailsPromise; +} + +async function fetchAndApplyAllPatches(assetDetails) { + const { diffURL, patchDetails } = await fetchPatchDetails(assetDetails); + if ( patchDetails === null ) { + assetDetails.error = 'nopatch'; + return assetDetails; + } + const diffDetails = patchDetails.get(assetDetails.diffName); + if ( diffDetails === undefined ) { + assetDetails.error = 'nodiff'; + return assetDetails; + } + if ( assetDetails.text === undefined ) { + assetDetails.status = 'needtext'; + return assetDetails; + } + const outcome = await applyPatchAndValidate(assetDetails, diffDetails); + if ( outcome !== true ) { return assetDetails; } + assetDetails.status = 'updated'; + assetDetails.diffURL = diffURL; + return assetDetails; +} + +/******************************************************************************/ + +const bc = new globalThis.BroadcastChannel('diffUpdater'); + +bc.onmessage = ev => { + const message = ev.data; + switch ( message.what ) { + case 'update': + fetchAndApplyAllPatches(message).then(response => { + bc.postMessage(response); + }); + break; + } +}; + +bc.postMessage({ what: 'ready' }); + +/******************************************************************************/ diff --git a/src/js/storage.js b/src/js/storage.js index a8e0a7c81c36d..4def990e21932 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1009,39 +1009,20 @@ import { µb.extractFilterListMetadata = function(assetKey, raw) { const listEntry = this.availableFilterLists[assetKey]; if ( listEntry === undefined ) { return; } - // Metadata expected to be found at the top of content. - const head = raw.slice(0, 1024); // https://github.com/gorhill/uBlock/issues/313 // Always try to fetch the name if this is an external filter list. - if ( listEntry.group === 'custom' ) { - let matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Title[\t ]*:([^\n]+)/i); - const title = matches && matches[1].trim() || ''; - if ( title !== '' && title !== listEntry.title ) { - listEntry.title = orphanizeString(title); - io.registerAssetSource(assetKey, { title }); - } - matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Homepage[\t ]*:[\t ]*(https?:\/\/\S+)\s/i); - const supportURL = matches && matches[1] || ''; - if ( supportURL !== '' && supportURL !== listEntry.supportURL ) { - listEntry.supportURL = orphanizeString(supportURL); - io.registerAssetSource(assetKey, { supportURL }); - } - } - // Extract update frequency information - const matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Expires[\t ]*:[\t ]*(\d+)[\t ]*(h)?/i); - if ( matches !== null ) { - let updateAfter = parseInt(matches[1], 10); - if ( isNaN(updateAfter) === false ) { - if ( matches[2] !== undefined ) { - updateAfter = Math.ceil(updateAfter / 12) / 2; - } - updateAfter = Math.max(updateAfter, 0.5); - if ( updateAfter !== listEntry.updateAfter ) { - listEntry.updateAfter = updateAfter; - io.registerAssetSource(assetKey, { updateAfter }); - } + if ( listEntry.group !== 'custom' ) { return; } + const data = io.extractMetadataFromList(raw, [ 'Title', 'Homepage' ]); + const props = {}; + if ( data.title && data.title !== listEntry.title ) { + props.title = listEntry.title = orphanizeString(data.title); + } + if ( data.homepage && /^https?:\/\/\S+/.test(data.homepage) ) { + if ( data.homepage !== listEntry.supportURL ) { + props.supportURL = listEntry.supportURL = orphanizeString(data.homepage); } } + io.registerAssetSource(assetKey, props); }; /******************************************************************************/