From 643996d21739c16eda363751945325371bd85633 Mon Sep 17 00:00:00 2001 From: yan Date: Wed, 21 Jun 2017 16:04:55 -0700 Subject: [PATCH 1/2] remove HTTPS Everywhere build scripts this is being moved to https://github.com/brave/https-everywhere-builder --- package.json | 4 +- preload-httpse.js | 155 ---------------------------------------------- 2 files changed, 1 insertion(+), 158 deletions(-) delete mode 100644 preload-httpse.js diff --git a/package.json b/package.json index 2cd4fffb486..3c099a75d97 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,6 @@ "flow": "flow; test $? -eq 0 -o $? -eq 2", "lint": "standard", "postinstall": "npm run download-sync-client && webpack", - "preload-httpse": "node ./preload-httpse.js", "start-log": "node ./tools/start.js --user-data-dir=brave-development --enable-logging=stderr --v=1 --enable-extension-activity-logging --enable-sandbox-logging --enable-dcheck", "start": "node ./tools/start.js --user-data-dir=brave-development --enable-logging --v=0 --enable-extension-activity-logging --enable-sandbox-logging --enable-dcheck", "start2": "node ./tools/start.js --user-data-dir=brave-development-2 --enable-logging --v=0 --enable-extension-activity-logging --enable-sandbox-logging --enable-dcheck", @@ -166,8 +165,7 @@ "jsonfile": "^2.2.3", "less": "^2.5.3", "less-loader": "^2.2.1", - "level": "1.5.0", - "leveldown": "1.5.0", + "level": "^1.7.0", "mkdirp": "^0.5.1", "mocha": "^2.3.4", "mockery": "^1.7.0", diff --git a/preload-httpse.js b/preload-httpse.js deleted file mode 100644 index fcb9f375388..00000000000 --- a/preload-httpse.js +++ /dev/null @@ -1,155 +0,0 @@ -// 1. Download https://www.eff.org/files/https-everywhere-latest.xpi -// 2. unzip https-everywhere-latest.xpi -// 3. cp /path/to/https-everywhere/chrome/content/rulesets.json . -// 4. npm run preload-httpse -// 5. Push httpse.json to AWS -// TODO: Automate this with a git hook. - -'use strict' -var fs = require('fs') -var path = require('path') -var parseString = require('xml2js').parseString -var sqlite3 = require('sqlite3') -var levelup = require('level') - -// Manually exclude sites that are broken until they are fixed in the next -// HTTPS Everywhere release. -var exclusions = { - 'Nike.com.xml': 'breaks nikeplus.com', - 'PJ_Media.xml': 'mixed content on https://pjmedia.com/instapundit/', - 'Slashdot.xml': 'redirect loop on mobile slashdot.org', - 'Delta.com.xml': 'https://delta.com does not redirect to https://www.delta.com', - 'Cargo.xml': 'breaks cargocollective.com', - 'TMZ.com.xml': 'breaks www.tmz.com', - 'BusinessInsider.xml': 'breaks http://www.businessinsider.com/silicon-valley-100-2016-6?op=0', - 'Tesco.xml': 'breaks tesco.com due to CSP mismatch', - 'Vodafone.ie.xml': 'breaks pagination on http://shop.vodafone.ie/shop/phonesAndPlans/phonesAndPlansHome.jsp?subPage=phones&planFilter=onAccount', - 'IDownloadBlog.xml': 'breaks http://www.idownloadblog.com/', - 'EBay_static.com.xml': 'breaks suggested product image previews', - 'Cisco.xml': 'breaks http://www.cisco.com/c/m/en_us/training-events/events-webinars/techwise-tv/listings.html', - 'GQ.xml': 'mixed content on gq.com' -} - -var rulesets = JSON.parse(fs.readFileSync('rulesets.json', 'utf8')) - -// Convert XML rules to JSON -for (let id in rulesets.rulesetStrings) { - let contents = rulesets.rulesetStrings[id] - parseString(contents, function (err, result) { - if (err) { - throw new Error('FATAL: error parsing XML: ' + contents) - } - // Exclude broken rules - var ruleset = result.ruleset - if (ruleset.$.f in exclusions) { - console.log('NOTE: Excluding rule', JSON.stringify(result)) - ruleset.$.default_off = exclusions[ruleset.$.f] - } - rulesets.rulesetStrings[id] = result - }) -} -console.log('Writing httpse.json') -fs.writeFileSync('httpse.json', JSON.stringify(rulesets), 'utf8') - -// Convert httpse.json to sqlite for mobile -console.log('creating httpse.sqlite') -var db = new sqlite3.Database('httpse.sqlite', function (err) { - if (err !== null) { - throw new Error('FATAL: could not open db: ' + err) - } - - db.exec([ - 'DROP TABLE IF EXISTS rulesets', - 'CREATE TABLE rulesets (id INTEGER PRIMARY KEY, contents TEXT)', - 'DROP TABLE IF EXISTS targets', - 'CREATE TABLE targets (host TEXT UNIQUE, ids TEXT)'].join('; '), function (err) { - if (err !== null) { - throw new Error('FATAL: could not create tables: ' + err) - } - - var rulesetStatement = db.prepare('INSERT INTO rulesets (id, contents) VALUES(?, ?)') - var targetStatement = db.prepare('INSERT INTO targets (host, ids) VALUES(?, ?)') - - // TODO: Speed this up - for (var id in rulesets.rulesetStrings) { - let contents = JSON.stringify(rulesets.rulesetStrings[id]) - rulesetStatement.run(id, contents) - } - for (var target in rulesets.targets) { - let ids = JSON.stringify(rulesets.targets[target]) - targetStatement.run(target, ids) - } - }) -}) - -const rmDir = function (dirPath) { - try { - var files = fs.readdirSync(dirPath) - } catch (e) { - return - } - if (files.length > 0) { - for (var i = 0; i < files.length; i++) { - var filePath = path.join(dirPath, files[i]) - if (fs.statSync(filePath).isFile()) { - fs.unlinkSync(filePath) - } else { - rmDir(filePath) - } - } - } - fs.rmdirSync(dirPath) -} - -console.log('creating httpse.leveldb') -rmDir('./httpse.leveldb') - -const httpseLevelDB = levelup('httpse.leveldb', {compression: false, errorIfExists: true}) - -const ruleSets = {} -for (var id in rulesets.rulesetStrings) { - ruleSets[id] = rulesets.rulesetStrings[id] -} - -let batch = httpseLevelDB.batch() -for (var target in rulesets.targets) { - let targetRuleSets = [] - rulesets.targets[target].forEach((id) => { - let ruleset = ruleSets[id] - if (!ruleset.ruleset.$.default_off && !ruleset.ruleset.$.platform) { - let rule = { - r: ruleset.ruleset.rule.map((rule) => { - if (rule.$.from === '^http:' && rule.$.to === 'https:') { - return { d: 1 } - } else { - return { f: rule.$.from, t: rule.$.to } - } - }) - } - if (ruleset.ruleset.exclusion) { - rule.e = ruleset.ruleset.exclusion.map((exclusion) => { - return { p: exclusion.$.pattern } - }) - } - targetRuleSets = targetRuleSets.concat(rule) - } - }) - let reverseTarget = target.split('.').reverse().join('.') - if (targetRuleSets.length > 0) { - batch.put(reverseTarget, JSON.stringify(targetRuleSets), {sync: true}) - } -} - -batch.write((err) => { - if (err) { - console.error(err) - } else { - httpseLevelDB.close((err) => { - if (err) { - console.error(err) - } else { - console.log('done') - } - }) - } -}) From 70bfd751b3c6e753fae9bd1bb8df3aa78f2a5349 Mon Sep 17 00:00:00 2001 From: Brian Clifton Date: Wed, 28 Jun 2017 17:01:09 -0700 Subject: [PATCH 2/2] Cleaning up leveldown references - commented out code from ledger - stubs used in unit testing - electron builder hack (no longer needed) Auditors: @diracdeltas --- app/ledger.js | 29 ------------------- .../app/browser/reducers/tabsReducerTest.js | 1 - test/unit/app/browser/tabsTest.js | 1 - tools/electronBuilderHack.js | 9 ------ tools/lib/ignoredPaths.js | 1 - 5 files changed, 41 deletions(-) diff --git a/app/ledger.js b/app/ledger.js index daafd469080..c5593e880ad 100644 --- a/app/ledger.js +++ b/app/ledger.js @@ -1862,35 +1862,6 @@ var run = (delayTime) => { console.log('what? wait, how can this happen?') } -/* - * ledger client utilities - */ - -/* code that may never be needed... - -var rulesV2Reset = (callback) => { - if (clientOptions.verboseP) console.log('\n\nreset rulesets') - if (!v2RulesetDB) return - - if (client) { - delete client.state.rulesV2Stamp - client.state.updatesStamp = underscore.now() - } - v2RulesetDB.close((err) => { - if (err) console.log(v2RulesetPath + ' close error: ' + err.toString()) - - v2RulesetDB = null - require('leveldown').destroy(pathName(v2RulesetPath), (err) => { - if (err) console.log(v2RulesetPath + ' destroy error: ' + err.toString()) - - v2RulesetDB = levelup(pathName(v2RulesetPath)) - callback() - }) - }) -} - -*/ - var getStateInfo = (state) => { var ballots, i, transaction var info = state.paymentInfo diff --git a/test/unit/app/browser/reducers/tabsReducerTest.js b/test/unit/app/browser/reducers/tabsReducerTest.js index 3daf8c7444b..e42432ea37b 100644 --- a/test/unit/app/browser/reducers/tabsReducerTest.js +++ b/test/unit/app/browser/reducers/tabsReducerTest.js @@ -89,7 +89,6 @@ describe('tabsReducer unit tests', function () { }) mockery.registerMock('electron', fakeElectron) mockery.registerMock('ad-block', fakeAdBlock) - mockery.registerMock('leveldown', {}) this.tabsAPI = { isDevToolsFocused: (tabId) => { diff --git a/test/unit/app/browser/tabsTest.js b/test/unit/app/browser/tabsTest.js index 03d2eaa7aed..409865d388c 100644 --- a/test/unit/app/browser/tabsTest.js +++ b/test/unit/app/browser/tabsTest.js @@ -65,7 +65,6 @@ describe('tabs API unit tests', function () { mockery.registerMock('electron', fakeElectron) mockery.registerMock('ad-block', fakeAdBlock) - mockery.registerMock('leveldown', {}) mockery.registerMock('../../js/stores/appStore', this.appStore) mockery.registerMock('../filtering', { isResourceEnabled: (resourceName, url, isPrivate) => false diff --git a/tools/electronBuilderHack.js b/tools/electronBuilderHack.js index 734f8bcc41f..1829bc033d5 100644 --- a/tools/electronBuilderHack.js +++ b/tools/electronBuilderHack.js @@ -15,15 +15,6 @@ const rebuildCmd = '"../.bin/node-gyp" rebuild' const isDarwin = process.platform === 'darwin' const cmds = [] -if (process.env.TARGET_ARCH === 'ia32') { - cmds.push( - 'cp .npmrc ./node_modules/leveldown', - 'cd ./node_modules/leveldown', - // leveldown prebuild isn't respecting the npm_config_arch env - // for some reason, so always compile it. - 'npm run rebuild' - ) -} if (isDarwin) { cmds.push( diff --git a/tools/lib/ignoredPaths.js b/tools/lib/ignoredPaths.js index 663d5b2096f..2949ca6442a 100644 --- a/tools/lib/ignoredPaths.js +++ b/tools/lib/ignoredPaths.js @@ -21,7 +21,6 @@ module.exports = [ 'Release/obj', 'obj.target', 'ad-block/(node_modules|test|perf|sample|scripts|test|vendor|ABPFilterParserData.dat)', - 'leveldown/(node_modules|src|iojs)', 'tracking-protection/(node_modules|test|data|scripts|vendor)', 'tracking-protection/build/node_modules', 'tracking-protection/build/Release/(sample.exe|test.exe)',