Skip to content

Commit

Permalink
Adds media publishers support
Browse files Browse the repository at this point in the history 
Resolves brave#11851

Auditors:

Test Plan:
  • Loading branch information
@NejcZdovc
NejcZdovc committed Nov 10, 2017
1 parent ac5f0a8 commit 78252fa
Show file tree
Hide file tree
Showing 22 changed files with 969 additions and 243 deletions.
195 changes: 158 additions & 37 deletions app/browser/api/ledger.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ const request = require('../../../js/lib/request')
const ledgerUtil = require('../../common/lib/ledgerUtil')
const tabState = require('../../common/state/tabState')
const pageDataUtil = require('../../common/lib/pageDataUtil')
const ledgerVideoCache = require('../../common/cache/ledgerVideoCache')

// Caching
let locationDefault = 'NOOP'
Expand Down Expand Up @@ -557,7 +558,8 @@ const getPublisherData = (result, scorekeeper) => {
let data = {
verified: result.options.verified || false,
exclude: result.options.exclude || false,
site: result.publisherKey,
publisherKey: result.publisherKey,
siteName: result.publisherKey,
views: result.visits,
duration: duration,
daysSpent: 0,
Expand All @@ -569,9 +571,16 @@ const getPublisherData = (result, scorekeeper) => {
pinPercentage: result.pinPercentage,
weight: result.pinPercentage
}
// HACK: Protocol is sometimes blank here, so default to http:// so we can
// still generate publisherURL.
data.publisherURL = (result.protocol || 'http:') + '//' + result.publisherKey

data.publisherURL = result.publisherURL || ((result.protocol || 'https:') + '//' + result.publisherKey)

// media publisher
if (result.faviconName) {
data.siteName = locale.translation('publisherMediaName', {
publisherName: result.faviconName,
provider: result.providerName
})
}

if (duration >= miliseconds.day) {
data.daysSpent = Math.max(Math.round(duration / miliseconds.day), 1)
Expand Down Expand Up @@ -853,7 +862,7 @@ const excludeP = (publisherKey, callback) => {
return done(err, result)
}

let props = ledgerPublisher.getPublisherProps('https://' + publisherKey)
let props = ledgerPublisher.getPublisherProps(publisherKey)
if (!props) return done()

v2RulesetDB.createReadStream({lt: 'domain:'}).on('data', (data) => {
Expand Down Expand Up @@ -894,30 +903,26 @@ const excludeP = (publisherKey, callback) => {
})
}

const addVisit = (state, startTimestamp, location, tabId) => {
const addSiteVisit = (state, timestamp, location, tabId) => {
if (!synopsis) {
return state
}

location = pageDataUtil.getInfoKey(location)
const locationData = ledgerState.getLocation(state, location)
const timestamp = new Date().getTime()
const duration = new Date().getTime() - timestamp
if (_internal.verboseP) {
console.log(
`locations[${location}]=${JSON.stringify(locationData, null, 2)} ` +
`duration=${(timestamp - startTimestamp)} msec tabId= ${tabId}`
`duration=${(duration)} msec tabId= ${tabId}`
)
}
if (locationData.isEmpty() || !tabId) {
return state
}

let publisherKey = locationData.get('publisher')
if (!publisherKey) {
if (locationData.isEmpty()) {
return state
}

let duration = timestamp - startTimestamp
let publisherKey = locationData.get('publisher')
let revisitP = false

if (duration >= getSetting(settings.PAYMENTS_MINIMUM_VISIT_TIME)) {
Expand All @@ -937,13 +942,19 @@ const addVisit = (state, startTimestamp, location, tabId) => {
}
}

return saveVisit(state, publisherKey, duration, revisitP)
}

const saveVisit = (state, publisherKey, duration, revisited) => {
if (!synopsis || !publisherKey) {
return state
}

if (_internal.verboseP) {
console.log('\nadd publisher ' + publisherKey + ': ' + (duration / 1000) + ' sec' + ' revisitP=' + revisitP + ' state=' +
JSON.stringify(underscore.extend({location: location}, visitsByPublisher[publisherKey][location]),
null, 2))
console.log('\nadd publisher ' + publisherKey + ': ' + (duration / 1000) + ' sec' + ' revisitP=' + revisited)
}

synopsis.addPublisher(publisherKey, {duration: duration, revisitP: revisitP})
synopsis.addPublisher(publisherKey, {duration: duration, revisitP: revisited})
state = ledgerState.setPublisher(state, publisherKey, synopsis.publishers[publisherKey])
state = updatePublisherInfo(state)
state = checkVerifiedStatus(state, publisherKey)
Expand Down Expand Up @@ -994,7 +1005,7 @@ const addNewLocation = (state, location, tabId = tabState.TAB_ID_NONE, keepInfo

// Add visit to the ledger when we are not in a private tab
if (!isPrivate && !tabFromState.isEmpty() && ledgerUtil.shouldTrackView(tabFromState)) {
state = addVisit(state, currentTimestamp, currentUrl, currentTabId)
state = addSiteVisit(state, currentTimestamp, currentUrl, currentTabId)
}
}

Expand Down Expand Up @@ -1440,21 +1451,12 @@ const cacheRuleSet = (state, ruleset) => {
for (let item of publishers) {
const publisherKey = item[0]
const publisher = item[1]
const location = (publisher.get('protocol') || 'http:') + '//' + publisherKey
let ctx = urlParse(location)
const ctx = ledgerPublisher.getPublisherProps(publisherKey)

ctx.TLD = tldjs.getPublicSuffix(ctx.host)
if (!ctx.TLD) {
return state
}
if (!ctx.TLD) continue

ctx = underscore.mapObject(ctx, function (value) {
if (!underscore.isFunction(value)) return value
})
ctx.URL = location
ctx.SLD = tldjs.getDomain(ctx.host)
ctx.RLD = tldjs.getSubdomain(ctx.host)
ctx.QLD = ctx.RLD ? underscore.last(ctx.RLD.split('.')) : ''
if (publisher.publisherURL) ctx.URL = publisher.publisherURL
if (!ctx.URL) ctx.URL = (publisher.get('protocol') || 'https:') + '//' + publisherKey

stewed.forEach((rule) => {
if (rule.consequent !== null || rule.dom) return
Expand Down Expand Up @@ -1488,7 +1490,8 @@ const roundtrip = (params, options, callback) => {
let parts = typeof params.server === 'string' ? urlParse(params.server)
: typeof params.server !== 'undefined' ? params.server
: typeof options.server === 'string' ? urlParse(options.server) : options.server
const rawP = options.rawP
const binaryP = options.binaryP
const rawP = binaryP || options.rawP

if (!params.method) params.method = 'GET'
parts = underscore.extend(underscore.pick(parts, ['protocol', 'hostname', 'port']),
Expand All @@ -1515,7 +1518,7 @@ const roundtrip = (params, options, callback) => {
url: urlFormat(parts),
method: params.method,
payload: params.payload,
responseType: 'text',
responseType: binaryP ? 'binary' : 'text',
headers: underscore.defaults(params.headers || {}, {'content-type': 'application/json; charset=utf-8'}),
verboseP: options.verboseP
}
Expand All @@ -1530,7 +1533,7 @@ const roundtrip = (params, options, callback) => {
console.log('>>> ' + header + ': ' + response.headers[header])
})
console.log('>>>')
console.log('>>> ' + (body || '').split('\n').join('\n>>> '))
console.log('>>> ' + (rawP ? '...' : (body || '').split('\n').join('\n>>> ')))
}

if (err) return callback(err, response)
Expand Down Expand Up @@ -2457,6 +2460,117 @@ const transitionWalletToBat = () => {
}
}

let currentMediaKey = null
const onMediaRequest = (state, xhr, type, tabId) => {
if (!xhr || type == null) {
return state
}

const parsed = ledgerUtil.getMediaData(xhr, type)
const mediaId = ledgerUtil.getMediaId(parsed, type)
const mediaKey = ledgerUtil.getMediaKey(mediaId, type)
let duration = ledgerUtil.getMediaDuration(parsed, type)

if (mediaId == null || duration == null || mediaKey == null) {
return state
}

const minDuration = getSetting(settings.PAYMENTS_MINIMUM_VISIT_TIME)
duration = parseInt(duration)
if (duration > 0 && duration < minDuration) {
duration = minDuration
}

if (!ledgerPublisher) {
ledgerPublisher = require('bat-publisher')
}

let revisited = true
const activeTabId = tabState.getActiveTabId(state)
if (activeTabId === tabId && mediaKey !== currentMediaKey) {
revisited = false
currentMediaKey = mediaKey
}

const cache = ledgerVideoCache.getDataByVideoId(state, mediaKey)

if (!cache.isEmpty()) {
return module.exports.saveVisit(state, cache.get('publisher'), duration, revisited)
}

const options = underscore.extend({roundtrip: roundtrip}, clientOptions)
const mediaProps = {
mediaId,
providerName: type
}

ledgerPublisher.getMedia.getPublisherFromMediaProps(mediaProps, options, (error, response) => {
if (error) {
console.error('Error while getting publisher from media', error.toString())
return
}

// publisher not found
if (!response) {
return
}

appActions.onLedgerMediaPublisher(mediaKey, response, duration, revisited)
})

return state
}

const onMediaPublisher = (state, mediaKey, response, duration, revisited) => {
const publisherKey = response ? response.get('publisher') : null
if (publisherKey == null) {
return state
}

let publisher = ledgerState.getPublisher(state, publisherKey)
const faviconName = response.get('faviconName')
const faviconURL = response.get('faviconURL')
const publisherURL = response.get('publisherURL')

if (publisher.isEmpty()) {
revisited = false
synopsis.initPublisher(publisherKey)

synopsis.publishers[publisherKey].faviconName = faviconName
synopsis.publishers[publisherKey].faviconURL = faviconURL
synopsis.publishers[publisherKey].publisherURL = publisherURL
synopsis.publishers[publisherKey].providerName = response.get('providerName')

if (synopsis.publishers[publisherKey]) {
state = ledgerState.setPublisher(state, publisherKey, synopsis.publishers[publisherKey])
}

if (!getSetting(settings.PAYMENTS_SITES_AUTO_SUGGEST)) {
appActions.onPublisherOptionUpdate(publisherKey, 'exclude', true)
savePublisherOption(publisherKey, 'exclude', true)
} else {
excludeP(publisherKey, (unused, exclude) => {
appActions.onPublisherOptionUpdate(publisherKey, 'exclude', exclude)
savePublisherOption(publisherKey, 'exclude', exclude)
})
}
} else {
synopsis.publishers[publisherKey].faviconName = faviconName
synopsis.publishers[publisherKey].faviconURL = faviconURL
synopsis.publishers[publisherKey].publisherURL = publisherURL
state = ledgerState.setPublishersProp(state, publisherKey, 'faviconName', faviconName)
state = ledgerState.setPublishersProp(state, publisherKey, 'faviconURL', faviconURL)
state = ledgerState.setPublishersProp(state, publisherKey, 'publisherURL', publisherURL)
}

// Add to cache
state = ledgerVideoCache.setCacheByVideoId(state, mediaKey, response)

state = module.exports.saveVisit(state, publisherKey, duration, revisited)

return state
}

const getMethods = () => {
const publicMethods = {
backupKeys,
Expand Down Expand Up @@ -2491,15 +2605,18 @@ const getMethods = () => {
transitionWalletToBat,
getNewClient,
savePublisherData,
pruneSynopsis
pruneSynopsis,
onMediaRequest,
onMediaPublisher,
saveVisit
}

let privateMethods = {}

if (process.env.NODE_ENV === 'test') {
privateMethods = {
enable,
addVisit,
addSiteVisit,
checkBtcBatMigrated,
clearVisitsByPublisher: function () {
visitsByPublisher = {}
Expand All @@ -2514,6 +2631,10 @@ const getMethods = () => {
setClient: (data) => {
client = data
},
setCurrentMediaKey: (key) => {
currentMediaKey = key
},
getCurrentMediaKey: (key) => currentMediaKey,
synopsisNormalizer,
checkVerifiedStatus
}
Expand Down
16 changes: 16 additions & 0 deletions app/browser/reducers/ledgerReducer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,22 @@ const ledgerReducer = (state, action, immutableAction) => {
state = ledgerState.setLedgerValue(state, 'publisherTimestamp', action.get('timestamp'))
break
}
case appConstants.APP_ON_LEDGER_MEDIA_DATA:
{
state = ledgerApi.onMediaRequest(state, action.get('url'), action.get('type'), action.get('tabId'))
break
}
case appConstants.APP_ON_LEDGER_MEDIA_PUBLISHER:
{
state = ledgerApi.onMediaPublisher(
state,
action.get('mediaKey'),
action.get('response'),
action.get('duration'),
action.get('revisited')
)
break
}
}
return state
}
Expand Down
39 changes: 39 additions & 0 deletions app/common/cache/ledgerVideoCache.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

const Immutable = require('immutable')
const assert = require('assert')

const { makeImmutable, isMap } = require('../state/immutableUtil')

const validateState = function (state) {
state = makeImmutable(state)
assert.ok(isMap(state), 'state must be an Immutable.Map')
assert.ok(isMap(state.getIn(['cache', 'ledgerVideos'])), 'state must contain ledgerVideos as Immutable.Map')
return state
}

const getDataByVideoId = (state, key) => {
state = validateState(state)
if (key == null) {
return Immutable.Map()
}

return state.getIn(['cache', 'ledgerVideos', key]) || Immutable.Map()
}

const setCacheByVideoId = (state, key, data) => {
state = validateState(state)
if (key == null) {
return state
}

data = makeImmutable(data)

return state.setIn(['cache', 'ledgerVideos', key], data)
}

module.exports = {
getDataByVideoId,
setCacheByVideoId
}
9 changes: 9 additions & 0 deletions app/common/constants/ledgerMediaProviders.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

const providers = {
YOUTUBE: 'youtube'
}

module.exports = providers
Loading

0 comments on commit 78252fa

Please sign in to comment.