Skip to content
This repository has been archived by the owner on Jan 17, 2023. It is now read-only.

Commit

Permalink
Fix #1825, make the metrics page available again
Browse files Browse the repository at this point in the history
Fix #1854, implement the metrics queries
Puts into place an essentially new page at /metrics which shows metrics derived from database queries
Adds a table to store those queries so they are not made on demand, instead they are re-run on an hourly schedule
  • Loading branch information
ianb committed Oct 28, 2016
1 parent e4bdd81 commit 89a8d9c
Showing 9 changed files with 236 additions and 345 deletions.
8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
@@ -230,9 +230,9 @@ homepage_dependencies := $(shell ./bin/_bundle_dependencies homepage getdeps "$(
build/server/static/js/homepage-bundle.js: $(homepage_dependencies)
./bin/_bundle_dependencies homepage build ./build/server/pages/homepage/controller.js

admin_dependencies := $(shell ./bin/_bundle_dependencies admin getdeps "$(server_dest)")
build/server/static/js/admin-bundle.js: $(admin_dependencies)
./bin/_bundle_dependencies admin build ./build/server/pages/admin/controller.js
metrics_dependencies := $(shell ./bin/_bundle_dependencies metrics getdeps "$(server_dest)")
build/server/static/js/metrics-bundle.js: $(metrics_dependencies)
./bin/_bundle_dependencies metrics build ./build/server/pages/metrics/controller.js

shotindex_dependencies := $(shell ./bin/_bundle_dependencies shotindex getdeps "$(server_dest)")
build/server/static/js/shotindex-bundle.js: $(shotindex_dependencies)
@@ -257,7 +257,7 @@ build/server/build-time.js: homepage $(server_dest) $(shared_server_dest) $(sass
@mkdir -p $(@D)
./bin/_write_build_time > build/server/build-time.js

server: npm build/server/build-time.js build/server/static/js/shot-bundle.js build/server/static/js/homepage-bundle.js build/server/static/js/admin-bundle.js build/server/static/js/shotindex-bundle.js build/server/static/js/leave-bundle.js build/server/static/js/creating-bundle.js
server: npm build/server/build-time.js build/server/static/js/shot-bundle.js build/server/static/js/homepage-bundle.js build/server/static/js/metrics-bundle.js build/server/static/js/shotindex-bundle.js build/server/static/js/leave-bundle.js build/server/static/js/creating-bundle.js

## Homepage related rules:

4 changes: 4 additions & 0 deletions server/db-patches/patch-10-11.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
CREATE TABLE metrics_cache (
created TIMESTAMP DEFAULT NOW(),
data TEXT
);
1 change: 1 addition & 0 deletions server/db-patches/patch-11-10.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DROP TABLE metrics_cache;
2 changes: 1 addition & 1 deletion server/src/dbschema.js
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ const pgpatcher = require("pg-patcher");
const path = require("path");
const mozlog = require("mozlog")("dbschema");

const MAX_DB_LEVEL = 10;
const MAX_DB_LEVEL = 11;

/** Create all the tables */
exports.createTables = function () {
264 changes: 133 additions & 131 deletions server/src/pages/metrics/model.js
Original file line number Diff line number Diff line change
@@ -1,148 +1,150 @@
const db = require("../../db");

exports.lastShotCount = function (seconds) {
return db.select(
`SELECT COUNT(DISTINCT deviceid) AS shotcount
FROM data
WHERE created >= NOW() - ($1 || ' SECONDS')::INTERVAL`,
[seconds]
).then(function (rows) {
return rows[0].shotcount;
});
};
const queries = {
shotsCreatedByDay: {
title: "Shots By Day",
description: "Number of shots created each day (for the last 30 days)",
sql: `
SELECT COUNT(data.id)::INTEGER AS number_of_shots, day
FROM data,
date_trunc('day', data.created) AS day
WHERE data.created + INTERVAL '30 days' >= CURRENT_TIMESTAMP
AND NOT data.deleted
AND data.expire_time < CURRENT_TIMESTAMP
GROUP BY day
ORDER BY day DESC;
`,
columns: [
{title: "Number of shots", name: "number_of_shots"},
{title: "Day", type: "date", name: "day"}
]
},

exports.numberOfShots = function (seconds) {
return db.select(
`SELECT COUNT(data.id) AS shotcount
FROM data
WHERE created >= NOW() - ($1 || ' SECONDS')::INTERVAL
GROUP BY deviceid`,
[seconds]
).then(function (rows) {
let buckets = {
1: 0,
2: 0,
5: 0,
10: 0,
20: 0,
50: 0,
100: 0,
300: 0,
1000: 0
};
for (let row of rows) {
let c = row.shotcount;
if (c >= 1000) {
buckets[1000]++;
} else if (c >= 300) {
buckets[300]++;
} else if (c >= 100) {
buckets[100]++;
} else if (c >= 50) {
buckets[50]++;
} else if (c >= 20) {
buckets[20]++;
} else if (c >= 10) {
buckets[10]++;
} else if (c >= 5) {
buckets[5]++;
} else if (c >= 2) {
buckets[2]++;
} else if (c >= 1) {
buckets[1]++;
}
}
return buckets;
});
};
usersByDay: {
title: "Users By Day",
description: "Number of users who created at least one shot, by day (last 30 days)",
sql: `
SELECT COUNT(DISTINCT data.deviceid)::INTEGER AS number_of_users, day
FROM data,
date_trunc('day', data.created) AS day
WHERE data.created + INTERVAL '30 days' >= CURRENT_TIMESTAMP
AND NOT data.deleted
AND data.expire_time < CURRENT_TIMESTAMP
GROUP BY day
ORDER BY day DESC;
`,
columns: [
{title: "Number of users", name: "number_of_users"},
{title: "Day", type: "date", name: "day"}
]
},

exports.secondsInDay = 60*60*24;
shotsByUserHistogram: {
title: "Number of Shots per User",
description: "The number of users who have about N total shots",
sql: `
SELECT COUNT(counters.number_of_shots) AS count, counters.number_of_shots AS number_of_shots
FROM
(SELECT FLOOR(POWER(2, FLOOR(LOG(2, COUNT(data.id))))) AS number_of_shots
FROM data
WHERE NOT data.deleted AND data.expire_time < CURRENT_TIMESTAMP
GROUP BY data.deviceid) AS counters
GROUP BY counters.number_of_shots
ORDER BY counters.number_of_shots;
`,
columns: [
{title: "Number of users", name: "count"},
{title: "~Number of shots", name: "number_of_shots"}
]
},

exports.retention = function () {
return db.select(
`SELECT
date_trunc('day', MIN(created)) AS first_created,
date_trunc('day', MAX(created)) AS last_created
FROM data
GROUP BY deviceid`,
[]
).then(function (rows) {
let retentionBuckets = [0, 1, 2, 3, 5, 7, 14, 21, 30, 45, 60, 90, 120, 365, 3650];
let retentionByWeek = {};
let retentionByMonth = {};
let totalRetention = {};
function addToBucket(bucket, days) {
for (let i=retentionBuckets.length; i>=0; i--) {
let bucketDays = retentionBuckets[i];
if (days >= bucketDays) {
if (! bucket[bucketDays]) {
bucket[bucketDays] = 1;
} else {
bucket[bucketDays]++;
}
break;
}
}
}
retention: {
title: "Retention",
description: "Length of time users have been creating shots, grouped by week",
sql: `
SELECT COUNT(age.days)::INTEGER AS user_count, age.days, age.first_created_week
FROM
(SELECT
EXTRACT(EPOCH FROM AGE(span.last_created, span.first_created)) / 3600 AS days,
span.first_created_week
FROM
(SELECT
date_trunc('week', MIN(created)) AS first_created_week,
date_trunc('day', MIN(created)) AS first_created,
date_trunc('day', MAX(created)) AS last_created
FROM data
GROUP BY deviceid) AS span) AS age
GROUP BY age.days, age.first_created_week
ORDER BY age.first_created_week DESC, age.days;
`,
columns: [
{title: "Number of users", name: "user_count"},
{title: "Days the user has been creating shots", name: "days"},
{title: "Week the user started using Page Shot", type: "date", name: "first_created_week"}
]
}

};

function executeQuery(query) {
let start = Date.now();
return db.select(query.sql).then((rows) => {
let result = Object.assign({rows: [], created: Date.now()}, query);
for (let row of rows) {
let first = row.first_created;
let last = row.last_created;
let days = Math.floor((last - first) / (exports.secondsInDay * 1000));
let week = formatDate(truncateDateToWeek(first));
let weekBucket = retentionByWeek[week];
if (! weekBucket) {
weekBucket = retentionByWeek[week] = {};
}
let month = formatDate(truncateDateToMonth(first));
let monthBucket = retentionByMonth[month];
if (! monthBucket) {
monthBucket = retentionByMonth[month] = {};
let l = [];
for (let meta of query.columns) {
let value = row[meta.name];
if (value instanceof Date) {
value = value.getTime();
}
l.push(value);
}
addToBucket(monthBucket, days);
addToBucket(weekBucket, days);
addToBucket(totalRetention, days);
result.rows.push(l);
}
return {
byWeek: retentionByWeek,
byMonth: retentionByMonth,
total: totalRetention
};
result.timeToExecute = Date.now() - start;
return result;
});
};

function truncateDateToWeek(date) {
let millisecondsInDay = exports.secondsInDay * 1000;
let days = date.getDay();
let millisecondValue = date.getTime();
millisecondValue -= millisecondsInDay * days;
return new Date(millisecondValue);
}

function truncateDateToMonth(date) {
let newDate = new Date(date);
newDate.setDate(1);
return newDate;
}
exports.storeQueries = function () {
let allQueries = {};
let promises = [];
for (let name in queries) {
promises.push(executeQuery(queries[name]).then((result) => {
console.log("query result", name);
console.log("out:", result);
allQueries[name] = result;
}));
}
return Promise.all(promises).then(() => {
let body = JSON.stringify(allQueries);
return db.transaction((client) => {
return db.queryWithClient(client, `
DELETE FROM metrics_cache
`).then(() => {
return db.queryWithClient(client, `
INSERT INTO metrics_cache (data) VALUES ($1)
`, [body]);
});
});
});
};

function formatDate(date) {
return date.toISOString().substr(0, 10);
function getQueries() {
return db.select(`
SELECT data FROM metrics_cache ORDER BY created DESC LIMIT 1
`).then((rows) => {
return rows[0].data;
});
}

exports.createModel = function (req) {
let model = {
title: "Page Shot Metrics",
lastShotTimeDays: 7,
numberOfShotsTime: 30,
};
let secs = model.lastShotTimeDays * exports.secondsInDay;
return exports.lastShotCount(secs).then((lastShotCount) => {
model.lastShotCount = lastShotCount;
return exports.numberOfShots(model.numberOfShotsTime * exports.secondsInDay);
}).then((buckets) => {
model.numberOfShotsBuckets = buckets;
return exports.retention();
}).then((retention) => {
model.retention = retention;
return model;
return getQueries().then((data) => {
data = JSON.parse(data);
let model = {
title: "Page Shot Metrics",
data: data
};
return {serverModel: model, jsonModel: model};
});
};
Loading

0 comments on commit 89a8d9c

Please sign in to comment.