Skip to content

Commit

Permalink
Robust server that handles errors within metric sql results processing
Browse files Browse the repository at this point in the history 
If a metric sql returns 0 rows, handle it gracefully instead of letting the collector fail
added metric entry naming using associative array instead of simple array
start:verbose now correctly shows queries (typo)
UP metric is now a standard metric (using SELECT 1 query)
change metrics functions to new arrow format
Fix imports
  • Loading branch information
@awaragi
awaragi committed Apr 23, 2022
1 parent 7ff3b4c commit 56a3de2
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 54 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

Prometheus exporter for Microsoft SQL Server (MSSQL). Exposes the following metrics

- mssql_up UP Status
- mssql_product_version Instance version (Major.Minor)
- mssql_instance_local_time Number of seconds since epoch on local instance
- mssql_connections{database,state} Number of active connections
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"docker:run": "export DOCKERID=$(docker build -q .) && docker run --name prometheus-mssql-exporter --rm -it -p 4000:4000 -e SERVER=$(docker inspect mssql | jq -r '.[].NetworkSettings.Networks.bridge.IPAddress') -e USERNAME=SA -e PASSWORD=qkD4x3yy -e DEBUG=app,db,metrics $DOCKERID ; docker image rm $DOCKERID",
"docker:run:published": "export DOCKERID=awaragi/prometheus-mssql-exporter && docker run --name prometheus-mssql-exporter --rm -it -p 4000:4000 -e SERVER=$(docker inspect mssql | jq -r '.[].NetworkSettings.Networks.bridge.IPAddress') -e USERNAME=SA -e PASSWORD=qkD4x3yy -e DEBUG=app,db,metrics $DOCKERID ; docker image rm $DOCKERID",
"start": "DEBUG=app,db,metrics SERVER=localhost USERNAME=SA PASSWORD=qkD4x3yy node src/index.js",
"start:verbose": "DEBUG=app,db,metrics,metrics SERVER=localhost USERNAME=SA PASSWORD=qkD4x3yy node src/index.js",
"start:verbose": "DEBUG=app,db,metrics,queries SERVER=localhost USERNAME=SA PASSWORD=qkD4x3yy node src/index.js",
"test:mssql:2019": "docker run --name mssql --rm -e ACCEPT_EULA=Y -e SA_PASSWORD=qkD4x3yy -p 1433:1433 --name mssql mcr.microsoft.com/mssql/server:2019-latest",
"test:mssql:2017": "docker run --name mssql --rm -e ACCEPT_EULA=Y -e SA_PASSWORD=qkD4x3yy -p 1433:1433 --name mssql mcr.microsoft.com/mssql/server:2017-latest",
"test:fetch": "curl http://localhost:4000/metrics",
Expand Down
40 changes: 24 additions & 16 deletions src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@ const queriesLog = require("debug")("queries");
const Connection = require("tedious").Connection;
const Request = require("tedious").Request;
const app = require("express")();
const client = require("prom-client");

const client = require("./metrics").client;
const mssql_up = require("./metrics").mssql_up;
const metrics = require("./metrics").metrics;
const { entries } = require("./metrics");

let config = {
connect: {
Expand Down Expand Up @@ -41,7 +40,7 @@ if (!config.connect.authentication.options.password) {
}

/**
* Connects to a database server and if successful starts the metrics collection interval.
* Connects to a database server.
*
* @returns Promise<Connection>
*/
Expand Down Expand Up @@ -70,19 +69,29 @@ async function connect() {
*
* @param connection database connection
* @param collector single metric: {query: string, collect: function(rows, metric)}
* @param name name of collector variable
*
* @returns Promise of collect operation (no value returned)
*/
async function measure(connection, collector) {
async function measure(connection, collector, name) {
return new Promise((resolve) => {
queriesLog(`Executing query: ${collector.query}`);
queriesLog(`Executing metric ${name} query: ${collector.query}`);
let request = new Request(collector.query, (error, rowCount, rows) => {
if (!error) {
queriesLog(`Retrieved rows ${JSON.stringify(rows, null, 2)}`);
collector.collect(rows, collector.metrics);
queriesLog(`Retrieved metric ${name} rows (${rows.length}): ${JSON.stringify(rows, null, 2)}`);

if (rows.length > 0) {
try {
collector.collect(rows, collector.metrics);
} catch (error) {
console.error(`Error processing metric ${name} data`, collector.query, JSON.stringify(rows), error);
}
} else {
console.error(`Query for metric ${name} returned 0 rows to process`, collector.query);
}
resolve();
} else {
console.error("Error executing SQL query", collector.query, error);
console.error(`Error executing metric ${name} SQL query`, collector.query, error);
resolve();
}
});
Expand All @@ -98,9 +107,8 @@ async function measure(connection, collector) {
* @returns Promise of execution (no value returned)
*/
async function collect(connection) {
mssql_up.set(1);
for (let i = 0; i < metrics.length; i++) {
await measure(connection, metrics[i]);
for (const [metricName, metric] of Object.entries(entries)) {
await measure(connection, metric, metricName);
}
}

Expand All @@ -112,15 +120,15 @@ app.get("/metrics", async (req, res) => {
res.contentType(client.register.contentType);

try {
appLog("Received metrics request");
appLog("Received /metrics request");
let connection = await connect();
await collect(connection, metrics);
await collect(connection);
connection.close();
res.send(client.register.metrics());
appLog("Successfully processed metrics request");
appLog("Successfully processed /metrics request");
} catch (error) {
// error connecting
appLog("Error handling metrics request");
appLog("Error handling /metrics request");
mssql_up.set(0);
res.header("X-Error", error.message || error);
res.send(client.register.getSingleMetricAsString(mssql_up.name));
Expand Down
25 changes: 15 additions & 10 deletions src/metrics-docs.js
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,26 @@
const { metrics } = require("./metrics");
const { entries } = require("./metrics");

// DOCUMENTATION of queries and their associated metrics (targeted to DBAs)
metrics.forEach(function (m) {
for (let key in m.metrics) {
if (m.metrics.hasOwnProperty(key)) {
console.log("--", m.metrics[key].name, m.metrics[key].help);
Object.entries(entries).forEach(([entryName, entry]) => {
console.log("--[", entryName, "]");
for (let key in entry.metrics) {
if (entry.metrics.hasOwnProperty(key)) {
console.log("--", entry.metrics[key].name, entry.metrics[key].help);
}
}
console.log(m.query + ";");
console.log(entry.query + ";");
console.log("");
});

console.log("/*");
metrics.forEach(function (m) {
for (let key in m.metrics) {
if (m.metrics.hasOwnProperty(key)) {
console.log("- ", m.metrics[key].name + (m.metrics[key].labelNames.length > 0 ? "{" + m.metrics[key].labelNames + "}" : ""), m.metrics[key].help);
Object.values(entries).forEach((entry) => {
for (let key in entry.metrics) {
if (entry.metrics.hasOwnProperty(key)) {
console.log(
"-",
entry.metrics[key].name + (entry.metrics[key].labelNames.length > 0 ? "{" + entry.metrics[key].labelNames + "}" : ""),
entry.metrics[key].help
);
}
}
});
Expand Down
59 changes: 32 additions & 27 deletions src/metrics.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,26 @@ const metricsLog = require("debug")("metrics");
const client = require("prom-client");
const { productVersionParse } = require("./utils");

// UP metric
const mssql_up = new client.Gauge({ name: "mssql_up", help: "UP Status" });
const mssql_up = {
metrics: {
mssql_up: new client.Gauge({ name: "mssql_up", help: "UP Status" }),
},
query: "SELECT 1",
collect: (rows, metrics) => {
let mssql_up = rows[0][0].value;
metricsLog("Fetched status of instance", mssql_up);
metrics.mssql_up.set(mssql_up);
},
};

// Query based metrics
// -------------------
const mssql_product_version = {
metrics: {
mssql_product_version: new client.Gauge({ name: "mssql_product_version", help: "Instance version (Major.Minor)" }),
},
query: `SELECT CONVERT(VARCHAR(128), SERVERPROPERTY ('productversion')) AS ProductVersion,
SERVERPROPERTY('ProductVersion') AS ProductVersion
`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
let v = productVersionParse(rows[0][0].value);
const mssql_product_version = v.major + v.minor / 10;
metricsLog("Fetched version of instance", mssql_product_version);
Expand All @@ -31,7 +38,7 @@ const mssql_instance_local_time = {
mssql_instance_local_time: new client.Gauge({ name: "mssql_instance_local_time", help: "Number of seconds since epoch on local instance" }),
},
query: `SELECT DATEDIFF(second, '19700101', GETUTCDATE())`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
const mssql_instance_local_time = rows[0][0].value;
metricsLog("Fetched current time", mssql_instance_local_time);
metrics.mssql_instance_local_time.set(mssql_instance_local_time);
Expand All @@ -46,7 +53,7 @@ const mssql_connections = {
, COUNT(sP.spid)
FROM sys.sysprocesses sP
GROUP BY DB_NAME(sP.dbid)`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const database = row[0].value;
Expand All @@ -69,7 +76,7 @@ const mssql_client_connections = {
FROM sys.dm_exec_sessions
WHERE is_user_process=1
GROUP BY host_name, database_id`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const client = row[0].value;
Expand All @@ -91,7 +98,7 @@ const mssql_deadlocks = {
query: `SELECT cntr_value
FROM sys.dm_os_performance_counters
WHERE counter_name = 'Number of Deadlocks/sec' AND instance_name = '_Total'`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
const mssql_deadlocks = rows[0][0].value;
metricsLog("Fetched number of deadlocks/sec", mssql_deadlocks);
metrics.mssql_deadlocks_per_second.set(mssql_deadlocks);
Expand All @@ -105,7 +112,7 @@ const mssql_user_errors = {
query: `SELECT cntr_value
FROM sys.dm_os_performance_counters
WHERE counter_name = 'Errors/sec' AND instance_name = 'User Errors'`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
const mssql_user_errors = rows[0][0].value;
metricsLog("Fetched number of user errors/sec", mssql_user_errors);
metrics.mssql_user_errors.set(mssql_user_errors);
Expand All @@ -119,7 +126,7 @@ const mssql_kill_connection_errors = {
query: `SELECT cntr_value
FROM sys.dm_os_performance_counters
WHERE counter_name = 'Errors/sec' AND instance_name = 'Kill Connection Errors'`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
const mssql_kill_connection_errors = rows[0][0].value;
metricsLog("Fetched number of kill connection errors/sec", mssql_kill_connection_errors);
metrics.mssql_kill_connection_errors.set(mssql_kill_connection_errors);
Expand All @@ -135,7 +142,7 @@ const mssql_database_state = {
}),
},
query: `SELECT name,state FROM master.sys.databases`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const database = row[0].value;
Expand All @@ -157,7 +164,7 @@ const mssql_log_growths = {
query: `SELECT rtrim(instance_name), cntr_value
FROM sys.dm_os_performance_counters
WHERE counter_name = 'Log Growths' and instance_name <> '_Total'`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const database = row[0].value;
Expand All @@ -177,7 +184,7 @@ const mssql_database_filesize = {
}),
},
query: `SELECT DB_NAME(database_id) AS database_name, name AS logical_name, type, physical_name, (size * CAST(8 AS BIGINT)) size_kb FROM sys.master_files`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const database = row[0].value;
Expand Down Expand Up @@ -213,8 +220,7 @@ const mssql_buffer_manager = {
mssql_lazy_write_total: new client.Gauge({ name: "mssql_lazy_write_total", help: "Lazy writes/sec" }),
mssql_page_checkpoint_total: new client.Gauge({ name: "mssql_page_checkpoint_total", help: "Checkpoint pages/sec" }),
},
query: `
SELECT * FROM
query: `SELECT * FROM
(
SELECT rtrim(counter_name) as counter_name, cntr_value
FROM sys.dm_os_performance_counters
Expand All @@ -227,7 +233,7 @@ const mssql_buffer_manager = {
FOR counter_name IN ([Page reads/sec], [Page writes/sec], [Page life expectancy], [Lazy writes/sec], [Checkpoint pages/sec])
) piv
`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
const row = rows[0];
const page_read = row[0].value;
const page_write = row[1].value;
Expand Down Expand Up @@ -272,7 +278,7 @@ FROM
sys.dm_io_virtual_file_stats(null, null) a
INNER JOIN sys.master_files b ON a.database_id = b.database_id and a.file_id = b.file_id
GROUP BY a.database_id`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const database = row[0].value;
Expand Down Expand Up @@ -301,7 +307,7 @@ const mssql_batch_requests = {
query: `SELECT TOP 1 cntr_value
FROM sys.dm_os_performance_counters
WHERE counter_name = 'Batch Requests/sec'`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const mssql_batch_requests = row[0].value;
Expand All @@ -322,7 +328,7 @@ const mssql_transactions = {
query: `SELECT rtrim(instance_name), cntr_value
FROM sys.dm_os_performance_counters
WHERE counter_name = 'Transactions/sec' AND instance_name <> '_Total'`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const database = row[0].value;
Expand All @@ -340,7 +346,7 @@ const mssql_os_process_memory = {
},
query: `SELECT page_fault_count, memory_utilization_percentage
FROM sys.dm_os_process_memory`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
const page_fault_count = rows[0][0].value;
const memory_utilization_percentage = rows[0][1].value;
metricsLog("Fetched page fault count", page_fault_count);
Expand All @@ -358,7 +364,7 @@ const mssql_os_sys_memory = {
},
query: `SELECT total_physical_memory_kb, available_physical_memory_kb, total_page_file_kb, available_page_file_kb
FROM sys.dm_os_sys_memory`,
collect: function (rows, metrics) {
collect: (rows, metrics) => {
const mssql_total_physical_memory_kb = rows[0][0].value;
const mssql_available_physical_memory_kb = rows[0][1].value;
const mssql_total_page_file_kb = rows[0][2].value;
Expand All @@ -381,7 +387,8 @@ FROM sys.dm_os_sys_memory`,
},
};

const metrics = [
const entries = {
mssql_up,
mssql_product_version,
mssql_instance_local_time,
mssql_connections,
Expand All @@ -398,10 +405,8 @@ const metrics = [
mssql_transactions,
mssql_os_process_memory,
mssql_os_sys_memory,
];
};

module.exports = {
client: client,
mssql_up,
metrics: metrics,
entries,
};

0 comments on commit 56a3de2

Please sign in to comment.