From 665b5d6885122d7a95afd8de93e565f577cfc7e6 Mon Sep 17 00:00:00 2001
From: Ceilzcx <1758619238@qq.com>
Date: Tue, 8 Nov 2022 20:28:29 +0800
Subject: [PATCH 01/22] prometheus exporter metrics parser
---
collector/pom.xml | 2 +-
.../http/promethus/ParseException.java | 12 +
.../promethus/exporter/ExporterParser.java | 313 ++++++++++++++++++
.../http/promethus/exporter/MetricFamily.java | 49 +++
.../http/promethus/exporter/MetricType.java | 32 ++
5 files changed, 407 insertions(+), 1 deletion(-)
create mode 100644 collector/src/main/java/com/usthe/collector/collect/http/promethus/ParseException.java
create mode 100644 collector/src/main/java/com/usthe/collector/collect/http/promethus/exporter/ExporterParser.java
create mode 100644 collector/src/main/java/com/usthe/collector/collect/http/promethus/exporter/MetricFamily.java
create mode 100644 collector/src/main/java/com/usthe/collector/collect/http/promethus/exporter/MetricType.java
diff --git a/collector/pom.xml b/collector/pom.xml
index 665e549d46d..b2e91abe774 100644
--- a/collector/pom.xml
+++ b/collector/pom.xml
@@ -47,7 +47,7 @@
com.usthe.tancloud
common
1.0
- provided
+
diff --git a/collector/src/main/java/com/usthe/collector/collect/http/promethus/ParseException.java b/collector/src/main/java/com/usthe/collector/collect/http/promethus/ParseException.java
new file mode 100644
index 00000000000..5b6c6408eb6
--- /dev/null
+++ b/collector/src/main/java/com/usthe/collector/collect/http/promethus/ParseException.java
@@ -0,0 +1,12 @@
+package com.usthe.collector.collect.http.promethus;
+
+/**
+ * @author ceilzcx
+ * @since 7/11/2022
+ */
+public class ParseException extends RuntimeException {
+
+ public ParseException(String msg) {
+ super(msg);
+ }
+}
diff --git a/collector/src/main/java/com/usthe/collector/collect/http/promethus/exporter/ExporterParser.java b/collector/src/main/java/com/usthe/collector/collect/http/promethus/exporter/ExporterParser.java
new file mode 100644
index 00000000000..dc554d9c546
--- /dev/null
+++ b/collector/src/main/java/com/usthe/collector/collect/http/promethus/exporter/ExporterParser.java
@@ -0,0 +1,313 @@
+package com.usthe.collector.collect.http.promethus.exporter;
+
+import com.usthe.collector.collect.http.promethus.ParseException;
+import com.usthe.common.util.Pair;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author ceilzcx
+ * @since 7/11/2022
+ * 参考: prometheus的text_parse.go的代码, 入口: TextToMetricFamilies
+ * todo 使用String.subString()的方式会生成很多的对象,后续改为buffer
+ */
+@Slf4j
+public class ExporterParser {
+ private static final String HELP = "HELP";
+ private static final String TYPE = "TYPE";
+
+ private static final String QUANTILE_LABEL = "quantile";
+ private static final String BUCKET_LABEL = "le";
+
+ public Map textToMetric(String resp) {
+ // key: metric name, value: metric family
+ Map metricMap = new ConcurrentHashMap<>();
+ String[] lines = resp.split("\n");
+ for (String line : lines) {
+ this.parseLine(metricMap, line);
+ }
+ return metricMap;
+ }
+
+ private void parseLine(Map metricMap, String line) {
+ line = this.skipBlankTab(line);
+ if (line == null) return;
+ switch (line.charAt(0)) {
+ case '#':
+ this.parseComment(metricMap, line.substring(1));
+ break;
+ case '\n':
+ break;
+ default:
+ this.parseMetric(metricMap, line);
+ }
+ }
+
+ private void parseComment(Map metricMap, String line) {
+ line = this.skipBlankTab(line);
+ if (line == null) return;
+ Pair pair = this.readTokenUnitWhitespace(line);
+ String token = pair.getLeft();
+ String subLine = pair.getRight();
+ if (!token.equals(HELP) && !token.equals(TYPE)) {
+ // handle
+ return;
+ }
+ pair = this.readTokenAsMetricName(subLine);
+ String metricName = pair.getLeft();
+ MetricFamily metricFamily = metricMap.computeIfAbsent(metricName, key -> new MetricFamily());
+ metricFamily.setName(metricName);
+ subLine = pair.getRight();
+ switch (token) {
+ case HELP:
+ this.parseHelp(metricFamily, subLine);
+ break;
+ case TYPE:
+ this.parseType(metricFamily, subLine);
+ break;
+ default:
+ }
+ }
+
+ private void parseHelp(MetricFamily metricFamily, String line) {
+ metricFamily.setHelp(this.skipBlankTab(line));
+ }
+
+ private void parseType(MetricFamily metricFamily, String line) {
+ String type = this.skipBlankTab(line).toLowerCase();
+ MetricType metricType = MetricType.getType(type);
+ if (metricType == null) {
+ throw new ParseException("pare type error");
+ }
+ metricFamily.setMetricType(metricType);
+ }
+
+ private void parseMetric(Map metricMap, String line) {
+ Pair pair = this.readTokenAsMetricName(line);
+ String metricName = pair.getLeft();
+ String subLine = pair.getRight();
+ MetricFamily metricFamily = metricMap.get(metricName);
+ if (metricFamily == null) {
+ log.error("line {} parse error, no such HELP and TYPE", line);
+ return;
+ }
+ List metricList = metricFamily.getMetricList();
+ MetricFamily.Metric metric = new MetricFamily.Metric();
+ this.readLabels(metricFamily, metric, subLine);
+ if (metricList == null) {
+ metricList = new ArrayList<>();
+ metricFamily.setMetricList(metricList);
+ }
+ metricList.add(metric);
+ }
+
+ private void readLabels(MetricFamily metricFamily, MetricFamily.Metric metric, String line) {
+ if (metricFamily.getMetricType().equals(MetricType.SUMMARY) || metricFamily.getMetricType().equals(MetricType.HISTOGRAM)) {
+ // need handle
+ }
+ metric.setLabelPair(new ArrayList<>());
+ if (line.charAt(0) == '{') {
+ this.startReadLabelName(metricFamily, metric, line.substring(1));
+ } else {
+ this.readLabelValue(metricFamily, metric, new MetricFamily.Label(), line);
+ }
+ }
+
+ private void startReadLabelName(MetricFamily metricFamily, MetricFamily.Metric metric, String line) {
+ String subLine = this.skipBlankTab(line);
+ if (line == null) return;
+ if (line.charAt(0) == '}') {
+ subLine = this.skipBlankTab(line.substring(1));
+ if (subLine.isEmpty()) return;
+ this.startReadLabelValue(metricFamily, metric, new MetricFamily.Label(), subLine);
+ return;
+ }
+ Pair pair = this.readTokenAsLabelName(line);
+ String labelName = pair.getLeft();
+ subLine = pair.getRight();
+ if (labelName == null || labelName.isEmpty() || labelName.equals("__name__")) {
+ throw new ParseException("invalid label name, label name size = 0 or label name equals __name__");
+ }
+ MetricFamily.Label label = new MetricFamily.Label();
+ if (!(metricFamily.getMetricType().equals(MetricType.SUMMARY) && labelName.equals(QUANTILE_LABEL))
+ && !(metricFamily.getMetricType().equals(MetricType.HISTOGRAM) && labelName.equals(BUCKET_LABEL))) {
+ label.setName(labelName);
+ }
+ if (subLine.charAt(0) != '=') {
+ throw new ParseException("parse error, not match the format of labelName=labelValue");
+ }
+ this.startReadLabelValue(metricFamily, metric, label, subLine.substring(1));
+ }
+
+ private void startReadLabelValue(MetricFamily metricFamily, MetricFamily.Metric metric, MetricFamily.Label label, String line) {
+ String subLine = this.skipBlankTab(line);
+ if (subLine == null) return;
+ if (subLine.charAt(0) != '"') {
+ throw new ParseException("expected '\"' at start of label value");
+ }
+ Pair pair = this.readTokenAsLabelValue(line.substring(1));
+ subLine = this.skipBlankTab(pair.getRight());
+ label.setValue(pair.getLeft());
+ metric.getLabelPair().add(label);
+ // todo add method: judge label value is valid
+ if (!subLine.isEmpty()) {
+ switch (subLine.charAt(0)) {
+ case ',':
+ this.startReadLabelName(metricFamily, metric, subLine);
+ break;
+ case '}':
+ return;
+ default:
+ throw new ParseException("expected '}' or ',' at end of label value");
+ }
+ }
+ }
+
+ private void readLabelValue(MetricFamily metricFamily, MetricFamily.Metric metric, MetricFamily.Label label, String line) {
+ label.setValue(line);
+ metric.getLabelPair().add(label);
+ // todo handle by metric type
+ }
+
+ /**
+ * 读取第一个空格符前的token
+ * @return Pair
+ */
+ private Pair readTokenUnitWhitespace(String s) {
+ int whitespaceIndex = s.indexOf(" ");
+ if (whitespaceIndex < 0 || whitespaceIndex > s.length()) {
+ return Pair.of(s, null);
+ }
+ return Pair.of(s.substring(0, whitespaceIndex).trim(), s.substring(whitespaceIndex).trim());
+ }
+
+ /**
+ * 获取指标的名称
+ * @return Pair
+ */
+ private Pair readTokenAsMetricName(String s) {
+ if (this.isValidMetricNameStart(s.charAt(0))) {
+ int i;
+ for (i = 1; i < s.length(); i++) {
+ if (!this.isValidMetricNameContinuation(s.charAt(i))) {
+ break;
+ }
+ }
+ return Pair.of(s.substring(0, i), s.substring(i));
+ }
+ throw new ParseException("parse metric name error");
+ }
+
+ /**
+ * 获取Label的名称
+ * @return Pair
+ */
+ private Pair readTokenAsLabelName(String s) {
+ if (this.isValidLabelNameStart(s.charAt(0))) {
+ int i;
+ for(i = 1; i < s.length(); i++) {
+ if (!this.isValidLabelNameContinuation(s.charAt(i))) {
+ break;
+ }
+ }
+ return Pair.of(s.substring(0, i), s.substring(i));
+ }
+ throw new ParseException("parse label name error");
+ }
+
+ /**
+ * 获取Label的值
+ * @return Pair
+ */
+ private Pair readTokenAsLabelValue(String s) {
+ StringBuilder builder = new StringBuilder();
+ boolean escaped = false;
+ for (int i = 0; i < s.length(); i++) {
+ char c = s.charAt(i);
+ // 处理 '\\' 转义
+ if (escaped) {
+ switch (c) {
+ case '"':
+ case '\\':
+ builder.append(c);
+ break;
+ case 'n':
+ builder.append('\n');
+ break;
+ default:
+ throw new ParseException("parse label value error");
+ }
+ } else {
+ switch (c) {
+ case '"':
+ return Pair.of(builder.toString(), s.substring(i + 1));
+ case '\n':
+ throw new ParseException("parse label value error, next line");
+ case '\\':
+ escaped = true;
+ break;
+ default:
+ builder.append(c);
+ }
+ }
+ }
+ return Pair.of(builder.toString(), "");
+ }
+
+ // 清除字符两边空格
+ private String skipBlankTab(String s) {
+ if (s == null) {
+ return null;
+ }
+ s = s.trim();
+ return s.isEmpty() ? null : s;
+ }
+
+ // 是否符合metric name首字符规则
+ private boolean isValidMetricNameStart(char c) {
+ return isValidLabelNameStart(c) || c == ':';
+ }
+
+ // 是否符合metric name除首字符其他字符规则
+ private boolean isValidMetricNameContinuation(char c) {
+ return isValidLabelNameContinuation(c) || c == ':';
+ }
+
+ // 是否符合label name首字符规则
+ private boolean isValidLabelNameStart(char c) {
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_';
+ }
+
+ // 是否符合label name除首字符其他字符规则
+ private boolean isValidLabelNameContinuation(char c) {
+ return isValidLabelNameStart(c) || (c >= '0' && c <= '9');
+ }
+
+ public static void main(String[] args) {
+ String resp = "# HELP go_gc_cycles_automatic_gc_cycles_total Count of completed GC cycles generated by the Go runtime.\n" +
+ "# TYPE go_gc_cycles_automatic_gc_cycles_total counter\n" +
+ "go_gc_cycles_automatic_gc_cycles_total 0\n" +
+ "# HELP go_gc_cycles_forced_gc_cycles_total Count of completed GC cycles forced by the application.\n" +
+ "# TYPE go_gc_cycles_forced_gc_cycles_total counter\n" +
+ "go_gc_cycles_forced_gc_cycles_total 0\n" +
+ "# HELP go_gc_cycles_total_gc_cycles_total Count of all completed GC cycles.\n" +
+ "# TYPE go_gc_cycles_total_gc_cycles_total counter\n" +
+ "go_gc_cycles_total_gc_cycles_total 0\n" +
+ "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n" +
+ "# TYPE go_gc_duration_seconds summary\n" +
+ "go_gc_duration_seconds{quantile=\"0\"} 0\n" +
+ "go_gc_duration_seconds{quantile=\"0.25\"} 0\n" +
+ "go_gc_duration_seconds{quantile=\"0.5\"} 0\n" +
+ "go_gc_duration_seconds{quantile=\"0.75\"} 0\n" +
+ "go_gc_duration_seconds{quantile=\"1\"} 0\n";
+ ExporterParser parser = new ExporterParser();
+ Map metricFamilyMap = parser.textToMetric(resp);
+ for (MetricFamily metricFamily : metricFamilyMap.values()) {
+ System.out.println(metricFamily);
+ }
+ }
+}
diff --git a/collector/src/main/java/com/usthe/collector/collect/http/promethus/exporter/MetricFamily.java b/collector/src/main/java/com/usthe/collector/collect/http/promethus/exporter/MetricFamily.java
new file mode 100644
index 00000000000..ecb110741b5
--- /dev/null
+++ b/collector/src/main/java/com/usthe/collector/collect/http/promethus/exporter/MetricFamily.java
@@ -0,0 +1,49 @@
+package com.usthe.collector.collect.http.promethus.exporter;
+
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+/**
+ * @author ceilzcx
+ * @since 7/11/2022
+ */
+@Data
+@ToString
+public class MetricFamily {
+
+ private String name;
+
+ private String help;
+
+ private MetricType metricType;
+
+ private List metricList;
+
+ @Data
+ static class Metric {
+
+ private List