Release 0.4.1 (#44)

* fix(api): Fix the issue of statically defined file in the /export API (#35) Signed-off-by: hayk96 <hayko5999@gmail.com> * feature(api): Add support of exporting files in multiple formats (#37) Signed-off-by: hayk96 <hayko5999@gmail.com> * feature(api): Add support of replacing fields in /export API (#39) Signed-off-by: hayk96 <hayko5999@gmail.com> * feature(api): Add support of converting time formats in /export API (#41) Signed-off-by: hayk96 <hayko5999@gmail.com> * feature(ui): Add Reporting page web UI (#43) * feature(ui): Add Reporting page web UI Signed-off-by: hayk96 <hayko5999@gmail.com> * fix(api): Fix exception handling of /export API Signed-off-by: hayk96 <hayko5999@gmail.com> --------- Signed-off-by: hayk96 <hayko5999@gmail.com> * docs: Update CHANGELOG.md Signed-off-by: hayk96 <hayko5999@gmail.com> * chore(api): Bump app version #minor Signed-off-by: hayk96 <hayko5999@gmail.com> * fix(api): Fix async function name #minor Signed-off-by: hayk96 <hayko5999@gmail.com> --------- Signed-off-by: hayk96 <hayko5999@gmail.com>
hayk96 · Jun 30, 2024 · 16a5a82 · 16a5a82
1 parent c534f22
commit 16a5a82
Show file tree

Hide file tree

Showing 13 changed files with 631 additions and 24 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.4.1 / 2024-06-30
+
+* [ENHANCEMENT] Added a new web page for reports. This page allows exporting Prometheus data in various formats directly from the web UI. #43
+* [ENHANCEMENT] Added functionality to change the timestamp format while exporting data via the /export API. Previously, the default value was Unix timestamp. Now, you can choose from the following options: iso8601, rfc2822, rfc3339, friendly, and unix (default). #41
+* [ENHANCEMENT] Added a new feature that allows replacing Prometheus labels (fields) in the final dataset: CSV, JSON, etc. #39
+* [ENHANCEMENT] Added support for exporting files in multiple formats via the /export API. Supported formats include: CSV, YAML (or YML), JSON, and JSON Lines (or NDJSON). E.g., ?format=csv|yaml|yml|json|ndjson|jsonlines. #37
+* [ENHANCEMENT] Improved the functionality that generates CSV files to ensure they have unique names instead of static names, resolving issues with responses getting mixed up between users. #35
+* [BUGFIX] Fixed exception handling for replace_fields in the /export API. #43
+
 ## 0.4.0 / 2024-06-23
 
 * [ENHANCEMENT] Added a new API endpoint: `/export` for exporting data from Prometheus as a CSV file. This feature allows users to export data from Prometheus easily. 

diff --git a/src/api/v1/endpoints/export.py b/src/api/v1/endpoints/export.py
@@ -64,12 +64,15 @@ async def export(
             Body(
                 openapi_examples=ExportData._request_body_examples,
             )
-        ]
+        ],
+        format: str = "csv"
 ):
     data = data.dict()
-    filename = "data.csv"
     expr, start = data.get("expr"), data.get("start")
     end, step = data.get("end"), data.get("step")
+    file, file_format = None, format.lower()
+    custom_fields, timestamp_format = data.get(
+        "replace_fields"), data.get("timestamp_format")
     validation_status, response.status_code, sts, msg = exp.validate_request(
         "export.json", data)
     if validation_status:
@@ -79,9 +82,10 @@ async def export(
             query=expr, start=start,
             end=end, step=step)
         if resp_status:
-            labels, data_processed = exp.data_processor(source_data=resp_data)
-            csv_generator_status, sts, msg = exp.csv_generator(
-                data=data_processed, fields=labels, filename=filename)
+            labels, data_processed = exp.data_processor(
+                source_data=resp_data, custom_fields=custom_fields, timestamp_format=timestamp_format)
+            file_generator_status, sts, response.status_code, file, msg = exp.file_generator(
+                file_format=file_format, data=data_processed, fields=labels)
         else:
             sts, msg = resp_data.get("status"), resp_data.get("error")
 
@@ -91,8 +95,8 @@ async def export(
             "status": response.status_code,
             "query": expr,
             "method": request.method,
-            "request_path": request.url.path})
+            "request_path": f"{request.url.path}{'?' + request.url.query if request.url.query else ''}"})
     if sts == "success":
-        return FileResponse(path=filename,
-                            background=BackgroundTask(exp.cleanup_files, filename))
+        return FileResponse(path=file,
+                            background=BackgroundTask(exp.cleanup_files, file))
     return {"status": sts, "query": expr, "message": msg}
diff --git a/src/api/v1/endpoints/web.py b/src/api/v1/endpoints/web.py
@@ -10,6 +10,7 @@
 if arg_parser().get("web.enable_ui") == "true":
     rules_management = "ui/rules-management"
     metrics_management = "ui/metrics-management"
+    reports = "ui/reports"
     logger.info("Starting web management UI")
 
     @router.get("/", response_class=HTMLResponse,
@@ -41,7 +42,7 @@ async def rules_management_files(path, request: Request):
         return f"{sts} {msg}"
 
     @router.get("/metrics-management",
-                description="RRenders metrics management HTML page of this application",
+                description="Renders metrics management HTML page of this application",
                 include_in_schema=False)
     async def metrics_management_page():
         return FileResponse(f"{metrics_management}/index.html")
@@ -61,3 +62,25 @@ async def metrics_management_files(path, request: Request):
                 "method": request.method,
                 "request_path": request.url.path})
         return f"{sts} {msg}"
+
+    @router.get("/reports",
+                description="Renders Reports HTML page of this application",
+                include_in_schema=False)
+    async def reports_page():
+        return FileResponse(f"{reports}/index.html")
+
+    @router.get(
+        "/reports/{path}",
+        description="Returns JavaScript and CSS files of the Reports",
+        include_in_schema=False)
+    async def reports_files(path, request: Request):
+        if path in ["script.js", "style.css"]:
+            return FileResponse(f"{reports}/{path}")
+        sts, msg = "404", "Not Found"
+        logger.info(
+            msg=msg,
+            extra={
+                "status": sts,
+                "method": request.method,
+                "request_path": request.url.path})
+        return f"{sts} {msg}"
diff --git a/src/core/export.py b/src/core/export.py
@@ -1,7 +1,11 @@
 from jsonschema import validate, exceptions
 from src.utils.arguments import arg_parser
+from email.utils import formatdate
+from datetime import datetime
+from uuid import uuid4
 import requests
 import json
+import yaml
 import copy
 import csv
 import os
@@ -31,7 +35,41 @@ def prom_query(query, range_query=False, start="0", end="0",
         return True if r.status_code == 200 else False, r.status_code, r.json()
 
 
-def data_processor(source_data: dict) -> tuple[list, list]:
+def replace_fields(data, custom_fields) -> None:
+    """
+    This function replaces (renames) the
+    final Prometheus labels (fields) based
+    on the 'replace_fields' object.
+    """
+    for source_field, target_field in custom_fields.items():
+        try:
+            if isinstance(data, list):
+                data[data.index(source_field)] = target_field
+            elif isinstance(data, dict):
+                data[target_field] = data.pop(source_field)
+        except (ValueError, KeyError):
+            pass
+
+
+def format_timestamp(timestamp, fmt) -> str:
+    """
+    This function converts Unix timestamps
+    to several common time formats.
+    """
+    timestamp_formats = {
+        "unix": timestamp,
+        "rfc2822": formatdate(timestamp, localtime=True),
+        "iso8601": datetime.fromtimestamp(timestamp).isoformat(),
+        "rfc3339": datetime.fromtimestamp(timestamp).astimezone().isoformat(timespec='milliseconds'),
+        "friendly": datetime.fromtimestamp(timestamp).strftime('%A, %B %d, %Y %I:%M:%S %p')
+    }
+
+    return timestamp_formats[fmt]
+
+
+def data_processor(source_data: dict,
+                   custom_fields: dict,
+                   timestamp_format: str) -> tuple[list, list]:
     """
     This function preprocesses the results
     of the Prometheus query for future formatting.
@@ -47,8 +85,10 @@ def vector_processor():
             ts_labels = set(ts["metric"].keys())
             unique_labels.update(ts_labels)
             series = ts["metric"]
-            series["timestamp"] = ts["value"][0]
+            series["timestamp"] = format_timestamp(
+                ts["value"][0], timestamp_format)
             series["value"] = ts["value"][1]
+            replace_fields(series, custom_fields)
             data_processed.append(series)
 
     def matrix_processor():
@@ -58,8 +98,10 @@ def matrix_processor():
             series = ts["metric"]
             for idx in range(len(ts["values"])):
                 series_nested = copy.deepcopy(series)
-                series_nested["timestamp"] = ts["values"][idx][0]
+                series_nested["timestamp"] = format_timestamp(
+                    ts["values"][idx][0], timestamp_format)
                 series_nested["value"] = ts["values"][idx][1]
+                replace_fields(series_nested, custom_fields)
                 data_processed.append(series_nested)
                 del series_nested
 
@@ -70,6 +112,7 @@ def matrix_processor():
 
     unique_labels = sorted(unique_labels)
     unique_labels.extend(["timestamp", "value"])
+    replace_fields(unique_labels, custom_fields)
     return unique_labels, data_processed
 
 
@@ -102,18 +145,31 @@ def cleanup_files(file) -> tuple[True, str]:
         return True, "File has been removed successfully"
 
 
-def csv_generator(data, fields, filename) -> tuple[bool, str, str]:
+def file_generator(file_format, data, fields):
     """
-    This function generates a CSV file
-    based on the provided objects.
+    This function generates a file depending
+    on the provided file format/extension
     """
+
+    file_path = f"/tmp/{str(uuid4())}.{file_format}"
     try:
-        with open(filename, 'w') as csvfile:
-            writer = csv.DictWriter(
-                csvfile, fieldnames=fields, extrasaction='ignore')
-            writer.writeheader()
-            writer.writerows(data)
+        with open(file_path, 'w') as f:
+            if file_format == "csv":
+                writer = csv.DictWriter(
+                    f, fieldnames=fields, extrasaction='ignore')
+                writer.writeheader()
+                writer.writerows(data)
+            elif file_format in ["yml", "yaml"]:
+                f.write(yaml.dump(data))
+            elif file_format == "json":
+                f.write(json.dumps(data))
+            elif file_format in ["ndjson", "jsonlines"]:
+                for i in data:
+                    f.write(f"{json.dumps(i)}\n")
+            else:
+                cleanup_files(file_path)
+                return False, "error", 400, "", f"Unsupported file format '{file_format}'"
     except BaseException as e:
-        return False, "error", str(e)
+        return False, "error", 500, "", str(e)
     else:
-        return True, "success", "CSV file has been generated successfully"
+        return True, "success", 200, file_path, f"{file_format.upper()} file has been generated successfully"
diff --git a/src/models/export.py b/src/models/export.py
@@ -7,14 +7,41 @@ class ExportData(BaseModel, extra=Extra.allow):
     start: Optional[str] = None
     end: Optional[str] = None
     step: Optional[str] = None
+    timestamp_format: Optional[str] = "unix"
+    replace_fields: Optional[dict] = dict()
     _request_body_examples = {
-        "Count of successful logins by users per hour in a day": {
+        "User logins per hour in a day": {
             "description": "Count of successful logins by users per hour in a day",
             "value": {
                 "expr": "users_login_count{status='success'}",
                 "start": "2024-01-30T00:00:00Z",
                 "end": "2024-01-31T23:59:59Z",
                 "step": "1h"
             }
+        },
+        "User logins per hour in a day with a user-friendly time format": {
+            "description": "Count of successful user logins per hour in a day with a user-friendly time format",
+            "value": {
+                "expr": "users_login_count{status='success'}",
+                "start": "2024-01-30T00:00:00Z",
+                "end": "2024-01-31T23:59:59Z",
+                "step": "1h",
+                "timestamp_format": "friendly"
+            }
+        },
+        "User logins per hour with friendly time format and custom fields": {
+            "description": "Count of successful user logins per hour in a day "
+                           "with a user-friendly time format and custom fields",
+            "value": {
+                "expr": "users_login_count{status='success'}",
+                "start": "2024-01-30T00:00:00Z",
+                "end": "2024-01-31T23:59:59Z",
+                "step": "1h",
+                "timestamp_format": "friendly",
+                "replace_fields": {
+                    "__name__": "Name",
+                    "timestamp": "Time"
+                }
+            }
         }
     }
diff --git a/src/schemas/export.json b/src/schemas/export.json
@@ -16,6 +16,13 @@
     "step": {
       "type": ["string", "null"],
       "pattern": "^((([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?|0)$"
+    },
+    "timestamp_format": {
+      "type": ["string"],
+      "pattern": "^(unix|iso8601|rfc2822|rfc3339|friendly)$"
+    },
+    "replace_fields": {
+      "type": "object"
     }
   },
   "required": ["expr"],

diff --git a/src/utils/openapi.py b/src/utils/openapi.py
@@ -16,7 +16,7 @@ def openapi(app: FastAPI):
                     "providing additional features and addressing its limitations. "
                     "Running as a sidecar alongside the Prometheus server enables "
                     "users to extend the capabilities of the API.",
-        version="0.4.0",
+        version="0.4.1",
         contact={
             "name": "Hayk Davtyan",
             "url": "https://hayk96.github.io",

diff --git a/ui/homepage/index.html b/ui/homepage/index.html
@@ -117,6 +117,7 @@
             color: #ffffff;
             background-image: linear-gradient(45deg, #f6d365, #fda085);
             box-shadow: 0 3px 6px rgba(0, 0, 0, 0.16);
+            width: 200px;
         }
         button:hover {
             animation: buttonPulse 0.5s ease;
@@ -170,6 +171,7 @@ <h1>The easiest Prometheus management interface</h1>
         <button id="openPrometheusButton">Open Prometheus</button>
         <button id="rulesManagementButton">Rules Management</button>
         <button id="metricsManagementButton">Metrics Management</button>
+        <button id="reportsButton">Reports</button>
     </div>
     <script>
         document.addEventListener('DOMContentLoaded', function() {
@@ -183,6 +185,9 @@ <h1>The easiest Prometheus management interface</h1>
             document.getElementById('metricsManagementButton').onclick = function() {
                 window.location.href = window.location.origin + '/metrics-management';
             };
+            document.getElementById('reportsButton').onclick = function() {
+                window.location.href = window.location.origin + '/reports';
+            };
         });
     </script>
 </body>

diff --git a/ui/metrics-management/index.html b/ui/metrics-management/index.html