Merge branch 'KindlingProject:main' into glcc-rocketmq

.github/workflows/kindling-ci.yaml

@@ -1,4 +1,4 @@
-name: KINDLING-CI
+name: Kindling-agent-CI
 
 on:
   push:

.github/workflows/kindling-front-CI.yml

@@ -0,0 +1,29 @@
+name: Kindling-front-CI
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  build-latest-test:
+    if: github.repository == 'CloudDectective-Harmonycloud/kindling'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Set TAG
+        run: echo "TAG=latesttest" >> $GITHUB_ENV
+      - uses: actions/checkout@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME}}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: ${{ github.workspace }}/camera-front
+          file: ${{ github.workspace }}/camera-front/Dockerfile
+          push: true
+          tags: kindlingproject/kindling-camera-front:${{ env.TAG }}

.gitignore

@@ -4,6 +4,12 @@
 build/
 probe/src/probe/cmake-build-debug/
 probe/build/
+probe/cmake-build-debug
 
+collector/docker/kindling-collector
+collector/docker/libso/libkindling.so
+collector/docker/kindling-falcolib-probe.tar.gz
+
+node_modules
 *.log
 logs

CHANGELOG.md

@@ -3,11 +3,36 @@
 1. All notable changes to this project will be documented in this file.
 2. Records in this file are not identical to the title of their Pull Requests. A detailed description is necessary for understanding what changes are and why they are made.
 
-## Unreleased 
-## Enhancements
+## Unreleased
+### New features
+- 
+- 
+- Add a new tool: A debug tool for Trace Profiling is provided for developers to troubleshoot problems.([#363](https://github.com/CloudDectective-Harmonycloud/kindling/pull/363))
+
+
+### Enhancements
+- 
+- 
+- 
+
+### Bug fixes
+- 
+- 
+- Fix potential deadlock of exited thread delay queue. ([#373](https://github.com/CloudDectective-Harmonycloud/kindling/pull/373))
+- Fix the bug that cpuEvent cache size continuously increases even if trace profiling is not enabled.([#362](https://github.com/CloudDectective-Harmonycloud/kindling/pull/362))
+- Fix the bug that duplicate CPU events are indexed into Elasticsearch. ([#359](https://github.com/KindlingProject/kindling/pull/359))
+- Implement the delay queue for exited thread, so as to avoid losing the data in the period before the thread exits. ([#365](https://github.com/CloudDectective-Harmonycloud/kindling/pull/365))
+- Fix the bug of incomplete records when threads arrive at the cpu analyzer for the first time. ([#364](https://github.com/CloudDectective-Harmonycloud/kindling/pull/364))
+
+## v0.5.0 - 2022-11-02
+### New features
+- Add a new feature: Trace Profiling. See more details about it on our [website](http://kindling.harmonycloud.cn). ([#335](https://github.com/CloudDectective-Harmonycloud/kindling/pull/335))
+
+### Enhancements
 - Add request and response payload of `Redis` protocol message to `Span` data. ([#325](https://github.com/CloudDectective-Harmonycloud/kindling/pull/325))
 
 ### Bug fixes
+- Fix the topology node naming error in the default namespace.([#346](https://github.com/CloudDectective-Harmonycloud/kindling/pull/346))
 - Fix the bug that if `ReadBytes` receives negative numbers as arguments, the program panics with the error of slice outofbound. ([#327](https://github.com/CloudDectective-Harmonycloud/kindling/pull/327))
 
 ## v0.4.1 - 2022-09-21

camera-front/Dockerfile

@@ -0,0 +1,19 @@
+FROM node:16.18.0-alpine3.15 AS appbuild
+RUN npm install -g cnpm --registry=https://registry.npmmirror.com
+WORKDIR /app/camera-front
+COPY package*.json .
+RUN cnpm install
+COPY . .
+RUN npm run build
+
+FROM node:16.18.0-alpine3.15
+RUN npm install -g cnpm --registry=https://registry.npmmirror.com
+RUN cnpm install pm2 -g
+WORKDIR /app/camera-front
+COPY node/ ./node
+RUN chmod +x /app/camera-front/node/start.sh
+WORKDIR /app/camera-front/node
+RUN cnpm install
+WORKDIR /app/camera-front
+COPY --from=appbuild /app/camera-front/dist ./dist
+CMD ["/app/camera-front/node/start.sh"]

camera-front/README.md

@@ -0,0 +1,10 @@
+# The Camera Front-end
+## How to build the image
+```
+docker build -t camera-front .
+```
+
+## How to run the image
+```
+docker run -d --rm -p 9504:9504 camera-front:latest 
+```

camera-front/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/src/favicon.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Kindling APP</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

camera-front/main.d.ts

@@ -0,0 +1,7 @@
+declare module 'rc-calendar/lib/RangeCalendar';
+declare module 'rc-calendar/lib/locale/zh_CN';
+
+declare module 'rc-time-picker/lib/Panel';
+
+declare module 'd3-tip';
+declare module 'd3-flame-graph';

camera-front/node/index.js

@@ -0,0 +1,75 @@
+'use strict';
+
+const express = require('express');
+const app = express();
+const http = require('http');
+const serveStatic = require('serve-static');
+const bodyParser = require('body-parser');
+const _ = require('lodash');
+const async = require('async');
+const path = require('path');
+const compression = require('compression');
+const history = require('connect-history-api-fallback');
+const { createProxyMiddleware } = require('http-proxy-middleware');
+
+const settings = require('./settings');
+const port = settings.port;
+
+const restream = function (proxyReq, req) {
+    let masterIp = req.headers.authorization;
+    proxyReq.setHeader('Authorization', masterIp || '');
+};
+const esserverAdress = 'http://' + settings.apmServerConfig.host + ':' + settings.apmServerConfig.port;
+const esserverProxy = createProxyMiddleware('/camera', {
+    target: esserverAdress,
+    secure: false,
+    changeOrigin: true,
+    // onProxyReq: restream,
+    // onProxyRes: async function (proxyRes, req, res) {
+    //     // console.log(proxyRes, 'proxyRes');
+    //     if (typeof req.session.username === 'undefined') {
+    //         req.session.destroy();
+    //         res.status(401).json({
+    //             msg: noAuthorizeMsg,
+    //         });
+    //     }
+    // },
+});
+app.use(esserverProxy);
+const profileAddress = 'http://' + settings.profileConfig.host + ':' + settings.profileConfig.port;
+const profileProxy = createProxyMiddleware('/profile', {
+    target: profileAddress,
+    secure: false,
+    changeOrigin: true,
+});
+app.use(profileProxy);
+
+
+app.use(
+    bodyParser.urlencoded({
+        extended: true,
+    })
+);
+app.use(bodyParser.json());
+app.use(compression());
+// app.use(history());
+
+app.use(express.static('../dist'));
+console.log(__dirname);
+// app.set('views', path.join(__dirname, 'dist'));
+// app.use(serveStatic(path.join(__dirname, 'dist')));
+
+app.get("/test", function(req, res, next) {
+    // res.render("test.html");
+    res.status(200).json({
+        test: true
+    });
+});
+
+app.use('/file', require('./routers/file.js'));
+
+var httpServer = http.createServer(app);
+// var httpsServer = https.createServer(credentials, app);
+httpServer.listen(port, function() {
+    console.log('Express http server listening on port ' + port);
+});

camera-front/node/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "camera_node",
+  "version": "1.0.0",
+  "description": "node for kindling camera",
+  "main": "index.js",
+  "author": "thousand",
+  "license": "MIT",
+  "dependencies": {
+    "async": "^3.2.4",
+    "compression": "^1.7.4",
+    "connect-history-api-fallback": "^2.0.0",
+    "express": "^4.18.2",
+    "http-proxy-middleware": "^2.0.6",
+    "lodash": "^4.17.21"
+  }
+}

camera-front/node/routers/file.js

@@ -0,0 +1,147 @@
+
+'use strict';
+const router = require('express').Router();
+const _ = require('lodash');
+const fs = require('fs');
+
+const settings = require('../settings');
+const basicFloder = settings.traceFilePath;
+
+router.get("/getFolders", function(req, res, next) {
+    let list = [], result = [];
+    fs.readdir(basicFloder, function(err, files) {
+        if (err) {
+            console.error("Error: ", err);
+            res.status(500).json({
+                success: false,
+                message: err
+            })
+        }
+        _.forEach(files, file => {
+            let fileStats = fs.statSync(basicFloder + "/" + file);
+            if (fileStats.isDirectory()) {
+                let fileNameList = file.split('_');
+                if (fileNameList.length === 4) {
+                    list.push(fileNameList);
+                }
+            }
+        });
+        let workloadlist = _.groupBy(list, item => item[0]);
+        _.forEach(workloadlist, (list, workload) => {
+            let workloadObj = {
+                title: workload,
+                value: workload,
+                selectable: false,
+                children: []
+            };
+            let podList = _.groupBy(list, item => item[1]);
+            _.forEach(podList, (list2, pod) => {
+                let podObj = {
+                    title: pod,
+                    value: `${workload}_${pod}`,
+                    selectable: false,
+                    children: []
+                };
+                _.forEach(list2, item => {
+                    podObj.children.push({
+                        title: `${item[2]}(${item[3]})`,
+                        value: _.join(item, '_')
+                    });
+                });
+                workloadObj.children.push(podObj);
+            });
+            result.push(workloadObj);
+        });
+        // console.log(result);
+        res.status(200).json({
+            success: true,
+            data: result
+        });
+    });
+}); 
+
+router.get("/getAllTraceFileList", function(req, res, next) {
+    let folderName = req.query.folderName;
+    const filePath = basicFloder + "/" + folderName;
+    let list = [];
+	try {
+        let files = fs.readdirSync(filePath);
+        _.forEach(files, function(file) {
+            let fileStats = fs.statSync(filePath + "/" + file);
+            if (fileStats.isFile) {
+                let fileNameList = file.split('_');
+                let contentKeyBuffer = new Buffer.from(fileNameList[1], 'base64')
+                let contentKey = contentKeyBuffer.toString();
+                let newList = [].concat(fileNameList);
+                newList[1] = contentKey;
+                let showFileName = _.join(newList, '_');
+                list.push({
+                    fileName: file,
+                    showFileName,
+                    contentKey,
+                    ctime: new Date(fileStats.ctime).getTime()
+                });
+            }
+        });
+        list = _.sortBy(list, 'ctime');
+    } catch (err) {
+        console.error(err);
+    }
+    // console.log(list);
+	res.status(200).json({
+		"success": true,
+        "data": list
+	});
+});
+
+router.get('/getTraceFile', function(req, res, next) {
+    let fileName = req.query.fileName;
+    let folderName = req.query.folderName;
+    const filePath = basicFloder + '/' + folderName + '/' + fileName;
+    console.log(fileName, filePath)
+    fs.readFile(filePath, function(err, buffer) {
+        if (err) {
+            console.error("Error: ", err);
+            res.status(500).json({
+                success: false,
+                message: err
+            });
+        }
+        let result = buffer.toString();
+        let resList = result.split('------');
+        let traceData = JSON.parse(_.head(resList));
+        let cpuEventStrs = _.slice(resList, 1);
+        let cpuEventsList = [];
+        let cpuEvents = [];
+
+        _.forEach(cpuEventStrs, (str) => {
+            try {
+                cpuEventsList.push(JSON.parse(str))
+            } catch (error) {
+                console.error(str);
+            }
+        });
+        cpuEvents = _.map(cpuEventsList, 'labels');
+        _.forEach(cpuEvents, item => {
+            try {
+                item.cpuEvents = JSON.parse(item.cpuEvents);
+                item.javaFutexEvents = JSON.parse(item.javaFutexEvents);
+                item.transactionIds = JSON.parse(item.transactionIds);
+            } catch (error) {
+                console.error(error, item);
+            }
+        });
+
+        let finalResult = {
+            trace: traceData,
+            cpuEvents: cpuEvents
+        };
+
+        res.status(200).json({
+            "success": true,
+            "data": finalResult
+        });
+    });
+});
+
+module.exports = router;