Skip to content

Commit

Permalink
[Feature-14138][Metrics] Add metrics for api server (#14177)
Browse files Browse the repository at this point in the history
  • Loading branch information
rickchengx authored May 23, 2023
1 parent 0ab2447 commit 5c4ba41
Show file tree
Hide file tree
Showing 8 changed files with 762 additions and 7 deletions.
8 changes: 6 additions & 2 deletions docs/docs/en/guide/metrics/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ cd dolphinscheduler-meter/src/main/resources/grafana-demo
docker compose up
```

then access the `Grafana` by the url: `http://localhost/3001` for dashboards.
then access the `Grafana` by the url: `http://localhost:3001` for dashboards.

![image.png](../../../../img/metrics/metrics-master.png)
![image.png](../../../../img/metrics/metrics-worker.png)
Expand Down Expand Up @@ -111,7 +111,11 @@ For example, you can get the master metrics by `curl http://localhost:5679/actua

### Api Server Metrics

- Currently, we have not embedded any metrics in Api Server.
- ds.api.request.count: (counter) the number of requests received by the api server
- ds.api.response.count: (counter) the number of responses received by the api server, sliced by tag `code`
- ds.api.response.time: (histogram) the response time distribution of the api server
- ds.api.resource.upload.size: (histogram) size distribution of resource files uploaded by the api server (bytes)
- ds.api.resource.download.size: (histogram) size distribution of resource files download by the api server (bytes)

### Alert Server Related

Expand Down
8 changes: 6 additions & 2 deletions docs/docs/zh/guide/metrics/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ cd dolphinscheduler-meter/src/main/resources/grafana-demo
docker compose up
```

然后,您即可通过http://localhost/3001`链接访问`Grafana`面板。
然后,您即可通过`http://localhost:3001`链接访问`Grafana`面板。

![image.png](../../../../img/metrics/metrics-master.png)
![image.png](../../../../img/metrics/metrics-worker.png)
Expand Down Expand Up @@ -111,7 +111,11 @@ metrics exporter端口`server.port`是在application.yaml里定义的: master: `

### Api Server指标

- 目前我们尚未提供任何Api Server指标
- ds.api.request.count: (counter) api请求次数
- ds.api.response.count: (counter) api响应次数,可由标签`code`切分
- ds.api.response.time: (histogram) api响应时间分布
- ds.api.resource.upload.size: (histogram) api上传资源文件大小的分布(bytes)
- ds.api.resource.download.size: (histogram) api下载资源文件大小的分布(bytes)

### Alert Server指标

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.dolphinscheduler.api.aspect;

import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.dao.entity.User;
Expand Down Expand Up @@ -110,7 +111,9 @@ public Object doAround(ProceedingJoinPoint proceedingJoinPoint) throws Throwable

Object ob = proceedingJoinPoint.proceed();

log.info("Call {}:{} success, cost: {}ms", requestMethod, URI, (System.currentTimeMillis() - startTime));
long costTime = System.currentTimeMillis() - startTime;
log.info("Call {}:{} success, cost: {}ms", requestMethod, URI, costTime);
ApiServerMetrics.recordApiResponseTime(costTime);

return ob;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.api.interceptor;

import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.api.security.Authenticator;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
Expand Down Expand Up @@ -61,6 +62,8 @@ public class LoginHandlerInterceptor implements HandlerInterceptor {
*/
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
ApiServerMetrics.incApiRequestCount();

// get token
String token = request.getHeader("token");
User user;
Expand Down Expand Up @@ -96,5 +99,16 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons
public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler,
ModelAndView modelAndView) throws Exception {
ThreadLocalContext.getTimezoneThreadLocal().remove();

int code = response.getStatus();
if (code >= 200 && code < 300) {
ApiServerMetrics.incApiResponse2xxCount();
} else if (code >= 300 && code < 400) {
ApiServerMetrics.incApiResponse3xxCount();
} else if (code >= 400 && code < 500) {
ApiServerMetrics.incApiResponse4xxCount();
} else if (code >= 500 && code < 600) {
ApiServerMetrics.incApiResponse5xxCount();
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.dolphinscheduler.api.metrics;

import lombok.experimental.UtilityClass;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Metrics;

@UtilityClass
public class ApiServerMetrics {

private final Counter apiRequestCounter =
Counter.builder("ds.api.request.count")
.description("Api request count")
.register(Metrics.globalRegistry);

private final Counter apiResponse2xxCounter =
Counter.builder("ds.api.response.count")
.tag("code", "2xx")
.description("Api 2xx response count")
.register(Metrics.globalRegistry);

private final Counter apiResponse3xxCounter =
Counter.builder("ds.api.response.count")
.tag("code", "3xx")
.description("Api 3xx response count")
.register(Metrics.globalRegistry);

private final Counter apiResponse4xxCounter =
Counter.builder("ds.api.response.count")
.tag("code", "4xx")
.description("Api 4xx response count")
.register(Metrics.globalRegistry);

private final Counter apiResponse5xxCounter =
Counter.builder("ds.api.response.count")
.tag("code", "5xx")
.description("Api 5xx response count")
.register(Metrics.globalRegistry);

private final DistributionSummary apiResourceUploadSizeDistribution =
DistributionSummary.builder("ds.api.resource.upload.size")
.baseUnit("bytes")
.publishPercentiles(0.5, 0.75, 0.95, 0.99)
.publishPercentileHistogram()
.description("size of upload resource files on api")
.register(Metrics.globalRegistry);

private final DistributionSummary apiResourceDownloadSizeDistribution =
DistributionSummary.builder("ds.api.resource.download.size")
.baseUnit("bytes")
.publishPercentiles(0.5, 0.75, 0.95, 0.99)
.publishPercentileHistogram()
.description("size of download resource files on api")
.register(Metrics.globalRegistry);

private final DistributionSummary apiResponseTimeDistribution =
DistributionSummary.builder("ds.api.response.time")
.baseUnit("milliseconds")
.publishPercentiles(0.5, 0.75, 0.95, 0.99)
.publishPercentileHistogram()
.description("response time on api")
.register(Metrics.globalRegistry);

public void incApiRequestCount() {
apiRequestCounter.increment();
}

public void incApiResponse2xxCount() {
apiResponse2xxCounter.increment();
}

public void incApiResponse3xxCount() {
apiResponse3xxCounter.increment();
}

public void incApiResponse4xxCount() {
apiResponse4xxCounter.increment();
}

public void incApiResponse5xxCount() {
apiResponse5xxCounter.increment();
}

public void recordApiResourceUploadSize(final long size) {
apiResourceUploadSizeDistribution.record(size);
}

public void recordApiResourceDownloadSize(final long size) {
apiResourceDownloadSizeDistribution.record(size);
}

public void recordApiResponseTime(final long milliseconds) {
apiResponseTimeDistribution.record(milliseconds);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
Expand Down Expand Up @@ -64,6 +65,7 @@

import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.text.MessageFormat;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
Expand Down Expand Up @@ -277,8 +279,9 @@ public Result<Object> createResource(User loginUser,
throw new ServiceException(
String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename()));
} else
log.info("Upload resource file complete, resourceName:{}, fileName:{}.",
RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename()));
ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
log.info("Upload resource file complete, resourceName:{}, fileName:{}.",
RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename()));
return result;
}

Expand Down Expand Up @@ -467,6 +470,7 @@ public Result<Object> updateResource(User loginUser,
}
}

ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
return result;
}

Expand Down Expand Up @@ -1143,6 +1147,8 @@ public Result<Object> readResource(User loginUser, String fullName, String resTe
try {
if (storageOperate.exists(fullName)) {
content = storageOperate.vimFile(tenantCode, fullName, skipLineNum, limit);
long size = content.stream().mapToLong(String::length).sum();
ApiServerMetrics.recordApiResourceDownloadSize(size);
} else {
log.error("read file {} not exist in storage", fullName);
putMsg(result, Status.RESOURCE_FILE_NOT_EXIST, fullName);
Expand Down Expand Up @@ -1463,6 +1469,7 @@ public org.springframework.core.io.Resource downloadResource(User loginUser,

try {
storageOperate.download(tenantCode, fullName, localFileName, true);
ApiServerMetrics.recordApiResourceDownloadSize(java.nio.file.Files.size(Paths.get(localFileName)));
return org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName);
} catch (IOException e) {
log.error("Download resource error, the path is {}, and local filename is {}, the error message is {}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@
import org.apache.commons.collections4.CollectionUtils;

import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -127,6 +129,10 @@ public class ResourcesServiceTest {

private MockedStatic<PropertyUtils> mockedStaticPropertyUtils;

private MockedStatic<Paths> mockedStaticPaths;

private MockedStatic<java.nio.file.Files> filesMockedStatic;

private Throwable exception;

@BeforeEach
Expand All @@ -137,6 +143,8 @@ public void setUp() {
Mockito.mockStatic(org.apache.dolphinscheduler.api.utils.FileUtils.class);

mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
mockedStaticPaths = Mockito.mockStatic(Paths.class);
filesMockedStatic = Mockito.mockStatic(java.nio.file.Files.class);
}

@AfterEach
Expand All @@ -145,6 +153,8 @@ public void after() {
mockedStaticFiles.close();
mockedStaticDolphinschedulerFileUtils.close();
mockedStaticPropertyUtils.close();
mockedStaticPaths.close();
filesMockedStatic.close();
}

@Test
Expand Down Expand Up @@ -668,7 +678,10 @@ public void testDownloadResource() {
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class);
Path path = Mockito.mock(Path.class);
Mockito.when(Paths.get(Mockito.any())).thenReturn(path);
try {
Mockito.when(java.nio.file.Files.size(Mockito.any())).thenReturn(1L);
// resource null
org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), "");
Assertions.assertNull(resource);
Expand Down
Loading

0 comments on commit 5c4ba41

Please sign in to comment.