Merge branch 'master' into refact-base

apache · Nov 6, 2023 · a2fd4c5 · a2fd4c5
2 parents d420cd7 + 69e6b46
commit a2fd4c5
Show file tree

Hide file tree

Showing 17 changed files with 251 additions and 15 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,7 +19,7 @@ jobs:
       HEAD_BRANCH_NAME: ${{ github.head_ref }}
       BASE_BRANCH_NAME: ${{ github.base_ref }}
       TARGET_BRANCH_NAME: ${{ github.base_ref != '' && github.base_ref || github.ref_name }}
-      RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') || startsWith(github.base_ref, 'release-') }}
+      RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') }}
 
     strategy:
       fail-fast: false
@@ -42,7 +42,7 @@ jobs:
           restore-keys: ${{ runner.os }}-m2
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 2
 
@@ -88,6 +88,6 @@ jobs:
           $TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND tinkerpop
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3.0.0
+        uses: codecov/codecov-action@v3
         with:
           file: ${{ env.REPORT_DIR }}/*.xml
diff --git a/hugegraph-server/hugegraph-api/pom.xml b/hugegraph-server/hugegraph-api/pom.xml
@@ -151,7 +151,7 @@
         <dependency>
             <groupId>io.swagger.core.v3</groupId>
             <artifactId>swagger-jaxrs2-jakarta</artifactId>
-            <version>2.1.9</version>
+            <version>${swagger.version}</version>
         </dependency>
 
         <dependency>

diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/arthas/ArthasAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/arthas/ArthasAPI.java
@@ -27,6 +27,9 @@
 import com.codahale.metrics.annotation.Timed;
 import com.taobao.arthas.agent.attach.ArthasAgent;
 
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
 import jakarta.inject.Singleton;
 import jakarta.ws.rs.PUT;
 import jakarta.ws.rs.Path;
@@ -35,6 +38,7 @@
 
 @Path("arthas")
 @Singleton
+@Tag(name = "ArthasAPI")
 public class ArthasAPI extends API {
 
     @Context
@@ -43,6 +47,7 @@ public class ArthasAPI extends API {
     @PUT
     @Timed
     @Produces(APPLICATION_JSON_WITH_CHARSET)
+    @Operation(summary = "start arthas agent")
     public Object startArthas() {
         HugeConfig config = this.configProvider.get();
         HashMap<String, String> configMap = new HashMap<>(4);

diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/cypher/CypherAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/cypher/CypherAPI.java
@@ -33,6 +33,8 @@
 
 import com.codahale.metrics.annotation.Timed;
 
+import io.swagger.v3.oas.annotations.tags.Tag;
+
 import jakarta.inject.Singleton;
 import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.GET;
@@ -47,6 +49,7 @@
 
 @Path("graphs/{graph}/cypher")
 @Singleton
+@Tag(name = "CypherAPI")
 public class CypherAPI extends API {
 
     private static final Logger LOG = Log.logger(CypherAPI.class);

diff --git a/...h-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/...h-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
@@ -17,6 +17,7 @@
 
 package org.apache.hugegraph.api.filter;
 
+import static org.apache.hugegraph.api.filter.PathFilter.REQUEST_PARAMS_JSON;
 import static org.apache.hugegraph.api.filter.PathFilter.REQUEST_TIME;
 import static org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_FAILED_COUNTER;
 import static org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_RESPONSE_TIME_HISTOGRAM;
@@ -25,12 +26,20 @@
 
 import java.io.IOException;
 
+import org.apache.hugegraph.config.HugeConfig;
+import org.apache.hugegraph.config.ServerOptions;
 import org.apache.hugegraph.metrics.MetricsUtil;
+import org.apache.hugegraph.metrics.SlowQueryLog;
+import org.apache.hugegraph.util.JsonUtil;
+import org.apache.hugegraph.util.Log;
+import org.slf4j.Logger;
 
 import jakarta.inject.Singleton;
+import jakarta.ws.rs.HttpMethod;
 import jakarta.ws.rs.container.ContainerRequestContext;
 import jakarta.ws.rs.container.ContainerResponseContext;
 import jakarta.ws.rs.container.ContainerResponseFilter;
+import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.ext.Provider;
 
 
@@ -39,6 +48,14 @@
 public class AccessLogFilter implements ContainerResponseFilter {
 
     private static final String DELIMETER = "/";
+    private static final String GRAPHS = "graphs";
+    private static final String GREMLIN = "gremlin";
+    private static final String CYPHER = "cypher";
+
+    private static final Logger LOG = Log.logger(AccessLogFilter.class);
+
+    @Context
+    private jakarta.inject.Provider<HugeConfig> configProvider;
 
     /**
      * Use filter to log request info
@@ -62,13 +79,24 @@ public void filter(ContainerRequestContext requestContext, ContainerResponseCont
 
         // get responseTime
         Object requestTime = requestContext.getProperty(REQUEST_TIME);
-        if(requestTime!=null){
+        if(requestTime != null){
             long now = System.currentTimeMillis();
-            long responseTime = (now - (long)requestTime);
+            long start = (Long) requestTime;
+            long responseTime = now - start;
 
             MetricsUtil.registerHistogram(
                                join(metricsName, METRICS_PATH_RESPONSE_TIME_HISTOGRAM))
                        .update(responseTime);
+
+            HugeConfig config = configProvider.get();
+            long timeThreshold = config.get(ServerOptions.SLOW_QUERY_LOG_TIME_THRESHOLD);
+
+            // record slow query log
+            if (timeThreshold > 0 && isSlowQueryLogWhiteAPI(requestContext) && responseTime > timeThreshold) {
+                SlowQueryLog log = new SlowQueryLog(responseTime, start, (String) requestContext.getProperty(REQUEST_PARAMS_JSON),
+                                                    method, timeThreshold, path);
+                LOG.info("Slow query: {}", JsonUtil.toJson(log));
+            }
         }
     }
 
@@ -79,4 +107,18 @@ private String join(String path1, String path2) {
     private boolean statusOk(int status){
         return status == 200 || status == 201 || status == 202;
     }
+
+    public static boolean isSlowQueryLogWhiteAPI(ContainerRequestContext context) {
+        String path = context.getUriInfo().getPath();
+        String method = context.getRequest().getMethod();
+
+        // GraphsAPI/CypherAPI/Job GremlinAPI
+        if (path.startsWith(GRAPHS)) {
+            if (method.equals(HttpMethod.GET) || path.endsWith(CYPHER) || path.endsWith(GREMLIN) ){
+                return true;
+            }
+        }
+        // Raw GremlinAPI
+        return path.startsWith(GREMLIN);
+    }
 }
diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/PathFilter.java
@@ -17,12 +17,20 @@
 
 package org.apache.hugegraph.api.filter;
 
+import static org.apache.hugegraph.api.API.CHARSET;
+
 import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.io.Charsets;
+import org.apache.commons.io.IOUtils;
 
 import jakarta.inject.Singleton;
+import jakarta.ws.rs.HttpMethod;
 import jakarta.ws.rs.container.ContainerRequestContext;
 import jakarta.ws.rs.container.ContainerRequestFilter;
 import jakarta.ws.rs.container.PreMatching;
+import jakarta.ws.rs.core.MultivaluedMap;
 import jakarta.ws.rs.ext.Provider;
 
 @Provider
@@ -31,10 +39,26 @@
 public class PathFilter implements ContainerRequestFilter {
 
     public static final String REQUEST_TIME = "request_time";
+    public static final String REQUEST_PARAMS_JSON = "request_params_json";
 
     @Override
     public void filter(ContainerRequestContext context)
             throws IOException {
         context.setProperty(REQUEST_TIME, System.currentTimeMillis());
+
+        // record the request json
+        String method = context.getMethod();
+        String requestParamsJson = "";
+        if (method.equals(HttpMethod.POST)) {
+            requestParamsJson = IOUtils.toString(context.getEntityStream(), Charsets.toCharset(CHARSET));
+            // replace input stream because we have already read it
+            InputStream in = IOUtils.toInputStream(requestParamsJson, Charsets.toCharset(CHARSET));
+            context.setEntityStream(in);
+        } else if(method.equals(HttpMethod.GET)){
+            MultivaluedMap<String, String> pathParameters = context.getUriInfo().getPathParameters();
+            requestParamsJson = pathParameters.toString();
+        }
+
+        context.setProperty(REQUEST_PARAMS_JSON, requestParamsJson);
     }
 }
diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/job/AlgorithmAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/job/AlgorithmAPI.java
@@ -36,6 +36,8 @@
 import com.codahale.metrics.annotation.Timed;
 import com.google.common.collect.ImmutableMap;
 
+import io.swagger.v3.oas.annotations.tags.Tag;
+
 import jakarta.inject.Singleton;
 import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.NotFoundException;
@@ -47,6 +49,7 @@
 
 @Path("graphs/{graph}/jobs/algorithm")
 @Singleton
+@Tag(name = "AlgorithmAPI")
 public class AlgorithmAPI extends API {
 
     private static final Logger LOG = Log.logger(RestServer.class);

diff --git a/...graph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/metrics/MetricsAPI.java b/...graph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/metrics/MetricsAPI.java
@@ -70,7 +70,9 @@
 import com.codahale.metrics.Metric;
 import com.codahale.metrics.annotation.Timed;
 
+import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.tags.Tag;
+
 import jakarta.annotation.security.RolesAllowed;
 import jakarta.inject.Singleton;
 import jakarta.ws.rs.GET;
@@ -103,6 +105,7 @@ public MetricsAPI() {
     @Path("system")
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed({"admin", "$owner= $action=metrics_read"})
+    @Operation(summary = "get the system metrics")
     public String system() {
         return JsonUtil.toJson(this.systemMetrics.metrics());
     }
@@ -112,6 +115,7 @@ public String system() {
     @Path("backend")
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed({"admin", "$owner= $action=metrics_read"})
+    @Operation(summary = "get the backend metrics")
     public String backend(@Context GraphManager manager) {
         Map<String, Map<String, Object>> results = InsertionOrderUtil.newMap();
         for (String graph : manager.graphs()) {
@@ -134,6 +138,7 @@ public String backend(@Context GraphManager manager) {
     @Path("gauges")
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed({"admin", "$owner= $action=metrics_read"})
+    @Operation(summary = "get the gauges metrics")
     public String gauges() {
         ServerReporter reporter = ServerReporter.instance();
         return JsonUtil.toJson(reporter.gauges());
@@ -144,6 +149,7 @@ public String gauges() {
     @Path("counters")
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed({"admin", "$owner= $action=metrics_read"})
+    @Operation(summary = "get the counters metrics")
     public String counters() {
         ServerReporter reporter = ServerReporter.instance();
         return JsonUtil.toJson(reporter.counters());
@@ -154,6 +160,7 @@ public String counters() {
     @Path("histograms")
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed({"admin", "$owner= $action=metrics_read"})
+    @Operation(summary = "get the histograms metrics")
     public String histograms() {
         ServerReporter reporter = ServerReporter.instance();
         return JsonUtil.toJson(reporter.histograms());
@@ -164,6 +171,7 @@ public String histograms() {
     @Path("meters")
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed({"admin", "$owner= $action=metrics_read"})
+    @Operation(summary = "get the meters metrics")
     public String meters() {
         ServerReporter reporter = ServerReporter.instance();
         return JsonUtil.toJson(reporter.meters());
@@ -174,6 +182,7 @@ public String meters() {
     @Path("timers")
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed({"admin", "$owner= $action=metrics_read"})
+    @Operation(summary = "get the timers metrics")
     public String timers() {
         ServerReporter reporter = ServerReporter.instance();
         return JsonUtil.toJson(reporter.timers());
@@ -183,6 +192,7 @@ public String timers() {
     @Timed
     @Produces(APPLICATION_TEXT_WITH_CHARSET)
     @RolesAllowed({"admin", "$owner= $action=metrics_read"})
+    @Operation(summary = "get all base metrics")
     public String all(@Context GraphManager manager,
                       @QueryParam("type") String type) {
         if (type != null && type.equals(JSON_STR)) {
@@ -197,6 +207,7 @@ public String all(@Context GraphManager manager,
     @Timed
     @Produces(APPLICATION_TEXT_WITH_CHARSET)
     @RolesAllowed({"admin", "$owner= $action=metrics_read"})
+    @Operation(summary = "get all statistics metrics")
     public String statistics(@QueryParam("type") String type) {
         Map<String, Map<String, Object>> metricMap = statistics();
 

diff --git a/...h-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/profile/WhiteIpListAPI.java b/...h-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/profile/WhiteIpListAPI.java
@@ -37,6 +37,8 @@
 import com.codahale.metrics.annotation.Timed;
 import com.google.common.collect.ImmutableMap;
 
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.tags.Tag;
 import jakarta.annotation.security.RolesAllowed;
 import jakarta.inject.Singleton;
 import jakarta.ws.rs.Consumes;
@@ -50,6 +52,7 @@
 
 @Path("whiteiplist")
 @Singleton
+@Tag(name = "WhiteIpListAPI")
 public class WhiteIpListAPI extends API {
 
     private static final Logger LOG = Log.logger(WhiteIpListAPI.class);
@@ -58,6 +61,7 @@ public class WhiteIpListAPI extends API {
     @Timed
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed("admin")
+    @Operation(summary = "list white ips")
     public Map<String, Object> list(@Context GraphManager manager) {
         LOG.debug("List white ips");
         AuthManager authManager = manager.authManager();
@@ -71,6 +75,7 @@ public Map<String, Object> list(@Context GraphManager manager) {
     @Consumes(APPLICATION_JSON)
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed("admin")
+    @Operation(summary = "update white ip list")
     public Map<String, Object> updateWhiteIPs(@Context GraphManager manager, Map<String, Object> actionMap) {
         E.checkArgument(actionMap != null,
                         "Missing argument: actionMap");
@@ -131,6 +136,7 @@ public Map<String, Object> updateWhiteIPs(@Context GraphManager manager, Map<Str
     @Timed
     @Produces(APPLICATION_JSON_WITH_CHARSET)
     @RolesAllowed("admin")
+    @Operation(summary = "enable/disable the white ip list")
     public Map<String, Object> updateStatus(@Context GraphManager manager, @QueryParam("status") String status) {
         LOG.debug("Enable or disable white ip list");
         E.checkArgument("true".equals(status) ||

diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/config/ServerOptions.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/config/ServerOptions.java
@@ -304,4 +304,13 @@ public static synchronized ServerOptions instance() {
             null,
             "jad"
         );
+
+    public static final ConfigOption<Long> SLOW_QUERY_LOG_TIME_THRESHOLD =
+            new ConfigOption<>(
+                    "log.slow_query_threshold",
+                    "The threshold time(ms) of logging slow query, " +
+                    "0 means logging slow query is disabled.",
+                    nonNegativeInt(),
+                    1000L
+            );
 }